def import_accidents(provider_code, accidents, streets, roads):
print("reading accidents from file %s" % (accidents.name(), ))
for accident in accidents:
if field_names.x_coordinate not in accident or field_names.y_coordinate not in accident:
raise ValueError(
"x and y coordinates are missing from the accidents file!")
if not accident[field_names.x_coordinate] or not accident[
field_names.y_coordinate]:
continue
lng, lat = coordinates_converter.convert(
accident[field_names.x_coordinate],
accident[field_names.y_coordinate])
marker = Marker(
user=None,
id=int("{0}{1}".format(provider_code, accident[field_names.id])),
title="Accident",
description=json.dumps(load_extra_data(accident, streets, roads),
encoding=models.db_encoding),
address=get_address(accident, streets),
latitude=lat,
longitude=lng,
type=Marker.MARKER_TYPE_ACCIDENT,
subtype=int(accident[field_names.accident_type]),
severity=int(accident[field_names.accident_severity]),
created=parse_date(accident),
locationAccuracy=int(accident[field_names.igun]),
)
yield marker
accidents.close()
def load_markers(self):
marker_data = self.__read_csv("marker")
#load marker
self.marker = []
for item in marker_data:
m = Marker()
self.__fill_members_with_csv_data(
m, ('spot_id', 'qr', 'nfc', 'eddystone', 'ibeacon_region_uid',
'ibeacon_major', 'ibeacon_minor'), item)
self.marker.append(m)
def addmarker(request):
if request.method == 'POST':
form = MarkerForm(request.POST)
if form.is_valid():
marker = Marker()
marker.phone = form.cleaned_data.get("phone")
marker.latitude = form.cleaned_data.get("latitude", "32.0")
marker.longitude = form.cleaned_data.get("longitude", "118.0")
marker.message = form.cleaned_data.get("message", "")
marker.date = datetime.now()
marker.save()
return HttpResponseRedirect(reverse('location:index'))
else:
form = MarkerForm()
return render_to_response('location/addmarker.html', {'form': form},
context_instance=RequestContext(request))
def import_to_datastore():
from models import User, Marker
i = 0
session = db_session()
commit_every = 500
for irow, data in enumerate(import_data()):
show_progress_spinner()
marker = Marker(
user=None,
title="Accident",
description=data["description"].decode("utf8"),
address=data["address"].decode("utf8"),
latitude=data["lat"],
longitude=data["lng"],
type=Marker.MARKER_TYPE_ACCIDENT,
subtype=data["severity"],
created=data["date"],
)
session.add(marker)
if irow > 0 and irow % commit_every == 0:
flush_and_commit(session)
import os
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base
import datetime
from constants import CONST
db_connection_string = os.environ.get('CLEARDB_DATABASE_URL')
print "using connection string: %s" % db_connection_string
engine = create_engine(db_connection_string, convert_unicode=True, echo=True)
autocommit = False # Autocommit does not seem to work
db_session = sessionmaker(autocommit=autocommit, autoflush=True, bind=engine)
Base = declarative_base()
session = db_session()
from models import Marker
marker = Marker(
user=None,
title="Accident",
description="sample accident",
address="sample address",
latitude="00.33",
longitude="00.22",
type=CONST.MARKER_TYPE_ACCIDENT,
subtype="",
created=datetime.datetime.now(),
)
session.add(marker)
if not autocommit:
session.commit()
def parse_xml(self, path, module):
logger.debug("parsing xml: %s", path)
# lookup tables
lookup = {}
lookup["encounter"] = {}
lookup["page"] = {}
lookup["map"] = {}
lookup["image"] = {}
lookup["npc"] = {}
lookup["quest"] = {}
# arrays
pages = []
maps = []
groups = []
encounters = []
# xml tree
tree = ElementTree.parse(path)
root = tree.getroot()
# NPCS
logger.info("parsing npcs")
for category in root.findall("./npc/category"):
for node in category.findall("*"):
tag = node.tag
name = node.find("name").text
npc = NPC()
npc.name = name
lookup["npc"][tag] = npc
# PAGES
logger.info("parsing pages")
parent = Group()
parent.name = "Story"
parent.slug = slugify(parent.name)
groups.append(parent)
for category in root.findall("./encounter/category"):
group = Group()
group.name = category.get("name")
group.slug = slugify(group.name)
group.parent = parent
if group.name == None or group.name == "":
group = parent
else:
groups.append(group)
# get all pages
for node in category.findall("*"):
# tag
tag = node.tag
# create page
page = Page()
page.meta["tag"] = tag
page.name = node.find("name").text
page.slug = slugify(page.name)
page.content = ElementTree.tostring(
node.find("text"), encoding='utf-8',
method='xml').decode('utf-8')
page.parent = group
pages.append(page)
lookup["page"][tag] = page
# QUESTS
logger.info("parsing quests")
parent = Group()
parent.name = "Quests"
parent.slug = slugify(parent.name)
groups.append(parent)
# some modules got, so use this instead
for node in root.findall("./quest/*/*"):
# for node in root.findall("./quest/*"):
# tag
tag = node.tag
# create quest
page = Page()
page.meta["tag"] = id
page.name = node.find("name").text
page.slug = slugify(page.name)
page.content = ElementTree.tostring(node.find("description"),
encoding='utf-8',
method='xml').decode('utf-8')
cr = node.find("cr").text if node.find("cr") else ""
xp = node.find("xp").text if node.find("xp") else ""
page.content += '<p><strong>CR:</strong> ' + cr + ' <strong>XP:</strong> ' + xp + '</p>'
page.parent = parent
pages.append(page)
lookup["quest"][tag] = page
# sort
pages_sorted = humansorted(pages, key=lambda x: x.name)
# MAPS & IMAGES
logger.info("parsing images and maps")
parent = Group()
parent.name = "Maps & Images"
parent.slug = slugify(parent.name)
groups.append(parent)
for category in root.findall("./image/category"):
group = Group()
group.name = category.get("name")
group.slug = slugify(group.name)
group.parent = parent
if group.name == None or group.name == "":
group = parent
else:
groups.append(group)
for node in category.findall("*"):
# tag
tag = node.tag
# create image
image = Image()
image.tag = tag
image.bitmap = node.find("./image/bitmap").text.replace(
"\\", "/")
image.name = node.find("name").text
lookup["image"][tag] = image
markers = []
# get shortcouts (markers)
for shortcut in node.findall("./image/shortcuts/shortcut"):
# create marker
marker = Marker()
marker.x = shortcut.find("x").text
marker.y = shortcut.find("y").text
shortcut_ref = shortcut.find("recordname").text.replace(
"encounter.", "").replace("@*", "")
page = None
if shortcut_ref in lookup["page"]:
page = lookup["page"][shortcut_ref]
# remove chapter numbers from page name
# maybe use a regex?
name = page.name
if " " in page.name:
first, second = page.name.split(' ', 1)
if "." in first:
name = second
marker.name = name
marker.contentRef = "/page/" + page.slug
markers.append(marker)
if markers:
# if markers not empty, its a map
map = Map()
map.parent = group
map.meta["tag"] = tag
map.name = image.name
map.slug = slugify(map.name)
map.image = image.bitmap
if node.find("./image/gridsize") != None:
map.gridSize = node.find("./image/gridsize").text
if node.find("./image/gridoffset") != None:
gridOffset = node.find("./image/gridoffset").text
map.gridOffsetX = gridOffset.split(",")[0]
map.gridOffsetY = gridOffset.split(",")[1]
map.markers = markers
maps.append(map)
lookup["map"][tag] = map
else:
# otherwise, its a image
page = Page()
page.parent = group
page.meta["tag"] = tag
page.name = image.name
page.slug = slugify(page.name)
page.content = '<p><img class="size-full" src="' + image.bitmap + '" /></p>'
pages_sorted.append(page)
# do not add to lookup tables
# sort
maps_sorted = humansorted(maps, key=lambda x: x.name)
# ENCOUNTERS
logger.info("parsing encounters")
parent = Group()
parent.name = "Encounters"
parent.slug = slugify(parent.name)
groups.append(parent)
for category in root.findall("./battle/category"):
group = Group()
group.name = category.get("name")
group.slug = slugify(group.name)
group.parent = parent
if group.name == None or group.name == "":
group = parent
else:
groups.append(group)
for node in category.findall("*"):
# tag
tag = node.tag
# create encounter
encounter = Encounter()
encounter.meta["tag"] = tag
encounter.parent = group
encounter.name = node.find("name").text
encounter.slug = slugify(encounter.name)
encounters.append(encounter)
lookup["encounter"][tag] = encounter
# get combatants
for npcnode in node.find("npclist").findall("*"):
# get positions
maplinks = npcnode.findall("./maplink/*")
# combatants count
count = int(npcnode.find("count").text)
# iterate
for x in range(count):
combatant = Combatant()
combatant.name = npcnode.find("name").text
encounter.combatants.append(combatant)
# if position on map
if len(maplinks) == count:
maplinknode = maplinks[x]
if maplinknode.find("./imagex") != None:
combatant.x = maplinknode.find("./imagex").text
if maplinknode.find("./imagey") != None:
combatant.y = maplinknode.find("./imagey").text
encounters_sorted = humansorted(encounters, key=lambda x: x.name)
# custom regex for processing links
def href_replace(match):
key = str(match.group(2)).split("@")[0]
type = match.group(1)
if type == "image" and key in lookup["map"]:
return 'href="/map/' + lookup["map"][key].slug
elif type == "image" and key in lookup["image"]:
return 'href="' + lookup["image"][key].bitmap
elif type == "encounter" and key in lookup["page"]:
return 'href="' + lookup["page"][key].slug
elif type == "battle" and key in lookup["encounter"]:
return 'href="/encounter/' + lookup["encounter"][key].slug
elif type == "quest" and key in lookup["quest"]:
return 'href="' + lookup["quest"][key].slug
else:
return key
# fix content tags in pages
for page in pages_sorted:
content = page.content
# maybe regex
content = content.replace('<text type="formattedtext">',
'').replace('<text>',
'').replace('</text>', '')
content = content.replace('<description type="formattedtext">',
'').replace('<description>', '').replace(
'</description>', '')
content = content.replace('<frame>',
'<blockquote class="read">').replace(
'</frame>', '</blockquote>')
content = content.replace('<frameid>DM</frameid>', '')
content = content.replace('\r', '<br />')
content = content.replace('<h>', '<h3>').replace('</h>', '</h3>')
content = content.replace('<list>',
'<ul>').replace('</list>', '</ul>')
# content = content.replace("<linklist>", "<ul>").replace("</linklist>", "</ul>")
content = content.replace('<linklist>',
'').replace('</linklist>', '')
content = content.replace('<link',
'<p><a').replace('</link>', '</a></p>')
content = content.replace(' recordname', ' href')
content = content.strip()
# fix links
content = re.sub(
r'href=[\'"]?(encounter|battle|image|quest)\.([^\'">]+)',
href_replace, content)
# add title
if content.startswith('<h3>'):
page.content = content.replace('<h3>', '<h2>',
1).replace('</h3>', '</h2>', 1)
else:
page.content = '<h2>' + page.name + '</h2>' + content
# assign data to module
module.groups = groups
module.pages = pages_sorted
module.maps = maps_sorted
module.encounters = encounters_sorted
return module
@app.route('/json/places/<int:place_id>/<int:marker_id>/', methods=['GET'])
def json_catalog_catagory_items(place_id, marker_id):
"""Handle JSON-requests for a marker"""
marker = Marker.query.filter_by(place_id=place_id,
id=marker_id,
archived=False).first()
return jsonify(marker.serialize)
# hook up extensions to app
db.init_app(app)
if __name__ == "__main__":
if "--setup" in sys.argv:
with app.app_context():
db.drop_all()
db.create_all()
import setup
for setup_place in setup.places:
db.session.add(Place(**setup_place))
for setup_marker in setup.markers:
db.session.add(Marker(**setup_marker))
db.session.commit()
print("Database tables created")
else:
if app.config['DEBUG'] is True:
# setup scss-folders
Scss(app, static_dir='static/css/', asset_dir='assets/scss/')
get_yelp_access_token()
app.run(host='0.0.0.0', port=app.config['PORT'])
