def import_accidents(provider_code, accidents, streets, roads): print("reading accidents from file %s" % (accidents.name(), )) for accident in accidents: if field_names.x_coordinate not in accident or field_names.y_coordinate not in accident: raise ValueError( "x and y coordinates are missing from the accidents file!") if not accident[field_names.x_coordinate] or not accident[ field_names.y_coordinate]: continue lng, lat = coordinates_converter.convert( accident[field_names.x_coordinate], accident[field_names.y_coordinate]) marker = Marker( user=None, id=int("{0}{1}".format(provider_code, accident[field_names.id])), title="Accident", description=json.dumps(load_extra_data(accident, streets, roads), encoding=models.db_encoding), address=get_address(accident, streets), latitude=lat, longitude=lng, type=Marker.MARKER_TYPE_ACCIDENT, subtype=int(accident[field_names.accident_type]), severity=int(accident[field_names.accident_severity]), created=parse_date(accident), locationAccuracy=int(accident[field_names.igun]), ) yield marker accidents.close()
def load_markers(self): marker_data = self.__read_csv("marker") #load marker self.marker = [] for item in marker_data: m = Marker() self.__fill_members_with_csv_data( m, ('spot_id', 'qr', 'nfc', 'eddystone', 'ibeacon_region_uid', 'ibeacon_major', 'ibeacon_minor'), item) self.marker.append(m)
def addmarker(request): if request.method == 'POST': form = MarkerForm(request.POST) if form.is_valid(): marker = Marker() marker.phone = form.cleaned_data.get("phone") marker.latitude = form.cleaned_data.get("latitude", "32.0") marker.longitude = form.cleaned_data.get("longitude", "118.0") marker.message = form.cleaned_data.get("message", "") marker.date = datetime.now() marker.save() return HttpResponseRedirect(reverse('location:index')) else: form = MarkerForm() return render_to_response('location/addmarker.html', {'form': form}, context_instance=RequestContext(request))
def import_to_datastore(): from models import User, Marker i = 0 session = db_session() commit_every = 500 for irow, data in enumerate(import_data()): show_progress_spinner() marker = Marker( user=None, title="Accident", description=data["description"].decode("utf8"), address=data["address"].decode("utf8"), latitude=data["lat"], longitude=data["lng"], type=Marker.MARKER_TYPE_ACCIDENT, subtype=data["severity"], created=data["date"], ) session.add(marker) if irow > 0 and irow % commit_every == 0: flush_and_commit(session)
import os from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from sqlalchemy.ext.declarative import declarative_base import datetime from constants import CONST db_connection_string = os.environ.get('CLEARDB_DATABASE_URL') print "using connection string: %s" % db_connection_string engine = create_engine(db_connection_string, convert_unicode=True, echo=True) autocommit = False # Autocommit does not seem to work db_session = sessionmaker(autocommit=autocommit, autoflush=True, bind=engine) Base = declarative_base() session = db_session() from models import Marker marker = Marker( user=None, title="Accident", description="sample accident", address="sample address", latitude="00.33", longitude="00.22", type=CONST.MARKER_TYPE_ACCIDENT, subtype="", created=datetime.datetime.now(), ) session.add(marker) if not autocommit: session.commit()
def parse_xml(self, path, module): logger.debug("parsing xml: %s", path) # lookup tables lookup = {} lookup["encounter"] = {} lookup["page"] = {} lookup["map"] = {} lookup["image"] = {} lookup["npc"] = {} lookup["quest"] = {} # arrays pages = [] maps = [] groups = [] encounters = [] # xml tree tree = ElementTree.parse(path) root = tree.getroot() # NPCS logger.info("parsing npcs") for category in root.findall("./npc/category"): for node in category.findall("*"): tag = node.tag name = node.find("name").text npc = NPC() npc.name = name lookup["npc"][tag] = npc # PAGES logger.info("parsing pages") parent = Group() parent.name = "Story" parent.slug = slugify(parent.name) groups.append(parent) for category in root.findall("./encounter/category"): group = Group() group.name = category.get("name") group.slug = slugify(group.name) group.parent = parent if group.name == None or group.name == "": group = parent else: groups.append(group) # get all pages for node in category.findall("*"): # tag tag = node.tag # create page page = Page() page.meta["tag"] = tag page.name = node.find("name").text page.slug = slugify(page.name) page.content = ElementTree.tostring( node.find("text"), encoding='utf-8', method='xml').decode('utf-8') page.parent = group pages.append(page) lookup["page"][tag] = page # QUESTS logger.info("parsing quests") parent = Group() parent.name = "Quests" parent.slug = slugify(parent.name) groups.append(parent) # some modules got, so use this instead for node in root.findall("./quest/*/*"): # for node in root.findall("./quest/*"): # tag tag = node.tag # create quest page = Page() page.meta["tag"] = id page.name = node.find("name").text page.slug = slugify(page.name) page.content = ElementTree.tostring(node.find("description"), encoding='utf-8', method='xml').decode('utf-8') cr = node.find("cr").text if node.find("cr") else "" xp = node.find("xp").text if node.find("xp") else "" page.content += '<p><strong>CR:</strong> ' + cr + ' <strong>XP:</strong> ' + xp + '</p>' page.parent = parent pages.append(page) lookup["quest"][tag] = page # sort pages_sorted = humansorted(pages, key=lambda x: x.name) # MAPS & IMAGES logger.info("parsing images and maps") parent = Group() parent.name = "Maps & Images" parent.slug = slugify(parent.name) groups.append(parent) for category in root.findall("./image/category"): group = Group() group.name = category.get("name") group.slug = slugify(group.name) group.parent = parent if group.name == None or group.name == "": group = parent else: groups.append(group) for node in category.findall("*"): # tag tag = node.tag # create image image = Image() image.tag = tag image.bitmap = node.find("./image/bitmap").text.replace( "\\", "/") image.name = node.find("name").text lookup["image"][tag] = image markers = [] # get shortcouts (markers) for shortcut in node.findall("./image/shortcuts/shortcut"): # create marker marker = Marker() marker.x = shortcut.find("x").text marker.y = shortcut.find("y").text shortcut_ref = shortcut.find("recordname").text.replace( "encounter.", "").replace("@*", "") page = None if shortcut_ref in lookup["page"]: page = lookup["page"][shortcut_ref] # remove chapter numbers from page name # maybe use a regex? name = page.name if " " in page.name: first, second = page.name.split(' ', 1) if "." in first: name = second marker.name = name marker.contentRef = "/page/" + page.slug markers.append(marker) if markers: # if markers not empty, its a map map = Map() map.parent = group map.meta["tag"] = tag map.name = image.name map.slug = slugify(map.name) map.image = image.bitmap if node.find("./image/gridsize") != None: map.gridSize = node.find("./image/gridsize").text if node.find("./image/gridoffset") != None: gridOffset = node.find("./image/gridoffset").text map.gridOffsetX = gridOffset.split(",")[0] map.gridOffsetY = gridOffset.split(",")[1] map.markers = markers maps.append(map) lookup["map"][tag] = map else: # otherwise, its a image page = Page() page.parent = group page.meta["tag"] = tag page.name = image.name page.slug = slugify(page.name) page.content = '<p><img class="size-full" src="' + image.bitmap + '" /></p>' pages_sorted.append(page) # do not add to lookup tables # sort maps_sorted = humansorted(maps, key=lambda x: x.name) # ENCOUNTERS logger.info("parsing encounters") parent = Group() parent.name = "Encounters" parent.slug = slugify(parent.name) groups.append(parent) for category in root.findall("./battle/category"): group = Group() group.name = category.get("name") group.slug = slugify(group.name) group.parent = parent if group.name == None or group.name == "": group = parent else: groups.append(group) for node in category.findall("*"): # tag tag = node.tag # create encounter encounter = Encounter() encounter.meta["tag"] = tag encounter.parent = group encounter.name = node.find("name").text encounter.slug = slugify(encounter.name) encounters.append(encounter) lookup["encounter"][tag] = encounter # get combatants for npcnode in node.find("npclist").findall("*"): # get positions maplinks = npcnode.findall("./maplink/*") # combatants count count = int(npcnode.find("count").text) # iterate for x in range(count): combatant = Combatant() combatant.name = npcnode.find("name").text encounter.combatants.append(combatant) # if position on map if len(maplinks) == count: maplinknode = maplinks[x] if maplinknode.find("./imagex") != None: combatant.x = maplinknode.find("./imagex").text if maplinknode.find("./imagey") != None: combatant.y = maplinknode.find("./imagey").text encounters_sorted = humansorted(encounters, key=lambda x: x.name) # custom regex for processing links def href_replace(match): key = str(match.group(2)).split("@")[0] type = match.group(1) if type == "image" and key in lookup["map"]: return 'href="/map/' + lookup["map"][key].slug elif type == "image" and key in lookup["image"]: return 'href="' + lookup["image"][key].bitmap elif type == "encounter" and key in lookup["page"]: return 'href="' + lookup["page"][key].slug elif type == "battle" and key in lookup["encounter"]: return 'href="/encounter/' + lookup["encounter"][key].slug elif type == "quest" and key in lookup["quest"]: return 'href="' + lookup["quest"][key].slug else: return key # fix content tags in pages for page in pages_sorted: content = page.content # maybe regex content = content.replace('<text type="formattedtext">', '').replace('<text>', '').replace('</text>', '') content = content.replace('<description type="formattedtext">', '').replace('<description>', '').replace( '</description>', '') content = content.replace('<frame>', '<blockquote class="read">').replace( '</frame>', '</blockquote>') content = content.replace('<frameid>DM</frameid>', '') content = content.replace('\r', '<br />') content = content.replace('<h>', '<h3>').replace('</h>', '</h3>') content = content.replace('<list>', '<ul>').replace('</list>', '</ul>') # content = content.replace("<linklist>", "<ul>").replace("</linklist>", "</ul>") content = content.replace('<linklist>', '').replace('</linklist>', '') content = content.replace('<link', '<p><a').replace('</link>', '</a></p>') content = content.replace(' recordname', ' href') content = content.strip() # fix links content = re.sub( r'href=[\'"]?(encounter|battle|image|quest)\.([^\'">]+)', href_replace, content) # add title if content.startswith('<h3>'): page.content = content.replace('<h3>', '<h2>', 1).replace('</h3>', '</h2>', 1) else: page.content = '<h2>' + page.name + '</h2>' + content # assign data to module module.groups = groups module.pages = pages_sorted module.maps = maps_sorted module.encounters = encounters_sorted return module
@app.route('/json/places/<int:place_id>/<int:marker_id>/', methods=['GET']) def json_catalog_catagory_items(place_id, marker_id): """Handle JSON-requests for a marker""" marker = Marker.query.filter_by(place_id=place_id, id=marker_id, archived=False).first() return jsonify(marker.serialize) # hook up extensions to app db.init_app(app) if __name__ == "__main__": if "--setup" in sys.argv: with app.app_context(): db.drop_all() db.create_all() import setup for setup_place in setup.places: db.session.add(Place(**setup_place)) for setup_marker in setup.markers: db.session.add(Marker(**setup_marker)) db.session.commit() print("Database tables created") else: if app.config['DEBUG'] is True: # setup scss-folders Scss(app, static_dir='static/css/', asset_dir='assets/scss/') get_yelp_access_token() app.run(host='0.0.0.0', port=app.config['PORT'])