def import_to_datastore():
from models import User, Marker
from google.appengine.ext import db
my_user = User.all().filter("email", "*****@*****.**").get()
i = 0
markers = []
for data in import_data():
#old_marker = Marker.get_by_key_name(str(data["id"]))
#if old_marker:
# old_marker.delete()
marker = Marker(
key_name=str(data["id"]),
user = my_user,
title = "Accident",
description = data["description"].decode("utf8"),
address = data["address"].decode("utf8"),
location = db.GeoPt(data["lat"], data["lng"]),
type = Marker.MARKER_TYPE_ACCIDENT,
subtype = data["severity"],
created = data["date"],
modified = data["date"],
)
#marker.put()
#marker.update_location()
markers.append(marker)
print marker.key().name()
if len(markers) == 100:
print "Writing to datastore..."
db.put(markers)
markers = []
async def new_marker(sid: int, data):
pr: PlayerRoom = game_state.get(sid)
marker = Marker.get_or_none(shape=data, user=pr.player)
if marker is not None:
return
Marker.create(shape=data, user=pr.player, location=pr.active_location)
def add_marker(request):
if not request.user.is_authenticated():
return False, 0
longitude = request.POST['longitude']
latitude = request.POST['latitude']
title = request.POST['title']
description = request.POST['description']
marker = Marker(user=User.objects.get(username=request.user.username),
longitude=longitude, latitude=latitude, title=title, description=description)
marker.save()
return True, marker.id
def get(self):
timestamps = self.request.params.getall('timestamp')
row = self.request.params.get('row_key')
for timestamp in timestamps:
t = datetime.datetime.fromtimestamp(float(timestamp))
r = db.get(row)
m = Marker(time = t, row = r, value = 3)
m.put()
referer = self.request.environ.get('HTTP_REFERER')
if referer:
self.redirect(referer)
else:
show = '/show?key=%s' % row.key()
self.redirect(show)
def retrieve_clusters(ne_lat, ne_lng, sw_lat, sw_lng, start_date, end_date, fatal, severe, light, inaccurate, zoom):
marker_boxes = divide_to_boxes(ne_lat, ne_lng, sw_lat, sw_lng)
result_futures = []
logging.info("number of cores: " + str(multiprocessing.cpu_count()))
with concurrent.futures.ThreadPoolExecutor(max_workers=multiprocessing.cpu_count()) as executor:
for marker_box in marker_boxes:
markers_in_box = Marker.bounding_box_query(
marker_box[0],
marker_box[1],
marker_box[2],
marker_box[3],
start_date,
end_date,
fatal,
severe,
light,
inaccurate,
).all()
result_futures.append(executor.submit(calculate_clusters, markers_in_box, zoom))
completed_futures = concurrent.futures.wait(result_futures)
result = []
for future in completed_futures.done:
result.extend(future.result())
return result
def import_accidents(provider_code, accidents, streets, roads):
print("reading accidents from file %s" % (accidents.name(), ))
for accident in accidents:
if field_names.x_coordinate not in accident or field_names.y_coordinate not in accident:
raise ValueError(
"x and y coordinates are missing from the accidents file!")
if not accident[field_names.x_coordinate] or not accident[
field_names.y_coordinate]:
continue
lng, lat = coordinates_converter.convert(
accident[field_names.x_coordinate],
accident[field_names.y_coordinate])
marker = Marker(
user=None,
id=int("{0}{1}".format(provider_code, accident[field_names.id])),
title="Accident",
description=json.dumps(load_extra_data(accident, streets, roads),
encoding=models.db_encoding),
address=get_address(accident, streets),
latitude=lat,
longitude=lng,
type=Marker.MARKER_TYPE_ACCIDENT,
subtype=int(accident[field_names.accident_type]),
severity=int(accident[field_names.accident_severity]),
created=parse_date(accident),
locationAccuracy=int(accident[field_names.igun]),
)
yield marker
accidents.close()
def markers():
logging.debug('getting markers')
kwargs = get_kwargs()
logging.debug('querying markers in bounding box: %s' % kwargs)
is_thin = (kwargs['zoom'] < CONST.MINIMAL_ZOOM)
accidents = Marker.bounding_box_query(is_thin,
yield_per=50,
involved_and_vehicles=False,
**kwargs)
discussion_args = ('ne_lat', 'ne_lng', 'sw_lat', 'sw_lng',
'show_discussions')
discussions = DiscussionMarker.bounding_box_query(
**{arg: kwargs[arg]
for arg in discussion_args})
if request.values.get('format') == 'csv':
date_format = '%Y-%m-%d'
return Response(
generate_csv(accidents),
headers={
"Content-Type":
"text/csv",
"Content-Disposition":
'attachment; '
'filename="Anyway-accidents-from-{0}-to-{1}.csv"'.format(
kwargs["start_date"].strftime(date_format),
kwargs["end_date"].strftime(date_format))
})
else: # defaults to json
return generate_json(accidents, discussions, is_thin)
def charts_data():
logging.debug('getting charts data')
kwargs = get_kwargs()
accidents, vehicles, involved = Marker.bounding_box_query(is_thin=False, yield_per=50, involved_and_vehicles=True, **kwargs)
accidents_list = [acc.serialize() for acc in accidents]
vehicles_list = [vehicles_data_refinement(veh.serialize()) for veh in vehicles]
involved_list = [involved_data_refinement(inv.serialize()) for inv in involved]
return Response(json.dumps({'accidents': accidents_list, 'vehicles': vehicles_list, 'involved': involved_list}), mimetype="application/json")
async def delete_marker(sid: int, uuid: str):
pr: PlayerRoom = game_state.get(sid)
marker = Marker.get_or_none(shape_id=uuid, user=pr.player)
if not marker:
return
marker.delete_instance()
def setUp(self):
kwargs = {'approx': True, 'show_day': 7, 'show_discussions': True, 'accurate': True, 'surface': 0, 'weather': 0,
'district': 0, 'show_markers': True, 'show_fatal': True, 'show_time': 24, 'show_intersection': 3,
'show_light': True, 'sw_lat': 32.06711066128336, 'controlmeasure': 0, 'ne_lng': 34.799307929669226,
'show_severe': True, 'start_time': 25, 'acctype': 0, 'separation': 0, 'show_urban': 3, 'show_lane': 3,
'sw_lng': 34.78879367033085, 'zoom': 17, 'show_holiday': 0, 'end_time': 25, 'road': 0,
'ne_lat': 32.07254745790576, 'start_date': "01/01/2014", 'end_date': "01/01/2015"}
self.query = Marker.bounding_box_query(yield_per=50, **kwargs)
print self.query
def setUp(self):
kwargs = {'approx': True, 'show_day': 7, 'show_discussions': True, 'accurate': True, 'surface': 0,
'weather': 0, 'district': 0, 'show_markers': True, 'show_fatal': True, 'show_time': 24,
'show_intersection': 3, 'show_light': True, 'sw_lat': 32.067363446951944, 'controlmeasure': 0,
'start_date': datetime.date(2014, 1, 1), 'ne_lng': 34.79928962966915, 'show_severe': True,
'end_date': datetime.date(2016, 1, 1), 'start_time': 25, 'acctype': 0, 'separation': 0,
'show_urban': 3, 'show_lane': 3, 'sw_lng': 34.78877537033077, 'zoom': 17, 'show_holiday': 0,
'end_time': 25, 'road': 0, 'ne_lat': 32.072427482938345}
self.query_args = kwargs
self.query = Marker.bounding_box_query(yield_per=50, **kwargs)
def load_markers(self):
marker_data = self.__read_csv("marker")
#load marker
self.marker = []
for item in marker_data:
m = Marker()
self.__fill_members_with_csv_data(
m, ('spot_id', 'qr', 'nfc', 'eddystone', 'ibeacon_region_uid',
'ibeacon_major', 'ibeacon_minor'), item)
self.marker.append(m)
def get(self):
rows = db.Query(Row)
text = []
for row in rows:
text.append('%s %s' % (row.key(), row.owner))
for marker in Marker.all().filter("row =", row):
text.append(' - %s %s' % (marker.time, marker.value))
text.append(str(self.request.environ))
text = '\n'.join(text)
self.response.out.write(template.render('templates/pre.html', {
'text': text}))
async def load_location(sid: int, location: Location):
pr: PlayerRoom = game_state.get(sid)
if pr.active_location != location:
pr.active_location = location
pr.save()
data = {}
data["locations"] = [
{"id": l.id, "name": l.name} for l in pr.room.locations.order_by(Location.index)
]
data["floors"] = [
f.as_dict(pr.player, pr.player == pr.room.creator)
for f in location.floors.order_by(Floor.index)
]
client_options = pr.player.as_dict()
client_options.update(
**LocationUserOption.get(user=pr.player, location=location).as_dict()
)
await sio.emit("Board.Set", data, room=sid, namespace=GAME_NS)
await sio.emit("Location.Set", location.as_dict(), room=sid, namespace=GAME_NS)
await sio.emit("Client.Options.Set", client_options, room=sid, namespace=GAME_NS)
await sio.emit(
"Notes.Set",
[
note.as_dict()
for note in Note.select().where(
(Note.user == pr.player) & (Note.room == pr.room)
)
],
room=sid,
namespace=GAME_NS,
)
await sio.emit(
"Markers.Set",
[
marker.as_string()
for marker in Marker.select(Marker.shape_id).where(
(Marker.user == pr.player) & (Marker.location == location)
)
],
room=sid,
namespace=GAME_NS,
)
location_data = InitiativeLocationData.get_or_none(location=location)
if location_data:
await send_client_initiatives(pr, pr.player)
await sio.emit(
"Initiative.Round.Update", location_data.round, room=sid, namespace=GAME_NS,
)
await sio.emit(
"Initiative.Turn.Set", location_data.turn, room=sid, namespace=GAME_NS
)
def addmarker(request):
if request.method == 'POST':
form = MarkerForm(request.POST)
if form.is_valid():
marker = Marker()
marker.phone = form.cleaned_data.get("phone")
marker.latitude = form.cleaned_data.get("latitude", "32.0")
marker.longitude = form.cleaned_data.get("longitude", "118.0")
marker.message = form.cleaned_data.get("message", "")
marker.date = datetime.now()
marker.save()
return HttpResponseRedirect(reverse('location:index'))
else:
form = MarkerForm()
return render_to_response('location/addmarker.html', {'form': form},
context_instance=RequestContext(request))
def image_markers(image_id, stack_id, project_id):
with DATABASE.transaction():
project_stack = models.ProjectStack.get(models.ProjectStack.project == project_id,
models.ProjectStack.stack == stack_id)
stack_image = models.StackImage.get(models.StackImage.stack == stack_id,
models.StackImage.image == image_id)
user = models.AppUser.get(models.AppUser.id == g.user_id)
markers = (Marker.select().where(
Marker.app_user == g.user_id,
Marker.stack_image == stack_image.id,
Marker.project_stack == project_stack.id)
)
return jsonify({ 'markers': [marker.serialized for marker in markers] })
def setUp(self):
self.query = Marker.bounding_box_query(ne_lat=32.36,
ne_lng=35.088,
sw_lat=32.292,
sw_lng=34.884,
start_date=start_date,
end_date=end_date,
fatal=False,
severe=True,
light=True,
inaccurate=False,
is_thin=False,
yield_per=None)
def addmarker(request):
if request.method == 'POST':
form = MarkerForm(request.POST)
if form.is_valid():
marker = Marker()
marker.phone = form.cleaned_data.get("phone")
marker.latitude = form.cleaned_data.get("latitude", "32.0")
marker.longitude = form.cleaned_data.get("longitude", "118.0")
marker.message = form.cleaned_data.get("message", "")
marker.date = datetime.now()
marker.save()
return HttpResponseRedirect(reverse('location:index'))
else:
form = MarkerForm()
return render_to_response('location/addmarker.html', {'form':form}, context_instance=RequestContext(request))
def xml(map_id):
if re.search('^\d+$', map_id):
maps = Map.all().filter("id = ", int(map_id)).fetch(1)
else:
map_id_list = [int(x) for x in map_id.split(',')]
maps = Map.all().filter("id IN", map_id_list).fetch(1000)
for map in maps:
map.polylines = Polyline.all().filter("map =", map).fetch(1000)
map.markers = Marker.all().filter("map =", map).fetch(1000)
xml = render_template('map.xml', maps=maps)
resp = app.make_response(xml)
resp.headers['Content-Type'] = 'application/xml; charset=UTF-8'
return resp
def retrieve_clusters(**kwargs):
marker_boxes = divide_to_boxes(kwargs['ne_lat'], kwargs['ne_lng'], kwargs['sw_lat'], kwargs['sw_lng'])
result_futures = []
logging.info('number of cores: ' + str(multiprocessing.cpu_count()))
with concurrent.futures.ThreadPoolExecutor(max_workers=multiprocessing.cpu_count()) as executor:
for marker_box in marker_boxes:
kwargs.update(marker_box)
markers_in_box = Marker.bounding_box_query(**kwargs).all()
result_futures.append(executor.submit(calculate_clusters, markers_in_box, kwargs['zoom']))
completed_futures = concurrent.futures.wait(result_futures)
result = []
for future in completed_futures.done:
result.extend(future.result())
return result
def retrieve_clusters(**kwargs):
marker_boxes = divide_to_boxes(kwargs["ne_lat"], kwargs["ne_lng"], kwargs["sw_lat"], kwargs["sw_lng"])
result_futures = []
logging.info("number of cores: " + str(multiprocessing.cpu_count()))
with concurrent.futures.ThreadPoolExecutor(max_workers=multiprocessing.cpu_count()) as executor:
for marker_box in marker_boxes:
kwargs.update(marker_box)
markers_in_box = Marker.bounding_box_query(**kwargs).all()
result_futures.append(executor.submit(calculate_clusters, markers_in_box, kwargs["zoom"]))
completed_futures = concurrent.futures.wait(result_futures)
result = []
for future in completed_futures.done:
result.extend(future.result())
return result
def charts_data():
logging.debug('getting charts data')
kwargs = get_kwargs()
accidents, vehicles, involved = Marker.bounding_box_query(
is_thin=False, yield_per=50, involved_and_vehicles=True, **kwargs)
accidents_list = [acc.serialize() for acc in accidents]
vehicles_list = [
vehicles_data_refinement(veh.serialize()) for veh in vehicles
]
involved_list = [
involved_data_refinement(inv.serialize()) for inv in involved
]
return Response(json.dumps({
'accidents': accidents_list,
'vehicles': vehicles_list,
'involved': involved_list
}),
mimetype="application/json")
def generate_report_xlsx(self, project_id, report_type):
project = Project.get(Project.id == project_id)
total_probes = len([p for p in project.project_stacks])
project_users = AppUser.select().join(ProjectUser).where(ProjectUser.project == project)
project_info = project.title.split('_')
report = {}
total_iteration = 0
for iteration, project_stack in enumerate(project.project_stacks):
stack = {}
stack_images = StackImage.select().where(StackImage.stack == project_stack.stack.id)
for stack_image in stack_images:
users = {}
for user in project_users:
markers = Marker.select().where(
Marker.app_user == user.id,
Marker.stack_image == stack_image.id,
Marker.project_stack == project_stack.id
)
user_markers = [m.serialized for m in markers]
if user_markers:
users[user.email] = user_markers
stack[stack_image.image.name] = users
report[project_stack.stack.title] = stack
self.update_state(state='PROGRESS', meta={
'current': iteration,
'total': total_probes,
'projectId': project_id,
'reportType': report_type
})
total_iteration += 1
return {
'current': total_iteration,
'total': total_probes,
'status': 'Task complete',
'projectId': project_id,
'report': report,
'reportType': report_type
}
def retrieve_clusters(ne_lat, ne_lng, sw_lat, sw_lng, start_date, end_date,
fatal, severe, light, inaccurate, zoom):
marker_boxes = divide_to_boxes(ne_lat, ne_lng, sw_lat, sw_lng)
result_futures = []
logging.info('number of cores: ' + str(multiprocessing.cpu_count()))
with concurrent.futures.ThreadPoolExecutor(
max_workers=multiprocessing.cpu_count()) as executor:
for marker_box in marker_boxes:
markers_in_box = Marker.bounding_box_query(
marker_box[0], marker_box[1], marker_box[2], marker_box[3],
start_date, end_date, fatal, severe, light, inaccurate).all()
result_futures.append(
executor.submit(calculate_clusters, markers_in_box, zoom))
completed_futures = concurrent.futures.wait(result_futures)
result = []
for future in completed_futures.done:
result.extend(future.result())
return result
def markers():
logging.debug('getting markers')
kwargs = get_kwargs()
logging.debug('querying markers in bounding box: %s' % kwargs)
is_thin = (kwargs['zoom'] < CONST.MINIMAL_ZOOM)
accidents = Marker.bounding_box_query(is_thin, yield_per=50, involved_and_vehicles=False, **kwargs)
discussion_args = ('ne_lat', 'ne_lng', 'sw_lat', 'sw_lng', 'show_discussions')
discussions = DiscussionMarker.bounding_box_query(**{arg: kwargs[arg] for arg in discussion_args})
if request.values.get('format') == 'csv':
date_format = '%Y-%m-%d'
return Response(generate_csv(accidents), headers={
"Content-Type": "text/csv",
"Content-Disposition": 'attachment; '
'filename="Anyway-accidents-from-{0}-to-{1}.csv"'
.format(kwargs["start_date"].strftime(date_format), kwargs["end_date"].strftime(date_format))
})
else: # defaults to json
return generate_json(accidents, discussions, is_thin)
def import_to_datastore():
from models import User, Marker
i = 0
session = db_session()
commit_every = 500
for irow, data in enumerate(import_data()):
show_progress_spinner()
marker = Marker(
user=None,
title="Accident",
description=data["description"].decode("utf8"),
address=data["address"].decode("utf8"),
latitude=data["lat"],
longitude=data["lng"],
type=Marker.MARKER_TYPE_ACCIDENT,
subtype=data["severity"],
created=data["date"],
)
session.add(marker)
if irow > 0 and irow % commit_every == 0:
flush_and_commit(session)
@app.route('/json/places/<int:place_id>/<int:marker_id>/', methods=['GET'])
def json_catalog_catagory_items(place_id, marker_id):
"""Handle JSON-requests for a marker"""
marker = Marker.query.filter_by(place_id=place_id,
id=marker_id,
archived=False).first()
return jsonify(marker.serialize)
# hook up extensions to app
db.init_app(app)
if __name__ == "__main__":
if "--setup" in sys.argv:
with app.app_context():
db.drop_all()
db.create_all()
import setup
for setup_place in setup.places:
db.session.add(Place(**setup_place))
for setup_marker in setup.markers:
db.session.add(Marker(**setup_marker))
db.session.commit()
print("Database tables created")
else:
if app.config['DEBUG'] is True:
# setup scss-folders
Scss(app, static_dir='static/css/', asset_dir='assets/scss/')
get_yelp_access_token()
app.run(host='0.0.0.0', port=app.config['PORT'])
def test_approx_filter(self):
kwargs = self.query_args.copy()
kwargs['accurate'] = False
markers = Marker.bounding_box_query(yield_per=50, **kwargs)
for marker in markers:
self.assertTrue(marker.locationAccuracy != 1)
def test_light_severity_filter(self):
kwargs = self.query_args.copy()
kwargs['show_light'] = False
markers = Marker.bounding_box_query(yield_per=50, **kwargs)
for marker in markers:
self.assertTrue(marker.severity != 3)
async def load_location(sid: str, location: Location, *, complete=False):
pr: PlayerRoom = game_state.get(sid)
if pr.active_location != location:
pr.active_location = location
pr.save()
# 1. Load client options
client_options = pr.player.as_dict()
client_options["location_user_options"] = LocationUserOption.get(
user=pr.player, location=location).as_dict()
client_options["default_user_options"] = pr.player.default_options.as_dict(
)
if pr.user_options:
client_options["room_user_options"] = pr.user_options.as_dict()
await sio.emit("Client.Options.Set",
client_options,
room=sid,
namespace=GAME_NS)
# 2. Load room info
if complete:
await sio.emit(
"Room.Info.Set",
{
"name":
pr.room.name,
"creator":
pr.room.creator.name,
"invitationCode":
str(pr.room.invitation_code),
"isLocked":
pr.room.is_locked,
"default_options":
pr.room.default_options.as_dict(),
"players": [{
"id": rp.player.id,
"name": rp.player.name,
"location": rp.active_location.id,
"role": rp.role,
} for rp in pr.room.players],
"publicName":
config.get("General", "public_name", fallback=""),
},
room=sid,
namespace=GAME_NS,
)
# 3. Load location
await sio.emit("Location.Set",
location.as_dict(),
room=sid,
namespace=GAME_NS)
# 4. Load all location settings (DM)
if complete and pr.role == Role.DM:
await sio.emit(
"Locations.Settings.Set",
{
l.id: {} if l.options is None else l.options.as_dict()
for l in pr.room.locations
},
room=sid,
namespace=GAME_NS,
)
# 5. Load Board
locations = [{
"id": l.id,
"name": l.name,
"archived": l.archived
} for l in pr.room.locations.order_by(Location.index)]
await sio.emit("Board.Locations.Set",
locations,
room=sid,
namespace=GAME_NS)
floors = [floor for floor in location.floors.order_by(Floor.index)]
if "active_floor" in client_options["location_user_options"]:
index = next(i for i, f in enumerate(floors) if f.name ==
client_options["location_user_options"]["active_floor"])
lower_floors = floors[index - 1::-1] if index > 0 else []
higher_floors = floors[index + 1:] if index < len(floors) else []
floors = [floors[index], *lower_floors, *higher_floors]
for floor in floors:
await sio.emit(
"Board.Floor.Set",
floor.as_dict(pr.player, pr.role == Role.DM),
room=sid,
namespace=GAME_NS,
)
# 6. Load Initiative
location_data = Initiative.get_or_none(location=location)
if location_data:
await sio.emit("Initiative.Set",
location_data.as_dict(),
room=sid,
namespace=GAME_NS)
# 7. Load labels
if complete:
labels = Label.select().where((Label.user == pr.player)
| (Label.visible == True))
label_filters = LabelSelection.select().where(
(LabelSelection.user == pr.player)
& (LabelSelection.room == pr.room))
await sio.emit(
"Labels.Set",
[l.as_dict() for l in labels],
room=sid,
namespace=GAME_NS,
)
await sio.emit(
"Labels.Filters.Set",
[l.label.uuid for l in label_filters],
room=sid,
namespace=GAME_NS,
)
# 8. Load Notes
await sio.emit(
"Notes.Set",
[
note.as_dict()
for note in Note.select().where((Note.user == pr.player)
& (Note.room == pr.room))
],
room=sid,
namespace=GAME_NS,
)
# 9. Load Markers
await sio.emit(
"Markers.Set",
[
marker.as_string()
for marker in Marker.select(Marker.shape_id).where(
(Marker.user == pr.player) & (Marker.location == location))
],
room=sid,
namespace=GAME_NS,
)
# 10. Load Assets
if complete:
await sio.emit(
"Asset.List.Set",
Asset.get_user_structure(pr.player),
room=sid,
namespace=GAME_NS,
)
def index(marker=None, message=None):
context = {'url': request.base_url, 'index_url': request.url_root}
context['CONST'] = CONST.to_dict()
#logging.debug("Debug CONST:{0}",context['CONST'])
if 'marker' in request.values:
markers = Marker.get_marker(request.values['marker'])
if markers.count() == 1:
marker = markers[0]
context['coordinates'] = (marker.latitude, marker.longitude)
context['marker'] = marker.id
else:
message = u"תאונה לא נמצאה: " + request.values['marker']
elif 'discussion' in request.values:
discussions = DiscussionMarker.get_by_identifier(request.values['discussion'])
if discussions.count() == 1:
marker = discussions[0]
context['coordinates'] = (marker.latitude, marker.longitude)
context['discussion'] = marker.identifier
else:
message = gettext(u"Discussion not found:") + request.values['discussion']
if 'start_date' in request.values:
context['start_date'] = string2timestamp(request.values['start_date'])
elif marker:
context['start_date'] = year2timestamp(marker.created.year)
if 'end_date' in request.values:
context['end_date'] = string2timestamp(request.values['end_date'])
elif marker:
context['end_date'] = year2timestamp(marker.created.year + 1)
for attr in 'show_inaccurate', 'zoom':
if attr in request.values:
context[attr] = request.values[attr]
if 'map_only' in request.values:
if request.values['map_only'] in ('1', 'true'):
context['map_only'] = 1
if 'lat' in request.values and 'lon' in request.values:
context['coordinates'] = (request.values['lat'], request.values['lon'])
for attr in 'approx', 'accurate', 'show_markers', 'show_discussions', 'show_urban', 'show_intersection', 'show_lane',\
'show_day', 'show_holiday', 'show_time', 'start_time', 'end_time', 'weather', 'road', 'separation',\
'surface', 'acctype', 'controlmeasure', 'district', 'case_type', 'show_fatal', 'show_severe', 'show_light':
value = request.values.get(attr)
if value is not None:
context[attr] = value or '-1'
if message:
context['message'] = message
pref_accident_severity = []
pref_light = PreferenceObject('prefLight', '2', u"קלה")
pref_severe = PreferenceObject('prefSevere', '1', u"חמורה")
pref_fatal = PreferenceObject('prefFatal', '0', u"קטלנית")
pref_accident_severity.extend([pref_light,pref_severe,pref_fatal])
context['pref_accident_severity'] = pref_accident_severity
pref_accident_report_severity = []
pref_report_light = PreferenceObject('prefReportLight', '2', u"קלה")
pref_report_severe = PreferenceObject('prefReportSevere', '1', u"חמורה")
pref_report_fatal = PreferenceObject('prefReportFatal', '0', u"קטלנית")
pref_accident_report_severity.extend([pref_report_light,pref_report_severe,pref_report_fatal])
context['pref_accident_report_severity'] = pref_accident_report_severity
pref_historical_report_periods = []
month_strings = [u"אחד", u"שניים", u"שלושה", u"ארבעה", u"חמישה", u"שישה", u"שבעה", u"שמונה", u"תשעה", \
u"עשרה", u"אחד עשר", u"שניים עשר"]
for x in range(0, 12):
pref_historical_report_periods.append(PreferenceObject('prefHistoricalReport' + str(x+1) + 'Month', str(x+1), month_strings[x]))
context['pref_historical_report_periods'] = pref_historical_report_periods
pref_radius = []
for x in range(1,5):
pref_radius.append(PreferenceObject('prefRadius' + str(x * 500), x * 500, x * 500))
context['pref_radius'] = pref_radius
today = datetime.date.today()
context['default_end_date_format'] = request.values.get('end_date', today.strftime('%Y-%m-%d'))
context['default_start_date_format'] = request.values.get('start_date', (today - datetime.timedelta(days=1095)).strftime('%Y-%m-%d'))
return render_template('index.html', **context)
def parse_xml(self, path, module):
logger.debug("parsing xml: %s", path)
# lookup tables
lookup = {}
lookup["encounter"] = {}
lookup["page"] = {}
lookup["map"] = {}
lookup["image"] = {}
lookup["npc"] = {}
lookup["quest"] = {}
# arrays
pages = []
maps = []
groups = []
encounters = []
# xml tree
tree = ElementTree.parse(path)
root = tree.getroot()
# NPCS
logger.info("parsing npcs")
for category in root.findall("./npc/category"):
for node in category.findall("*"):
tag = node.tag
name = node.find("name").text
npc = NPC()
npc.name = name
lookup["npc"][tag] = npc
# PAGES
logger.info("parsing pages")
parent = Group()
parent.name = "Story"
parent.slug = slugify(parent.name)
groups.append(parent)
for category in root.findall("./encounter/category"):
group = Group()
group.name = category.get("name")
group.slug = slugify(group.name)
group.parent = parent
if group.name == None or group.name == "":
group = parent
else:
groups.append(group)
# get all pages
for node in category.findall("*"):
# tag
tag = node.tag
# create page
page = Page()
page.meta["tag"] = tag
page.name = node.find("name").text
page.slug = slugify(page.name)
page.content = ElementTree.tostring(
node.find("text"), encoding='utf-8',
method='xml').decode('utf-8')
page.parent = group
pages.append(page)
lookup["page"][tag] = page
# QUESTS
logger.info("parsing quests")
parent = Group()
parent.name = "Quests"
parent.slug = slugify(parent.name)
groups.append(parent)
# some modules got, so use this instead
for node in root.findall("./quest/*/*"):
# for node in root.findall("./quest/*"):
# tag
tag = node.tag
# create quest
page = Page()
page.meta["tag"] = id
page.name = node.find("name").text
page.slug = slugify(page.name)
page.content = ElementTree.tostring(node.find("description"),
encoding='utf-8',
method='xml').decode('utf-8')
cr = node.find("cr").text if node.find("cr") else ""
xp = node.find("xp").text if node.find("xp") else ""
page.content += '<p><strong>CR:</strong> ' + cr + ' <strong>XP:</strong> ' + xp + '</p>'
page.parent = parent
pages.append(page)
lookup["quest"][tag] = page
# sort
pages_sorted = humansorted(pages, key=lambda x: x.name)
# MAPS & IMAGES
logger.info("parsing images and maps")
parent = Group()
parent.name = "Maps & Images"
parent.slug = slugify(parent.name)
groups.append(parent)
for category in root.findall("./image/category"):
group = Group()
group.name = category.get("name")
group.slug = slugify(group.name)
group.parent = parent
if group.name == None or group.name == "":
group = parent
else:
groups.append(group)
for node in category.findall("*"):
# tag
tag = node.tag
# create image
image = Image()
image.tag = tag
image.bitmap = node.find("./image/bitmap").text.replace(
"\\", "/")
image.name = node.find("name").text
lookup["image"][tag] = image
markers = []
# get shortcouts (markers)
for shortcut in node.findall("./image/shortcuts/shortcut"):
# create marker
marker = Marker()
marker.x = shortcut.find("x").text
marker.y = shortcut.find("y").text
shortcut_ref = shortcut.find("recordname").text.replace(
"encounter.", "").replace("@*", "")
page = None
if shortcut_ref in lookup["page"]:
page = lookup["page"][shortcut_ref]
# remove chapter numbers from page name
# maybe use a regex?
name = page.name
if " " in page.name:
first, second = page.name.split(' ', 1)
if "." in first:
name = second
marker.name = name
marker.contentRef = "/page/" + page.slug
markers.append(marker)
if markers:
# if markers not empty, its a map
map = Map()
map.parent = group
map.meta["tag"] = tag
map.name = image.name
map.slug = slugify(map.name)
map.image = image.bitmap
if node.find("./image/gridsize") != None:
map.gridSize = node.find("./image/gridsize").text
if node.find("./image/gridoffset") != None:
gridOffset = node.find("./image/gridoffset").text
map.gridOffsetX = gridOffset.split(",")[0]
map.gridOffsetY = gridOffset.split(",")[1]
map.markers = markers
maps.append(map)
lookup["map"][tag] = map
else:
# otherwise, its a image
page = Page()
page.parent = group
page.meta["tag"] = tag
page.name = image.name
page.slug = slugify(page.name)
page.content = '<p><img class="size-full" src="' + image.bitmap + '" /></p>'
pages_sorted.append(page)
# do not add to lookup tables
# sort
maps_sorted = humansorted(maps, key=lambda x: x.name)
# ENCOUNTERS
logger.info("parsing encounters")
parent = Group()
parent.name = "Encounters"
parent.slug = slugify(parent.name)
groups.append(parent)
for category in root.findall("./battle/category"):
group = Group()
group.name = category.get("name")
group.slug = slugify(group.name)
group.parent = parent
if group.name == None or group.name == "":
group = parent
else:
groups.append(group)
for node in category.findall("*"):
# tag
tag = node.tag
# create encounter
encounter = Encounter()
encounter.meta["tag"] = tag
encounter.parent = group
encounter.name = node.find("name").text
encounter.slug = slugify(encounter.name)
encounters.append(encounter)
lookup["encounter"][tag] = encounter
# get combatants
for npcnode in node.find("npclist").findall("*"):
# get positions
maplinks = npcnode.findall("./maplink/*")
# combatants count
count = int(npcnode.find("count").text)
# iterate
for x in range(count):
combatant = Combatant()
combatant.name = npcnode.find("name").text
encounter.combatants.append(combatant)
# if position on map
if len(maplinks) == count:
maplinknode = maplinks[x]
if maplinknode.find("./imagex") != None:
combatant.x = maplinknode.find("./imagex").text
if maplinknode.find("./imagey") != None:
combatant.y = maplinknode.find("./imagey").text
encounters_sorted = humansorted(encounters, key=lambda x: x.name)
# custom regex for processing links
def href_replace(match):
key = str(match.group(2)).split("@")[0]
type = match.group(1)
if type == "image" and key in lookup["map"]:
return 'href="/map/' + lookup["map"][key].slug
elif type == "image" and key in lookup["image"]:
return 'href="' + lookup["image"][key].bitmap
elif type == "encounter" and key in lookup["page"]:
return 'href="' + lookup["page"][key].slug
elif type == "battle" and key in lookup["encounter"]:
return 'href="/encounter/' + lookup["encounter"][key].slug
elif type == "quest" and key in lookup["quest"]:
return 'href="' + lookup["quest"][key].slug
else:
return key
# fix content tags in pages
for page in pages_sorted:
content = page.content
# maybe regex
content = content.replace('<text type="formattedtext">',
'').replace('<text>',
'').replace('</text>', '')
content = content.replace('<description type="formattedtext">',
'').replace('<description>', '').replace(
'</description>', '')
content = content.replace('<frame>',
'<blockquote class="read">').replace(
'</frame>', '</blockquote>')
content = content.replace('<frameid>DM</frameid>', '')
content = content.replace('\r', '<br />')
content = content.replace('<h>', '<h3>').replace('</h>', '</h3>')
content = content.replace('<list>',
'<ul>').replace('</list>', '</ul>')
# content = content.replace("<linklist>", "<ul>").replace("</linklist>", "</ul>")
content = content.replace('<linklist>',
'').replace('</linklist>', '')
content = content.replace('<link',
'<p><a').replace('</link>', '</a></p>')
content = content.replace(' recordname', ' href')
content = content.strip()
# fix links
content = re.sub(
r'href=[\'"]?(encounter|battle|image|quest)\.([^\'">]+)',
href_replace, content)
# add title
if content.startswith('<h3>'):
page.content = content.replace('<h3>', '<h2>',
1).replace('</h3>', '</h2>', 1)
else:
page.content = '<h2>' + page.name + '</h2>' + content
# assign data to module
module.groups = groups
module.pages = pages_sorted
module.maps = maps_sorted
module.encounters = encounters_sorted
return module
def retrieve_clusters(ne_lat, ne_lng, sw_lat, sw_lng, start_date, end_date, fatal, severe, light, inaccurate, zoom):
start_time = time.time()
filtered_markers = Marker.bounding_box_query(ne_lat, ne_lng, sw_lat, sw_lng, start_date, end_date, fatal,
severe, light, inaccurate).all()
print('bounding_box_query took ' + str(time.time() - start_time))
return generate_clusters_json(filtered_markers, zoom)
def index(marker=None, message=None):
context = {'url': request.base_url, 'index_url': request.url_root}
context['CONST'] = CONST.to_dict()
#logging.debug("Debug CONST:{0}",context['CONST'])
if 'marker' in request.values:
markers = Marker.get_marker(request.values['marker'])
if markers.count() == 1:
marker = markers[0]
context['coordinates'] = (marker.latitude, marker.longitude)
context['marker'] = marker.id
else:
message = u"תאונה לא נמצאה: " + request.values['marker']
elif 'discussion' in request.values:
discussions = DiscussionMarker.get_by_identifier(
request.values['discussion'])
if discussions.count() == 1:
marker = discussions[0]
context['coordinates'] = (marker.latitude, marker.longitude)
context['discussion'] = marker.identifier
else:
message = gettext(
u"Discussion not found:") + request.values['discussion']
if 'start_date' in request.values:
context['start_date'] = string2timestamp(request.values['start_date'])
elif marker:
context['start_date'] = year2timestamp(marker.created.year)
if 'end_date' in request.values:
context['end_date'] = string2timestamp(request.values['end_date'])
elif marker:
context['end_date'] = year2timestamp(marker.created.year + 1)
for attr in 'show_inaccurate', 'zoom':
if attr in request.values:
context[attr] = request.values[attr]
if 'map_only' in request.values:
if request.values['map_only'] in ('1', 'true'):
context['map_only'] = 1
if 'lat' in request.values and 'lon' in request.values:
context['coordinates'] = (request.values['lat'], request.values['lon'])
for attr in 'approx', 'accurate', 'show_markers', 'show_discussions', 'show_urban', 'show_intersection', 'show_lane',\
'show_day', 'show_holiday', 'show_time', 'start_time', 'end_time', 'weather', 'road', 'separation',\
'surface', 'acctype', 'controlmeasure', 'district', 'case_type', 'show_fatal', 'show_severe', 'show_light':
value = request.values.get(attr)
if value is not None:
context[attr] = value or '-1'
if message:
context['message'] = message
pref_accident_severity = []
pref_light = PreferenceObject('prefLight', '2', u"קלה")
pref_severe = PreferenceObject('prefSevere', '1', u"חמורה")
pref_fatal = PreferenceObject('prefFatal', '0', u"קטלנית")
pref_accident_severity.extend([pref_light, pref_severe, pref_fatal])
context['pref_accident_severity'] = pref_accident_severity
pref_accident_report_severity = []
pref_report_light = PreferenceObject('prefReportLight', '2', u"קלה")
pref_report_severe = PreferenceObject('prefReportSevere', '1', u"חמורה")
pref_report_fatal = PreferenceObject('prefReportFatal', '0', u"קטלנית")
pref_accident_report_severity.extend(
[pref_report_light, pref_report_severe, pref_report_fatal])
context['pref_accident_report_severity'] = pref_accident_report_severity
pref_historical_report_periods = []
month_strings = [u"אחד", u"שניים", u"שלושה", u"ארבעה", u"חמישה", u"שישה", u"שבעה", u"שמונה", u"תשעה", \
u"עשרה", u"אחד עשר", u"שניים עשר"]
for x in range(0, 12):
pref_historical_report_periods.append(
PreferenceObject('prefHistoricalReport' + str(x + 1) + 'Month',
str(x + 1), month_strings[x]))
context['pref_historical_report_periods'] = pref_historical_report_periods
pref_radius = []
for x in range(1, 5):
pref_radius.append(
PreferenceObject('prefRadius' + str(x * 500), x * 500, x * 500))
context['pref_radius'] = pref_radius
today = datetime.date.today()
context['default_end_date_format'] = request.values.get(
'end_date', today.strftime('%Y-%m-%d'))
context['default_start_date_format'] = request.values.get(
'start_date',
(today - datetime.timedelta(days=1095)).strftime('%Y-%m-%d'))
return render_template('index.html', **context)
def get_project_report_user(project_id, user_id):
markers = Marker.select().join(ProjectStack).join(Project).where(
ProjectStack.project == project_id,
Marker.app_user == user_id)
markers = [m.marker_type for m in markers]
return jsonify({ 'markers': [markers] })
def setUp(self):
self.query = Marker.bounding_box_query(ne_lat=32.36, ne_lng=35.088, sw_lat=32.292, sw_lng=34.884,
start_date=start_date, end_date=end_date,
fatal=False, severe=True, light=True, inaccurate=False,
is_thin=False, yield_per=None)
import os
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base
import datetime
from constants import CONST
db_connection_string = os.environ.get('CLEARDB_DATABASE_URL')
print "using connection string: %s" % db_connection_string
engine = create_engine(db_connection_string, convert_unicode=True, echo=True)
autocommit = False # Autocommit does not seem to work
db_session = sessionmaker(autocommit=autocommit, autoflush=True, bind=engine)
Base = declarative_base()
session = db_session()
from models import Marker
marker = Marker(
user=None,
title="Accident",
description="sample accident",
address="sample address",
latitude="00.33",
longitude="00.22",
type=CONST.MARKER_TYPE_ACCIDENT,
subtype="",
created=datetime.datetime.now(),
)
session.add(marker)
if not autocommit:
session.commit()