def post(self, event_key_id):
self._require_admin()
event = Event.get_by_id(event_key_id)
if not event:
self.abort(404)
place_id = self.request.get('place_id')
if not place_id:
self.abort(400)
# Construct a mostly empty input struct that'll get filled in
location_input = {
'place_id': place_id,
'geometry': {
'location': {
'lat': '',
'lng': '',
},
},
'name': '',
'types': [],
}
location_info = LocationHelper.construct_location_info_async(location_input).get_result()
event.normalized_location = LocationHelper.build_normalized_location(location_info)
EventManipulator.createOrUpdate(event)
self.redirect('/admin/event/{}'.format(event_key_id))
def postUpdateHook(cls, events, updated_attr_list, is_new_list):
"""
To run after models have been updated
"""
for (event, updated_attrs) in zip(events, updated_attr_list):
try:
LocationHelper.update_event_location(event)
except Exception, e:
logging.error("update_event_location for {} errored!".format(
event.key.id()))
logging.exception(e)
try:
if event.normalized_location and event.normalized_location.lat_lng:
timezone_id = LocationHelper.get_timezone_id(
None, lat_lng=event.normalized_location.lat_lng)
if not timezone_id:
logging.warning(
"Timezone update for event {} failed!".format(
event.key_name))
else:
event.timezone_id = timezone_id
else:
logging.warning(
"No Lat/Lng to update timezone_id for event {}!".
format(event.key_name))
except Exception, e:
logging.error("Timezone update for {} errored!".format(
event.key.id()))
logging.exception(e)
def post(self, event_key_id):
self._require_admin()
event = Event.get_by_id(event_key_id)
if not event:
self.abort(404)
place_id = self.request.get('place_id')
if not place_id:
self.abort(400)
# Construct a mostly empty input struct that'll get filled in
location_input = {
'place_id': place_id,
'geometry': {
'location': {
'lat': '',
'lng': '',
},
},
'name': '',
'types': [],
}
location_info = LocationHelper.construct_location_info_async(
location_input).get_result()
event.normalized_location = LocationHelper.build_normalized_location(
location_info)
EventManipulator.createOrUpdate(event)
self.redirect('/admin/event/{}'.format(event_key_id))
def test_2012ct_bad_location(self):
with open('test_data/usfirst_html/usfirst_event_details_2012ct.html', 'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
event['location'] = "somewhere on mars"
self.assertEqual(LocationHelper.get_lat_lng(event['location']), None)
self.assertEqual(LocationHelper.get_timezone_id(event['location']), None)
def test_2013flbr(self):
with open('test_data/usfirst_html/usfirst_event_details_2013flbr.html',
'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
self.assertEqual(LocationHelper.get_lat_lng(event['location']),
(26.1224386, -80.13731740000001))
self.assertEqual(LocationHelper.get_timezone_id(event['location']),
'America/New_York')
def test_2001sj(self):
with open('test_data/usfirst_html/usfirst_event_details_2001ca2.html',
'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
self.assertEqual(LocationHelper.get_lat_lng(event['location']),
(37.3382082, -121.8863286))
self.assertEqual(LocationHelper.get_timezone_id(event['location']),
'America/Los_Angeles')
def test_2005is(self):
with open('test_data/usfirst_html/usfirst_event_details_2005is.html',
'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
self.assertEqual(LocationHelper.get_lat_lng(event['location']),
(32.7940463, 34.989571))
self.assertEqual(LocationHelper.get_timezone_id(event['location']),
'Asia/Jerusalem')
def test_1997il(self):
with open('test_data/usfirst_html/usfirst_event_details_1997il.html',
'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
self.assertEqual(LocationHelper.get_lat_lng(event['location']),
(41.8781136, -87.6297982))
self.assertEqual(LocationHelper.get_timezone_id(event['location']),
'America/Chicago')
def test_2012ct(self):
with open('test_data/usfirst_html/usfirst_event_details_2012ct.html',
'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
self.assertEqual(LocationHelper.get_lat_lng(event['location']),
(41.76371109999999, -72.6850932))
self.assertEqual(LocationHelper.get_timezone_id(event['location']),
'America/New_York')
def test_2005or(self):
with open('test_data/usfirst_html/usfirst_event_details_2005or.html',
'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
self.assertEqual(LocationHelper.get_lat_lng(event['location']),
(45.5230622, -122.6764816))
self.assertEqual(LocationHelper.get_timezone_id(event['location']),
'America/Los_Angeles')
def test_2013casj(self):
with open('test_data/usfirst_html/usfirst_event_details_2013casj.html',
'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
location = LocationHelper.get_lat_lng(event['location'])
self.assertAlmostEqual(location[0], 37.3382082, places=6)
self.assertAlmostEqual(location[1], -121.8863286, places=6)
self.assertEqual(LocationHelper.get_timezone_id(event['location']),
'America/Los_Angeles')
def test_2012ct(self):
with open('test_data/usfirst_html/usfirst_event_details_2012ct.html', 'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
location = None
for _ in xrange(5):
location = LocationHelper.get_lat_lng(event['location'])
if location:
break
self.assertAlmostEqual(location[0], 41.76371109999999, places=6)
self.assertAlmostEqual(location[1], -72.6850932, places=6)
self.assertEqual(LocationHelper.get_timezone_id(event['location']), 'America/New_York')
def test_1997il(self):
with open('test_data/usfirst_html/usfirst_event_details_1997il.html', 'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
location = None
for _ in xrange(5):
location = LocationHelper.get_lat_lng(event['location'])
if location:
break
self.assertAlmostEqual(location[0], 41.8781136, places=6)
self.assertAlmostEqual(location[1], -87.6297982, places=6)
self.assertEqual(LocationHelper.get_timezone_id(event['location']), 'America/Chicago')
def test_2013flbr(self):
with open('test_data/usfirst_html/usfirst_event_details_2013flbr.html', 'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
location = None
for _ in xrange(5):
location = LocationHelper.get_lat_lng(event['location'])
if location:
break
self.assertAlmostEqual(location[0], 26.1224386, places=6)
self.assertAlmostEqual(location[1], -80.13731740000001, places=6)
self.assertEqual(LocationHelper.get_timezone_id(event['location']), 'America/New_York')
def test_2005is(self):
with open('test_data/usfirst_html/usfirst_event_details_2005is.html', 'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
location = None
for _ in xrange(5):
location = LocationHelper.get_lat_lng(event['location'])
if location:
break
self.assertAlmostEqual(location[0], 32.7940463, places=6)
self.assertAlmostEqual(location[1], 34.989571, places=6)
self.assertEqual(LocationHelper.get_timezone_id(event['location']), 'Asia/Jerusalem')
def test_2005or(self):
with open('test_data/usfirst_html/usfirst_event_details_2005or.html', 'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
location = None
for _ in xrange(5):
location = LocationHelper.get_lat_lng(event['location'])
if location:
break
self.assertAlmostEqual(location[0], 45.5230622, places=6)
self.assertAlmostEqual(location[1], -122.6764816, places=6)
self.assertEqual(LocationHelper.get_timezone_id(event['location']), 'America/Los_Angeles')
def get(self, event_key_id):
self._require_admin()
event = Event.get_by_id(event_key_id)
if not event:
self.abort(404)
event.normalized_location = None
LocationHelper.update_event_location(event)
event = EventManipulator.createOrUpdate(event)
self.response.out.write("New location: {}".format(event.normalized_location))
def postUpdateHook(cls, teams, updated_attr_list, is_new_list):
"""
To run after models have been updated
"""
for (team, updated_attrs) in zip(teams, updated_attr_list):
try:
LocationHelper.update_team_location(team)
except Exception, e:
logging.error("update_team_location for {} errored!".format(team.key.id()))
logging.exception(e)
try:
SearchHelper.update_team_location_index(team)
except Exception, e:
logging.error("update_team_location_index for {} errored!".format(team.key.id()))
logging.exception(e)
def postUpdateHook(cls, events, updated_attr_list, is_new_list):
"""
To run after models have been updated
"""
for (event, updated_attrs) in zip(events, updated_attr_list):
# Disabled due to unreliability. 2017-01-24 -fangeugene
# try:
# LocationHelper.update_event_location(event)
# except Exception, e:
# logging.error("update_event_location for {} errored!".format(event.key.id()))
# logging.exception(e)
try:
if event.normalized_location and event.normalized_location.lat_lng:
timezone_id = LocationHelper.get_timezone_id(
None, lat_lng=event.normalized_location.lat_lng)
if not timezone_id:
logging.warning("Timezone update for event {} failed!".format(event.key_name))
else:
event.timezone_id = timezone_id
else:
logging.warning("No Lat/Lng to update timezone_id for event {}!".format(event.key_name))
except Exception, e:
logging.error("Timezone update for {} errored!".format(event.key.id()))
logging.exception(e)
try:
SearchHelper.update_event_location_index(event)
except Exception, e:
logging.error("update_event_location_index for {} errored!".format(event.key.id()))
logging.exception(e)
def postUpdateHook(cls, events, updated_attr_list, is_new_list):
"""
To run after models have been updated
"""
for (event, updated_attrs) in zip(events, updated_attr_list):
# Disabled due to unreliability. 2017-01-24 -fangeugene
# try:
# LocationHelper.update_event_location(event)
# except Exception, e:
# logging.error("update_event_location for {} errored!".format(event.key.id()))
# logging.exception(e)
try:
if event.normalized_location and event.normalized_location.lat_lng:
timezone_id = LocationHelper.get_timezone_id(
None, lat_lng=event.normalized_location.lat_lng)
if not timezone_id:
logging.warning("Timezone update for event {} failed!".format(event.key_name))
else:
event.timezone_id = timezone_id
else:
logging.warning("No Lat/Lng to update timezone_id for event {}!".format(event.key_name))
except Exception, e:
logging.error("Timezone update for {} errored!".format(event.key.id()))
logging.exception(e)
try:
SearchHelper.update_event_location_index(event)
except Exception, e:
logging.error("update_event_location_index for {} errored!".format(event.key.id()))
logging.exception(e)
def postUpdateHook(cls, events, updated_attr_list, is_new_list):
"""
To run after models have been updated
"""
for (event, updated_attrs) in zip(events, updated_attr_list):
lat_lon = event.get_lat_lon()
if not lat_lon:
logging.warning("Lat/Lon update for event {} failed with location!".format(event.key_name))
else:
timezone_id = LocationHelper.get_timezone_id(None, lat_lon=lat_lon)
if not timezone_id:
logging.warning("Timezone update for event {} failed!".format(event.key_name))
else:
event.timezone_id = timezone_id
cls.createOrUpdate(event, run_post_update_hook=False)
# Add event to lat/lon info to search index
if lat_lon:
fields = [
search.NumberField(name='year', value=event.year),
search.GeoField(name='location', value=search.GeoPoint(lat_lon[0], lat_lon[1]))
]
search.Index(name="eventLocation").put(search.Document(doc_id=event.key.id(), fields=fields))
# Enqueue task to calculate district points
for event in events:
taskqueue.add(
url='/tasks/math/do/district_points_calc/{}'.format(event.key.id()),
method='GET')
def postUpdateHook(cls, teams, updated_attr_list, is_new_list):
"""
To run after models have been updated
"""
for (team, updated_attrs) in zip(teams, updated_attr_list):
try:
LocationHelper.update_team_location(team)
except Exception, e:
logging.error("update_team_location for {} errored!".format(
team.key.id()))
logging.exception(e)
try:
SearchHelper.update_team_location_index(team)
except Exception, e:
logging.error(
"update_team_location_index for {} errored!".format(
team.key.id()))
logging.exception(e)
def postUpdateHook(cls, events, updated_attr_list, is_new_list):
"""
To run after models have been updated
"""
for (event, updated_attrs) in zip(events, updated_attr_list):
try:
LocationHelper.update_event_location(event)
except Exception, e:
logging.error("update_event_location for {} errored!".format(event.key.id()))
logging.exception(e)
try:
if event.normalized_location and event.normalized_location.lat_lng:
timezone_id = LocationHelper.get_timezone_id(
None, lat_lng=event.normalized_location.lat_lng)
if not timezone_id:
logging.warning("Timezone update for event {} failed!".format(event.key_name))
else:
event.timezone_id = timezone_id
else:
logging.warning("No Lat/Lng to update timezone_id for event {}!".format(event.key_name))
except Exception, e:
logging.error("Timezone update for {} errored!".format(event.key.id()))
logging.exception(e)
def postUpdateHook(cls, events, updated_attr_list, is_new_list):
"""
To run after models have been updated
"""
for (event, updated_attrs) in zip(events, updated_attr_list):
lat_lon = event.get_lat_lon()
if not lat_lon:
logging.warning(
"Lat/Lon update for event {} failed with location!".format(
event.key_name))
else:
timezone_id = LocationHelper.get_timezone_id(None,
lat_lon=lat_lon)
if not timezone_id:
logging.warning(
"Timezone update for event {} failed!".format(
event.key_name))
else:
event.timezone_id = timezone_id
cls.createOrUpdate(event, run_post_update_hook=False)
# Add event to lat/lon info to search index
if lat_lon:
fields = [
search.NumberField(name='year', value=event.year),
search.GeoField(name='location',
value=search.GeoPoint(
lat_lon[0], lat_lon[1]))
]
search.Index(name="eventLocation").put(
search.Document(doc_id=event.key.id(), fields=fields))
# Enqueue task to calculate district points
for event in events:
taskqueue.add(url='/tasks/math/do/district_points_calc/{}'.format(
event.key.id()),
method='GET')
def get_lat_lon(self):
return LocationHelper.get_event_lat_lon(self)
def test_2002sj(self):
with open('test_data/usfirst_html/usfirst_event_details_2002sj.html', 'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
self.assertEqual(LocationHelper.get_lat_lon(event['location'])[0], (37.3382082, -121.8863286))
self.assertEqual(LocationHelper.get_timezone_id(event['location']), 'America/Los_Angeles')
def test_1997il(self):
with open('test_data/usfirst_html/usfirst_event_details_1997il.html', 'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
self.assertEqual(LocationHelper.get_lat_lon(event['location'])[0], (41.8781136, -87.6297982))
self.assertEqual(LocationHelper.get_timezone_id(event['location']), 'America/Chicago')
def test_2005or(self):
with open('test_data/usfirst_html/usfirst_event_details_2005or.html', 'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
self.assertEqual(LocationHelper.get_lat_lon(event['location'])[0], (45.5230622, -122.6764816))
self.assertEqual(LocationHelper.get_timezone_id(event['location']), 'America/Los_Angeles')
def get_lat_lon(self):
return LocationHelper.get_event_lat_lon(self)
def get_lat_lon(self):
return LocationHelper.get_team_lat_lon(self)
def test_2013flbr(self):
with open('test_data/usfirst_html/usfirst_event_details_2013flbr.html', 'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
self.assertEqual(LocationHelper.get_lat_lon(event['location'])[0], (26.1224386, -80.13731740000001))
self.assertEqual(LocationHelper.get_timezone_id(event['location']), 'America/New_York')
def get_lat_lon(self):
return LocationHelper.get_team_lat_lon(self)
def _render(self):
year, location, range_limit, search_type, page = self._get_params()
num_results = 0
results = []
distances = []
if location:
lat_lon = LocationHelper.get_lat_lng(location)
if lat_lon:
lat, lon = lat_lon
dist_expr = 'distance(location, geopoint({}, {}))'.format(lat, lon)
if search_type == 'teams':
query_string = '{} < {}'.format(dist_expr, range_limit * self.METERS_PER_MILE)
else:
query_string = '{} < {} AND year={}'.format(dist_expr, range_limit * self.METERS_PER_MILE, year)
offset = self.PAGE_SIZE * page
query = search.Query(
query_string=query_string,
options=search.QueryOptions(
limit=self.PAGE_SIZE,
offset=offset,
sort_options=search.SortOptions(
expressions=[
search.SortExpression(
expression=dist_expr,
direction=search.SortExpression.ASCENDING
)
]
),
returned_expressions=[
search.FieldExpression(
name='distance',
expression=dist_expr
)
],
)
)
if search_type == 'teams':
search_index = search.Index(name="teamLocation")
else:
search_index = search.Index(name="eventLocation")
docs = search_index.search(query)
num_results = docs.number_found
distances = {}
keys = []
event_team_count_futures = {}
for result in docs.results:
distances[result.doc_id] = result.expressions[0].value / self.METERS_PER_MILE
if search_type == 'teams':
event_team_count_futures[result.doc_id] = EventTeam.query(
EventTeam.team == ndb.Key('Team', result.doc_id),
EventTeam.year == year).count_async(limit=1, keys_only=True)
keys.append(ndb.Key('Team', result.doc_id))
else:
keys.append(ndb.Key('Event', result.doc_id))
result_futures = ndb.get_multi_async(keys)
if search_type == 'teams':
results = []
for result_future, team_key in zip(result_futures, keys):
if event_team_count_futures[team_key.id()].get_result() != 0:
results.append(result_future.get_result())
else:
results = [result_future.get_result() for result_future in result_futures]
self.template_values.update({
'valid_years': self.VALID_YEARS,
'valid_ranges': self.VALID_RANGES,
'page_size': self.PAGE_SIZE,
'page': page,
'year': year,
'location': location,
'range_limit': range_limit,
'search_type': search_type,
'num_results': num_results,
'results': results,
'distances': distances,
})
return jinja2_engine.render('nearby.html', self.template_values)
def test_2012ct(self):
with open('test_data/usfirst_html/usfirst_event_details_2012ct.html', 'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
self.assertEqual(LocationHelper.get_lat_lon(event['location'])[0], (41.76371109999999, -72.6850932))
self.assertEqual(LocationHelper.get_timezone_id(event['location']), 'America/New_York')
def _render(self):
year, location, range_limit, search_type, page = self._get_params()
num_results = 0
results = []
distances = []
if location:
lat_lon, _ = LocationHelper.get_lat_lon(location, geocode=True)
if lat_lon:
lat, lon = lat_lon
dist_expr = 'distance(location, geopoint({}, {}))'.format(
lat, lon)
if search_type == 'teams':
query_string = '{} < {}'.format(
dist_expr, range_limit * self.METERS_PER_MILE)
else:
query_string = '{} < {} AND year={}'.format(
dist_expr, range_limit * self.METERS_PER_MILE, year)
offset = self.PAGE_SIZE * page
query = search.Query(
query_string=query_string,
options=search.QueryOptions(
limit=self.PAGE_SIZE,
offset=offset,
sort_options=search.SortOptions(expressions=[
search.SortExpression(
expression=dist_expr,
direction=search.SortExpression.ASCENDING)
]),
returned_expressions=[
search.FieldExpression(name='distance',
expression=dist_expr)
],
))
if search_type == 'teams':
search_index = search.Index(name="teamLocation")
else:
search_index = search.Index(name="eventLocation")
docs = search_index.search(query)
num_results = docs.number_found
distances = {}
keys = []
event_team_count_futures = {}
for result in docs.results:
distances[result.doc_id] = result.expressions[
0].value / self.METERS_PER_MILE
if search_type == 'teams':
event_team_count_futures[
result.doc_id] = EventTeam.query(
EventTeam.team == ndb.Key(
'Team', result.doc_id),
EventTeam.year == year).count_async(
limit=1, keys_only=True)
keys.append(ndb.Key('Team', result.doc_id))
else:
keys.append(ndb.Key('Event', result.doc_id))
result_futures = ndb.get_multi_async(keys)
if search_type == 'teams':
results = []
for result_future, team_key in zip(result_futures, keys):
if event_team_count_futures[
team_key.id()].get_result() != 0:
results.append(result_future.get_result())
else:
results = [
result_future.get_result()
for result_future in result_futures
]
self.template_values.update({
'valid_years': self.VALID_YEARS,
'valid_ranges': self.VALID_RANGES,
'page_size': self.PAGE_SIZE,
'page': page,
'year': year,
'location': location,
'range_limit': range_limit,
'search_type': search_type,
'num_results': num_results,
'results': results,
'distances': distances,
})
return jinja2_engine.render('nearby.html', self.template_values)
def test_2005is(self):
with open('test_data/usfirst_html/usfirst_event_details_2005is.html', 'r') as f:
event, _ = UsfirstEventDetailsParser.parse(f.read())
self.assertEqual(LocationHelper.get_lat_lon(event['location'])[0], (32.7940463, 34.989571))
self.assertEqual(LocationHelper.get_timezone_id(event['location']), 'Asia/Jerusalem')