def _inner_make_event_findable_for_web_event(db_event, web_event, update_geodata):
logging.info("Making web_event %s findable." % db_event.id)
db_event.web_event = web_event
db_event.fb_event = None
db_event.owner_fb_uid = None
db_event.attendee_count = 0 # Maybe someday set attending counts when we fetch them?
db_event.start_time = datetime.datetime.strptime(web_event['start_time'], DATETIME_FORMAT)
if web_event.get('end_time'):
db_event.end_time = datetime.datetime.strptime(web_event['end_time'], DATETIME_FORMAT)
else:
db_event.end_time = None
db_event.search_time_period = _event_time_period(db_event)
db_event.event_keywords = event_classifier.relevant_keywords(db_event)
db_event.auto_categories = [x.index_name for x in categories.find_styles(db_event) + categories.find_event_types(db_event)]
geocode = None
if web_event.get('location_address'):
address = event_locations.clean_address(web_event.get('location_address'))
logging.info("Have location address, checking if it is geocodable: %s", web_event.get('location_address'))
logging.info("Stripping off any floor info, final address is: %s", address)
geocode = gmaps_api.lookup_address(address)
if geocode is None:
logging.warning("Received a location_address that was not geocodeable, treating as empty: %s", web_event['location_address'])
if geocode is None:
if web_event.get('latitude') or web_event.get('longitude'):
logging.info("Have latlong, let's geocode that way: %s, %s", web_event.get('latitude'), web_event.get('longitude'))
geocode = gmaps_api.lookup_latlng((web_event.get('latitude'), web_event.get('longitude')))
if geocode is None:
if web_event.get('geolocate_location_name'):
logging.info("Have magic geolocate_location_name, checking if it is a place: %s", web_event.get('geolocate_location_name'))
geocode = gmaps_api.lookup_location(web_event['geolocate_location_name'])
if geocode is None:
if web_event.get('location_name'):
logging.info("Have regular location_name, checking if it is a place: %s", web_event.get('location_name'))
geocode = gmaps_api.lookup_location(web_event['location_name'])
if geocode:
if 'name' in geocode.json_data:
web_event['location_name'] = geocode.json_data['name']
web_event['location_address'] = geocode.json_data['formatted_address']
logging.info("Found an address: %s", web_event['location_address'])
# BIG HACK!!!
if 'Japan' not in web_event['location_address'] and 'Korea' not in web_event['location_address']:
logging.error("Found incorrect address for venue!")
latlng = geocode.json_data['geometry']['location']
web_event['latitude'] = latlng['lat']
web_event['longitude'] = latlng['lng']
db_event.address = web_event.get('location_address')
_inner_cache_photo(db_event)
if update_geodata:
# Don't use cached/stale geocode when constructing the LocationInfo here
db_event.location_geocode = geocode
location_info = event_locations.LocationInfo(db_event=db_event)
_update_geodata(db_event, location_info)
def basic_match(fb_event):
found_styles = categories.find_styles(fb_event)
if not full_run:
# debug
print fb_event['info'].get('name', '').encode('utf-8')
print fb_event['info'].get('description', '').encode('utf-8')
if event_types.LOCK in found_styles:
print fb_event['info']['id']
#contexts = categories.get_context(fb_event, styles['LOCK'])
#for keyword, context in zip(styles['LOCK'], contexts):
# print keyword, repr(context)
return event_types.LOCK in found_styles
def basic_match(fb_event):
found_styles = categories.find_styles(fb_event)
if not full_run:
# debug
print fb_event['info'].get('name', '').encode('utf-8')
print fb_event['info'].get('description', '').encode('utf-8')
if event_types.LOCK in found_styles:
print fb_event['info']['id']
#contexts = categories.get_context(fb_event, styles['LOCK'])
#for keyword, context in zip(styles['LOCK'], contexts):
# print keyword, repr(context)
return event_types.LOCK in found_styles
def _inner_make_event_findable_for_fb_event(db_event, fb_dict, update_geodata):
"""set up any cached fields or bucketing or whatnot for this event"""
# Update this event with the latest time_period regardless (possibly overwritten below)
db_event.search_time_period = _event_time_period(db_event)
if fb_dict['empty'] == fb_api.EMPTY_CAUSE_DELETED:
# If this event has already past, don't allow it to be deleted. We want to keep history!
if db_event.end_time and db_event.end_time < datetime.datetime.now(
) - datetime.timedelta(days=2):
return
# If we don't have a db_event.end_time, then we've got something messed up, so let's delete the event
db_event.start_time = None
db_event.end_time = None
db_event.search_time_period = None
db_event.address = None
db_event.actual_city_name = None
db_event.city_name = None
db_event.fb_event = fb_dict
return
elif fb_dict['empty'] == fb_api.EMPTY_CAUSE_INSUFFICIENT_PERMISSIONS:
db_event.search_time_period = _event_time_period(db_event)
# Don't copy the fb_event over, or any of its fields
return
# Screw db-normalized form, store this here (and location_geocode down below)
db_event.fb_event = fb_dict
if 'owner' in fb_dict['info']:
db_event.owner_fb_uid = fb_dict['info']['owner']['id']
else:
db_event.owner_fb_uid = None
db_event.attendee_count = _all_attending_count(fb_dict)
db_event.start_time = dates.parse_fb_start_time(fb_dict)
db_event.end_time = dates.parse_fb_end_time(fb_dict)
db_event.search_time_period = _event_time_period(db_event)
db_event.event_keywords = event_classifier.relevant_keywords(fb_dict)
db_event.auto_categories = [
x.index_name for x in categories.find_styles(fb_dict) +
categories.find_event_types(fb_dict)
]
if update_geodata:
# Don't use cached/stale geocode when constructing the LocationInfo here
db_event.location_geocode = None
location_info = event_locations.LocationInfo(fb_dict,
db_event=db_event)
_update_geodata(db_event, location_info)
def _inner_make_event_findable_for_fb_event(db_event, fb_dict, update_geodata):
"""set up any cached fields or bucketing or whatnot for this event"""
# Update this event with the latest time_period regardless (possibly overwritten below)
db_event.search_time_period = _event_time_period(db_event)
if fb_dict['empty'] == fb_api.EMPTY_CAUSE_DELETED:
# If this event has already past, don't allow it to be deleted. We want to keep history!
if db_event.end_time and db_event.end_time < datetime.datetime.now() - datetime.timedelta(days=2):
return
# If we don't have a db_event.end_time, then we've got something messed up, so let's delete the event
db_event.start_time = None
db_event.end_time = None
db_event.search_time_period = None
db_event.address = None
db_event.actual_city_name = None
db_event.city_name = None
db_event.fb_event = fb_dict
return
elif fb_dict['empty'] == fb_api.EMPTY_CAUSE_INSUFFICIENT_PERMISSIONS:
db_event.search_time_period = _event_time_period(db_event)
# Don't copy the fb_event over, or any of its fields
return
# Screw db-normalized form, store this here (and location_geocode down below)
db_event.fb_event = fb_dict
if 'owner' in fb_dict['info']:
db_event.owner_fb_uid = fb_dict['info']['owner']['id']
else:
db_event.owner_fb_uid = None
db_event.attendee_count = _all_attending_count(fb_dict)
db_event.start_time = dates.parse_fb_start_time(fb_dict)
db_event.end_time = dates.parse_fb_end_time(fb_dict)
db_event.search_time_period = _event_time_period(db_event)
db_event.event_keywords = event_classifier.relevant_keywords(fb_dict)
db_event.auto_categories = [x.index_name for x in categories.find_styles(fb_dict) + categories.find_event_types(fb_dict)]
_inner_cache_photo(db_event)
if update_geodata:
# Don't use cached/stale geocode when constructing the LocationInfo here
db_event.location_geocode = None
location_info = event_locations.LocationInfo(fb_dict, db_event=db_event)
_update_geodata(db_event, location_info)
def _inner_make_event_findable_for_web_event(db_event, web_event, disable_updates):
logging.info("Making web_event %s findable." % db_event.id)
db_event.web_event = web_event
db_event.fb_event = None
db_event.owner_fb_uid = None
db_event.attendee_count = 0 # Maybe someday set attending counts when we fetch them?
if web_event['start_time'].endswith('Z'):
web_event['start_time'] = web_event['start_time'][:-1]
if web_event['end_time'] and web_event['end_time'].endswith('Z'):
web_event['end_time'] = web_event['start_time'][:-1]
db_event.start_time = dateutil.parser.parse(web_event['start_time']).replace(tzinfo=None)
if web_event.get('end_time'):
db_event.end_time = dateutil.parser.parse(web_event['end_time']).replace(tzinfo=None)
else:
db_event.end_time = None
db_event.search_time_period = _event_time_period(db_event)
db_event.event_keywords = event_classifier.relevant_keywords(db_event)
db_event.auto_categories = [x.index_name for x in categories.find_styles(db_event) + categories.find_event_types(db_event)]
geocode = None
if web_event.get('location_address'):
address = clean_address(web_event.get('location_address'))
logging.info("Have location address, checking if it is geocodable: %s", web_event.get('location_address'))
logging.info("Stripping off any floor info, final address is: %s", address)
geocode = gmaps_api.lookup_address(address)
if geocode is None:
logging.warning("Received a location_address that was not geocodeable, treating as empty: %s", web_event['location_address'])
if geocode is None:
if web_event.get('latitude') or web_event.get('longitude'):
logging.info("Have latlong, let's geocode that way: %s, %s", web_event.get('latitude'), web_event.get('longitude'))
geocode = gmaps_api.lookup_latlng((web_event.get('latitude'), web_event.get('longitude')))
if geocode is None:
if web_event.get('geolocate_location_name'):
logging.info("Have magic geolocate_location_name, checking if it is a place: %s", web_event.get('geolocate_location_name'))
geocode = gmaps_api.lookup_location(web_event['geolocate_location_name'])
if geocode is None:
if web_event.get('location_name'):
logging.info("Have regular location_name, checking if it is a place: %s", web_event.get('location_name'))
geocode = gmaps_api.lookup_location(web_event['location_name'])
if geocode:
if 'name' in geocode.json_data:
web_event['location_name'] = geocode.json_data['name']
web_event['location_address'] = geocode.json_data['formatted_address']
logging.info("Found an address: %s", web_event['location_address'])
# We have a formatted_address, but that's it. Let's get a fully componentized address
geocode_with_address = gmaps_api.lookup_address(clean_address(web_event['location_address']))
if not geocode_with_address:
logging.error("Could not get geocode for: %s", web_event['location_address'])
elif 'address_components' not in geocode_with_address.json_data:
logging.error("Could not get fully parsed address for: %s: %s", web_event['location_address'], geocode_with_address)
pass
elif 'country' not in web_event:
web_event['country'] = geocode_with_address.country()
elif web_event['country'] != geocode_with_address.country():
logging.error(
"Found incorrect address for venue! Expected %s but found %s", web_event['country'], geocode_with_address.country()
)
latlng = geocode.json_data['geometry']['location']
web_event['latitude'] = latlng['lat']
web_event['longitude'] = latlng['lng']
# Add timezones, and save them to the web_event strings, for use by eventdata accessors
timezone_string = timezone_finder.closest_timezone_at(lat=latlng['lat'], lng=latlng['lng'])
web_event['timezone'] = timezone_string
if timezone_string:
tz = pytz.timezone(timezone_string)
web_event['start_time'] = tz.localize(db_event.start_time).strftime(DATETIME_FORMAT_TZ)
if db_event.end_time:
web_event['end_time'] = tz.localize(db_event.end_time).strftime(DATETIME_FORMAT_TZ)
else:
logging.error('No timezone string found for latlng: %s', latlng)
db_event.address = web_event.get('location_address')
_inner_common_setup(db_event, disable_updates=disable_updates)
if 'geodata' not in (disable_updates or []):
# Don't use cached/stale geocode when constructing the LocationInfo here
#db_event.location_geocode = geocode
location_info = event_locations.LocationInfo(db_event=db_event)
_update_geodata(db_event, location_info, disable_updates)
def get(self):
event_id = None
if self.request.get('event_url'):
event_id = urls.get_event_id_from_url(self.request.get('event_url'))
elif self.request.get('event_id'):
event_id = self.request.get('event_id')
self.finish_preload()
fb_event = get_fb_event(self.fbl, event_id)
if not fb_event:
logging.error('No fetched data for %s, showing error page', event_id)
return self.show_barebones_page(event_id, "No fetched data")
e = eventdata.DBEvent.get_by_id(event_id)
if not fb_events.is_public_ish(fb_event):
if e:
fb_event = e.fb_event
else:
self.add_error('Cannot add secret/closed events to dancedeets!')
self.errors_are_fatal()
owner_location = None
if 'owner' in fb_event['info']:
owner_id = fb_event['info']['owner']['id']
location = self._get_location(owner_id, fb_api.LookupProfile, 'profile'
) or self._get_location(owner_id, fb_api.LookupThingPage, 'info')
if location:
owner_location = event_locations.city_for_fb_location(location)
self.display['owner_location'] = owner_location
display_event = search.DisplayEvent.get_by_id(event_id)
# Don't insert object until we're ready to save it...
if e and e.creating_fb_uid:
#STR_ID_MIGRATE
creating_user = self.fbl.get(fb_api.LookupProfile, str(e.creating_fb_uid))
if creating_user.get('empty'):
logging.warning(
'Have creating-user %s...but it is not publicly visible, so treating as None: %s', e.creating_fb_uid, creating_user
)
creating_user = None
else:
creating_user = None
potential_event = potential_events.make_potential_event_without_source(event_id)
classified_event = event_classifier.get_classified_event(fb_event, potential_event.language)
self.display['classified_event'] = classified_event
dance_words_str = ', '.join(list(classified_event.dance_matches()))
if classified_event.is_dance_event():
event_words_str = ', '.join(list(classified_event.event_matches()))
else:
event_words_str = 'NONE'
self.display['classifier_dance_words'] = dance_words_str
self.display['classifier_event_words'] = event_words_str
self.display['creating_user'] = creating_user
self.display['potential_event'] = potential_event
self.display['display_event'] = display_event
add_result = event_auto_classifier.is_auto_add_event(classified_event)
notadd_result = event_auto_classifier.is_auto_notadd_event(classified_event, auto_add_result=add_result)
auto_classified = ''
if add_result[0]:
auto_classified += 'add: %s.\n' % add_result[1]
if notadd_result[0]:
auto_classified += 'notadd: %s.\n' % notadd_result[1]
self.display['auto_classified_types'] = auto_classified
styles = categories.find_styles(fb_event)
event_types = styles + categories.find_event_types(fb_event)
self.display['auto_categorized_types'] = ', '.join(x.public_name for x in event_types)
location_info = event_locations.LocationInfo(fb_event, db_event=e, debug=True)
self.display['location_info'] = location_info
if location_info.fb_address:
fb_geocode = gmaps_api.lookup_address(location_info.fb_address)
self.display['fb_geocoded_address'] = formatting.format_geocode(fb_geocode)
else:
self.display['fb_geocoded_address'] = ''
city_name = 'Unknown'
if location_info.geocode:
city = cities_db.get_nearby_city(location_info.geocode.latlng(), country=location_info.geocode.country())
if city:
city_name = city.display_name()
self.display['ranking_city_name'] = city_name
fb_event_attending_maybe = get_fb_event(self.fbl, event_id, lookup_type=fb_api.LookupEventAttendingMaybe)
matcher = event_attendee_classifier.get_matcher(self.fbl, fb_event, fb_event_attending_maybe)
# print '\n'.join(matcher.results)
sorted_matches = sorted(matcher.matches, key=lambda x: -len(x.overlap_ids))
matched_overlap_ids = sorted_matches[0].overlap_ids if matcher.matches else []
self.display['auto_add_attendee_ids'] = sorted(matched_overlap_ids)
self.display['overlap_results'] = ['%s %s: %s' % (x.top_n, x.name, x.reason) for x in sorted_matches]
self.display['overlap_attendee_ids'] = sorted(matcher.overlap_ids)
if matcher.matches:
attendee_ids_to_admin_hash_and_event_ids = sorted_matches[0].get_attendee_lookups()
self.display['attendee_ids_to_admin_hash_and_event_ids'] = attendee_ids_to_admin_hash_and_event_ids
self.display['event'] = e
self.display['event_id'] = event_id
self.display['fb_event'] = fb_event
self.jinja_env.filters['highlight_keywords'] = event_classifier.highlight_keywords
self.display['track_analytics'] = False
self.render_template('admin_edit')
def get(self):
event_id = None
if self.request.get('event_url'):
event_id = urls.get_event_id_from_url(self.request.get('event_url'))
elif self.request.get('event_id'):
event_id = self.request.get('event_id')
self.fbl.request(fb_api.LookupEvent, event_id, allow_cache=False)
# DISABLE_ATTENDING
# self.fbl.request(fb_api.LookupEventAttending, event_id, allow_cache=False)
self.finish_preload()
try:
fb_event = self.fbl.fetched_data(fb_api.LookupEvent, event_id)
# DISABLE_ATTENDING
fb_event_attending = None
# fb_event_attending = fbl.fetched_data(fb_api.LookupEventAttending, event_id)
except fb_api.NoFetchedDataException:
return self.show_barebones_page(event_id, "No fetched data")
if not fb_events.is_public_ish(fb_event):
self.add_error('Cannot add secret/closed events to dancedeets!')
self.errors_are_fatal()
owner_location = None
if 'owner' in fb_event['info']:
owner_id = fb_event['info']['owner']['id']
location = self._get_location(owner_id, fb_api.LookupProfile, 'profile') or self._get_location(owner_id, fb_api.LookupThingFeed, 'info')
if location:
owner_location = event_locations.city_for_fb_location(location)
self.display['owner_location'] = owner_location
display_event = search.DisplayEvent.get_by_id(event_id)
# Don't insert object until we're ready to save it...
e = eventdata.DBEvent.get_by_id(event_id)
if e and e.creating_fb_uid:
creating_user = self.fbl.get(fb_api.LookupUser, e.creating_fb_uid)
else:
creating_user = None
potential_event = potential_events.make_potential_event_without_source(event_id, fb_event, fb_event_attending)
classified_event = event_classifier.get_classified_event(fb_event, potential_event.language)
self.display['classified_event'] = classified_event
if classified_event.is_dance_event():
dance_words_str = ', '.join(list(classified_event.dance_matches()))
event_words_str = ', '.join(list(classified_event.event_matches()))
else:
dance_words_str = 'NONE'
event_words_str = 'NONE'
self.display['classifier_dance_words'] = dance_words_str
self.display['classifier_event_words'] = event_words_str
self.display['creating_user'] = creating_user
self.display['potential_event'] = potential_event
self.display['display_event'] = display_event
add_result = event_auto_classifier.is_auto_add_event(classified_event)
notadd_result = event_auto_classifier.is_auto_notadd_event(classified_event, auto_add_result=add_result)
auto_classified = ''
if add_result[0]:
auto_classified += 'add: %s.\n' % add_result[1]
if notadd_result[0]:
auto_classified += 'notadd: %s.\n' % notadd_result[1]
self.display['auto_classified_types'] = auto_classified
styles = categories.find_styles(fb_event) + categories.find_event_types(fb_event)
self.display['auto_categorized_types'] = ', '.join(x.public_name for x in styles)
location_info = event_locations.LocationInfo(fb_event, db_event=e, debug=True)
self.display['location_info'] = location_info
fb_geocode = gmaps_api.lookup_address(location_info.fb_address)
self.display['fb_geocoded_address'] = formatting.format_geocode(fb_geocode)
self.display['event'] = e
self.display['event_id'] = event_id
self.display['fb_event'] = fb_event
self.jinja_env.filters['highlight_keywords'] = event_classifier.highlight_keywords
self.display['track_analytics'] = False
self.render_template('admin_edit')
def _inner_make_event_findable_for_web_event(db_event, json_body,
update_geodata):
logging.info("Making web_event %s findable." % db_event.id)
db_event.web_event = json_body
db_event.fb_event = None
db_event.owner_fb_uid = None
db_event.attendee_count = 0 # Maybe someday set attending counts when we fetch them?
db_event.start_time = datetime.datetime.strptime(json_body['start_time'],
DATETIME_FORMAT)
if json_body.get('end_time'):
db_event.end_time = datetime.datetime.strptime(json_body['end_time'],
DATETIME_FORMAT)
else:
db_event.end_time = None
db_event.search_time_period = _event_time_period(db_event)
db_event.event_keywords = event_classifier.relevant_keywords(db_event)
db_event.auto_categories = [
x.index_name for x in categories.find_styles(db_event) +
categories.find_event_types(db_event)
]
geocode = None
if json_body.get('location_address'):
logging.info("Have location address, checking if it is geocodable: %s",
json_body.get('location_address'))
geocode = gmaps_api.get_geocode(address=json_body['location_address'])
if geocode is None:
logging.warning(
"Received a location_address that was not geocodeable, treating as empty: %s",
json_body['location_address'])
if geocode is None:
results = None
if json_body.get('geolocate_location_name'):
logging.info(
"Have magic geolocate_location_name, checking if it is a place: %s",
json_body.get('geolocate_location_name'))
results = gmaps_api.fetch_place_as_json(
query=json_body['geolocate_location_name'])
if not results or results['status'] == 'ZERO_RESULTS':
if json_body.get('location_name'):
logging.info(
"Have regular location_name, checking if it is a place: %s",
json_body.get('location_name'))
results = gmaps_api.fetch_place_as_json(
query=json_body['location_name'])
if results and results['status'] == 'OK':
result = results['results'][0]
json_body['location_name'] = result['name']
json_body['location_address'] = result['formatted_address']
logging.info("Found an address: %s", json_body['location_address'])
# BIG HACK!!!
if 'Japan' not in json_body[
'location_address'] and 'Korea' not in json_body[
'location_address']:
logging.error("Found incorrect address for venue!")
latlng = result['geometry']['location']
json_body['latitude'] = latlng['lat']
json_body['longitude'] = latlng['lng']
db_event.address = json_body.get('location_address')
if db_event.image_url:
try:
mimetype, image_data = fetch.fetch_data(db_event.image_url)
image = images.Image(image_data)
json_body['photo_width'] = image.width
json_body['photo_height'] = image.height
logging.info("Found image width size %sx%s", image.width,
image.height)
except urllib2.HTTPError:
logging.warning("Couldn't find image: %s", db_event.image_url)
if update_geodata:
# Don't use cached/stale geocode when constructing the LocationInfo here
db_event.location_geocode = None
location_info = event_locations.LocationInfo(db_event=db_event)
_update_geodata(db_event, location_info)