def update_event(self, event_id, interest_key):
user_id = UserController().current_user_id
if user_id:
user_event = UserEvent.query.filter(
and_(UserEvent.user_id == user_id,
UserEvent.event_id == event_id)).first()
if user_event:
if interest_key == UserEvent.DONE:
if user_event.interest_key == interest_key:
user_event.interest = user_event.interest - 2
else:
user_event.interest = user_event.interest + 2
elif user_event.interest_key == interest_key:
user_event.interest = None
else:
user_event.interest = UserEvent.interest_level(
interest_key)
db_session.merge(user_event)
else:
user_event = UserEvent(
user_id=user_id,
event_id=event_id,
interest=UserEvent.interest_level(interest_key))
db_session.add(user_event)
db_session.commit()
return self.get_event(event_id)
return None
def _request_user_info(self):
credentials = google.oauth2.credentials.Credentials(
**session['credentials'])
people_service = build('people', 'v1', credentials=credentials)
profile = people_service.people().get(
resourceName='people/me',
personFields='names,emailAddresses,photos').execute()
primary_email = profile['emailAddresses'][0]
for cur_email in profile['emailAddresses']:
if cur_email['metadata']['primary']:
primary_email = cur_email
primary_name = profile['names'][0]
for cur_name in profile['names']:
if cur_name['metadata']['primary']:
primary_name = cur_name
primary_photo = profile['photos'][0]
for cur_photo in profile['photos']:
if cur_photo['metadata']['primary']:
primary_photo = cur_photo
google_auth_id = profile['resourceName'].split("/")[1]
user = {
'username': primary_email['value'].split("@")[0],
'email': primary_email['value'],
'display_name': primary_name['displayName'],
'first_name': primary_name['givenName'],
'last_name': primary_name['familyName'],
'image_url': primary_photo['url'],
}
row_user_auth = UserAuth.query.filter(
and_(UserAuth.auth_key == Auth.GOOGLE,
UserAuth.auth_id == google_auth_id)).first()
if not row_user_auth:
row_user = User(**user)
db_session.add(row_user)
db_session.commit()
row_user_auth = UserAuth(user_id=row_user.user_id,
auth_key=Auth.GOOGLE,
auth_id=google_auth_id)
db_session.add(row_user_auth)
db_session.commit()
else:
row_user = row_user_auth.user
for k, v in user.items():
setattr(row_user, k, v)
db_session.merge(row_user)
db_session.commit()
session['user'] = row_user.to_json()
return session['user']
def block_user(self, identifier, active):
current_user = self.current_user
user = self._get_user(identifier)
row_block = Block(user_id=current_user.user_id,
block_id=user.user_id,
active=active)
db_session.merge(row_block)
db_session.commit()
return self._get_user(identifier)
def follow_user(self, identifier, active):
current_user = self.current_user
user = self._get_user(identifier)
row_follow = Follow(user_id=current_user.user_id,
follow_id=user.user_id,
active=active)
db_session.merge(row_follow)
db_session.commit()
return self._get_user(identifier)
def get_connector(klass, read_only=False):
connector = ConnectorEvent.query.filter(
and_(ConnectorEvent.connector_type == klass.TYPE,
ConnectorEvent.connector_event_id == klass.ID)).first()
if not connector and not read_only:
connector = ConnectorEvent(connector_type=klass.TYPE,
connector_event_id=klass.ID)
connector.data = {}
db_session.merge(connector)
db_session.commit()
return connector
def extract(self):
row_connector_event = ConnectorEvent.query.filter(
and_(ConnectorEvent.connector_event_id == self.ID,
ConnectorEvent.connector_type == self.TYPE)).first()
if not row_connector_event:
row_connector_event = ConnectorEvent(connector_event_id=self.ID,
connector_type=self.TYPE)
events = {self.create_key(x['name'], x['city']): x for x in self.data}
row_connector_event.data = events
db_session.merge(row_connector_event)
db_session.commit()
for key, restaurant in row_connector_event.data.items():
yield restaurant['name'], restaurant
def seed(klass, purge=None):
u = User.query.filter(User.email == "*****@*****.**").first()
if not u:
u = User(email="*****@*****.**")
u.username = "******",
u.first_name = "Michael"
u.last_name = "Lin"
u.image_url = "https://pbs.twimg.com/profile_images/1115632604626259973/kMoP8dTJ_400x400.png"
db_session.merge(u)
db_session.commit()
u = User.query.filter(User.email == "*****@*****.**").first()
events = Event.query.filter(and_(Event.primary_type == Tag.FOOD_DRINK))
if purge:
UserEvent.query.filter(UserEvent.user_id == u.user_id).delete()
tier_to_interest = {
'♡': UserEvent.interest_level(UserEvent.GO),
'☆': UserEvent.interest_level(UserEvent.MAYBE)
}
for i, e in enumerate(events):
print(i, e.event_id, e.name)
tier = get_from(e.meta, [ExtractMMV.TYPE, 'tier'])
if tier in ['♡', '☆']:
print("\tAdded {}".format(tier))
interest = tier_to_interest[tier]
ue = UserEvent.query.filter(
UserEvent.user_id == u.user_id,
UserEvent.event_id == e.event_id).first()
if ue:
ue.interest = interest
else:
ue = UserEvent(user_id=u.user_id,
event_id=e.event_id,
interest=interest)
db_session.merge(ue)
db_session.commit()
def extract(self, start_date=None, end_date=None):
discover = tmdb.Discover()
event_params = {
'include_adult': False,
'include_video': False,
'sort_by': 'release_date.asc',
'with_original_language': 'en',
'vote_average.gte': 7,
'vote_count.gte': 1000
}
event_params.update(
self.parse_time_args(start_date=start_date, end_date=end_date))
print(json.dumps(event_params, indent=4))
sentinel = True
while sentinel:
raw_events = discover.movie(**event_params)
for i, event in enumerate(raw_events['results']):
connector_event_id = str(event['id'])
row_connector_event = ConnectorEvent.query.filter(
and_(
ConnectorEvent.connector_event_id ==
connector_event_id,
ConnectorEvent.connector_type == self.TYPE)).first()
if not row_connector_event:
row_connector_event = ConnectorEvent(
connector_event_id=connector_event_id,
connector_type=self.TYPE,
data=event)
db_session.merge(row_connector_event)
db_session.commit()
event_name = row_connector_event.data['title']
event_description = row_connector_event.data['overview']
event_short_name = row_connector_event.data['title']
event_img_url = 'https://image.tmdb.org/t/p/w342{}'.format(
row_connector_event.data['poster_path'])
event_backdrop_url = 'https://image.tmdb.org/t/p/original{}'.format(
row_connector_event.data['backdrop_path'])
event_start_time = datetime.datetime.strptime(
row_connector_event.data['release_date'], "%Y-%m-%d")
event_end_time = event_start_time + datetime.timedelta(
days=180)
genres = self.genres_by_ids(
row_connector_event.data['genre_ids'])
if row_connector_event.event_id:
row_event = Event.query.filter(
Event.event_id ==
row_connector_event.event_id).first()
row_event.name = event_name
row_event.description = event_description
row_event.short_name = event_name
row_event.img_url = event_img_url
row_event.backdrop_url = event_backdrop_url
row_event.start_time = event_start_time
row_event.end_time = event_end_time
db_session.merge(row_event)
db_session.commit()
row_event.remove_all_tags()
else:
row_event = Event(name=event_name,
description=event_description,
short_name=event_short_name,
img_url=event_img_url,
backdrop_url=event_backdrop_url,
start_time=event_start_time,
end_time=event_end_time)
row_event.primary_type = Tag.TVM
db_session.add(row_event)
db_session.commit()
row_connector_event.event_id = row_event.event_id
db_session.merge(row_connector_event)
db_session.commit()
for genre in genres:
row_event.add_tag(genre, Tag.TVM)
row_event.update_meta(self.TYPE, event)
db_session.merge(row_event)
db_session.commit()
yield row_event, "{} {}".format(row_event.name, genres)
del raw_events['results']
sentinel = raw_events['page'] < raw_events['total_pages']
if sentinel:
event_params['page'] = raw_events['page'] + 1
def extract(self, name=None, bad_city=None):
# Call the Sheets API
sheet = self.service.spreadsheets()
result = sheet.values().get(spreadsheetId=self.MM_VILLAGE_TRIX_ID,
range=self.MM_VILLAGE_SHEET_ID).execute()
rows = result.get('values', [])
self.data = []
headers = None
for row in rows:
if headers is None:
headers = [col.lower() for col in row]
continue
obj = {
val: get_from(row, [i], None)
for i, val in enumerate(headers)
}
alias = re.sub(r'-+', '-',
re.sub(r'[^a-z0-9]', "-", obj['address'].lower()))
obj['alias'] = alias
place_id = obj['place id']
if not (place_id and alias):
print(
"Missing Alias or Place ID: {} | place_id: {} | alias: {}".
format(obj['name'], place_id, alias))
print("\t{}".format(obj))
continue
connector_event_id = place_id
location_name = obj['name']
location_short_name = location_name
location_description = obj['notes']
location_categories = set(
[x.strip() for x in re.split(r'[/;]', obj['categories'])])
location_rating = obj['tier']
addr_components = [x.strip() for x in obj['address'].split(",")]
addr_street = " ".join(addr_components[:-3])
addr_city = addr_components[-3]
addr_state = addr_components[-2].split(" ")[0]
addr_country = addr_components[-1]
obj_city = obj['city']
city_sub = {
"SF": "San Francisco",
"South SF": "South San Francisco",
"San Jose - Local": "San Jose"
}
obj_city = get_from(city_sub, [obj_city], obj_city)
if obj_city != addr_city:
print("{} => \n\ttarget: {} | actual: {}\n".format(
location_name, obj_city, addr_city))
continue
elif addr_state not in ("CA", "California"):
print("{} => \n\tactual: {}\n".format(location_name,
addr_state))
continue
if bad_city:
continue
if name is not None and location_name != name:
continue
location_tags = set([x.strip() for x in obj['tags'].split(',')])
filter_location_tags = {
"aquarium",
"bakery",
"bar",
"cafe",
"campground",
"car_repair",
"grocery_or_supermarket",
"library",
"movie_theater",
"museum",
"natural_feature",
"park",
"spa",
"stadium",
"supermarket",
"tourist_attraction",
"zoo",
}
location_tags = location_tags & filter_location_tags
# location_categories = location_categories | location_tags
if location_rating not in ['♡', '☆', '◎']:
continue
obj['name'] = location_name
obj['short_name'] = location_short_name
obj['description'] = location_description
obj['address'] = addr_street
obj['city'] = addr_city
obj['state'] = addr_state
primary_tag_type = Tag.FOOD_DRINK
if ('Culture' in location_categories
or 'Entertainment' in location_categories
or 'Fitness' in location_categories
or 'Nature' in location_categories
or 'aquarium' in location_categories
or 'campground' in location_categories
or 'library' in location_categories
or 'movie_theater' in location_categories
or 'museum' in location_categories
or 'natural_feature' in location_categories
or 'park' in location_categories
or 'spa' in location_categories
or 'stadium' in location_categories
or 'tourist_attraction' in location_categories
or 'zoo' in location_categories):
primary_tag_type = Tag.ACTIVITY
if ('Utilities' in location_categories
or 'car_repair' in location_categories):
primary_tag_type = Tag.SERVICES
category_remappings = {
'Entertainment': None,
'Western': ['European'],
'SEA': ['Southeast'],
'PMT': ['Asian', 'Boba'],
'Fine Dining': ['Dinner', 'Fine Dining'],
'Mochi': ['Asian', 'Mochi'],
"car_repair": ["Repair"],
"grocery_or_supermarket": ["Market"],
"movie_theater": ["Theatre"],
"natural_feature": ["Nature"],
"park": ["Nature"],
'tourist_attraction': None
}
if primary_tag_type != Tag.FOOD_DRINK:
continue
for cats in location_categories:
if cats:
cats = get_from(category_remappings, [cats], [cats])
if cats:
cats = [x for x in cats if x is not None]
obj['tags'] = [cat.lower() for cat in cats]
obj['primary_type'] = primary_tag_type
self.data.append(obj)
print(json.dumps(obj, indent=2))
yield obj, (obj['name'], obj['city'], obj['tags'])
row_connector_event = ConnectorEvent.query.filter(
and_(ConnectorEvent.connector_event_id == self.ID,
ConnectorEvent.connector_type == self.TYPE)).first()
if not row_connector_event:
row_connector_event = ConnectorEvent(connector_event_id=self.ID,
connector_type=self.TYPE)
events = {self.create_key(x['name'], x['city']): x for x in self.data}
row_connector_event.data = events
db_session.merge(row_connector_event)
db_session.commit()
for key, restaurant in row_connector_event.data.items():
yield restaurant['name'], restaurant
def transform_event(self, event, skip_write=None):
ev_meta = self.get_event_metadata(event)
tags = get_from(ev_meta, [ExtractMMV.TYPE, 'tags'], [])
event.remove_all_tags()
for tag in tags:
if tag:
event.add_tag(tag, Tag.FOOD_DRINK)
yelp_tags = get_from(ev_meta, [ConnectYelp.TYPE, 'categories'], [])
for tag in yelp_tags:
tag_title = get_from(tag, ['title'])
if tag_title:
event.add_tag(tag_title, Tag.FOOD_DRINK)
accolades = []
for conn_type in self.connector_types:
if conn_type is ExtractMMV:
continue
tier = get_from(ev_meta, [conn_type.TYPE, 'tier'])
if tier: accolades.append(tier)
if accolades and len(accolades) > 0:
accolades = sorted(accolades)
else:
accolades = None
event.accolades = accolades
self.transform_event_details(event, ev_meta=ev_meta)
descriptions = [
(
x.TYPE,
get_from(ev_meta, [x.TYPE, 'url']),
get_from(ev_meta, [x.TYPE, 'description'])
) for x in self.connector_types
]
descriptions = [x for x in descriptions if x[2]]
if descriptions: event.description = descriptions
specialty_connectors = self.connector_types
specialties = [
[
x.TYPE,
get_from(ev_meta, [x.TYPE, 'order'])
] for x in specialty_connectors
]
specialties = [x for x in specialties if x[1]]
if specialties:
for i, x in enumerate(specialties):
if x[1].__class__ is not str:
specialties[i][1] = ", ".join(x[1])
event.details[Event.DETAILS_SPECIALTIES] = specialties
img_url = get_from(ev_meta, [ConnectYelp.TYPE, 'image_url'])
if img_url:
event.img_url = re.sub(r'/o.jpg', '/348s.jpg', img_url) # '/l.jpg'
backdrop_url = get_from(ev_meta, [ConnectYelp.TYPE, 'photos', 1])
if backdrop_url: event.backdrop_url = backdrop_url
latitude = get_from(event.details, [ConnectGoogle.TYPE, 'latitude'])
if latitude: event.latitude = latitude
longitude = get_from(event.details, [ConnectGoogle.TYPE, 'longitude'])
if longitude: event.longitude = longitude
address = get_from(event.details, [ConnectGoogle.TYPE, 'address'])
if address:
state_match = re.search(r'(.*), +[A-Z]{2} +[0-9]+', address)
if state_match:
event.address = ", ".join(state_match.group(1).split(",")[:-1])
else:
event.address = ""
city = get_from(event.details, [ConnectGoogle.TYPE, 'city'])
if city: event.city = city
state = get_from(event.details, [ConnectGoogle.TYPE, 'state'])
if state: event.state = state
cost = get_from(event.details, [ConnectGoogle.TYPE, 'cost'])
if not cost:
cost = get_from(event.details, [ConnectYelp.TYPE, 'cost'])
if cost:
event.cost = cost
event.add_tag("$"*event.cost, Tag.FOOD_DRINK)
urls = [
(
conn.TYPE,
get_from(event.details, [conn.TYPE, Event.DETAILS_URL])
) for conn in [ConnectGoogle, ConnectYelp] if get_from(event.details, [conn.TYPE, Event.DETAILS_URL])
]
event.urls = urls
statuses = set(
get_from(event.details, [conn.TYPE, Event.DETAILS_STATUS]) for conn in [ConnectGoogle, ConnectYelp]
)
status = Event.STATUS_OPEN
if Event.STATUS_CLOSED_PERM in statuses:
status = Event.STATUS_CLOSED_PERM
elif Event.STATUS_CLOSED_TEMP in statuses:
status = Event.STATUS_CLOSED_TEMP
event.status = status
hours = {}
# for conn in [ConnectGoogle, ConnectYelp]:
# h = get_from(event.details, [conn.TYPE, Event.DETAILS_HOURS])
# if h:
# for d, r in h.items():
# if d not in hours:
# hours[d] = r
# continue
# else:
# op = get_from(r, ["open"])
# cl = get_from(r, ["close"])
# cur = hours[d]
# cur_op = get_from(cur, ["open"])
# cur_cl = get_from(cur, ["closes"])
# if op < cur_op: hours[d]["open"] = op
# if cl > cur_op: hours[d]["close"] = cl
# event.details[Event.DETAILS_HOURS] = days
event.meta = ev_meta
if not skip_write:
db_session.merge(event)
db_session.commit()
return event
def extract(self, name=None, event_id=None, backfill=None):
connector = self.get_connector()
for key, event in self.events.items():
place_id = get_from(event, ['place_id'])
place_name = get_from(event, ['name'])
if not place_id: continue
if event_id is not None and place_id != event_id: continue
if name is not None and name != place_name: continue
if backfill and place_id in connector.data:
print("Found Place ID {} => {}".format(
place_id, connector.data[place_id]['name']))
continue
search_results = None
b_details = None
event_name = event['name']
event_addr = get_from(event, ['address'])
event_city = event['city']
event_state = get_from(event, ['state'], 'CA')
if not search_results:
kwargs = {
'name': event_name,
'address1': event_addr,
'city': event_city,
'state': event_state
}
print(" | ".join(
["{}: \"{}\"".format(k, v) for k, v in kwargs.items()]))
try:
search_results = self.api.business_match_query(
country="US", **kwargs)
except Exception as e:
print("business_match_query: {}".format(e))
if not search_results or len(search_results['businesses']) == 0:
try:
term = " ".join([event_name, event_city, event_state])
location = " ".join([event_city, event_state])
kwargs = {'term': term, 'location': location}
print(" | ".join([
"{}: \"{}\"".format(k, v) for k, v in kwargs.items()
]))
search_results = self.api.search_query(limit=1, **kwargs)
except Exception as e:
print("search_query: {}".format(e))
if search_results:
for r in search_results['businesses']:
b_details = None
try:
b_details = self.api.business_query(id=r['id'])
except Exception as e:
print("business_query: {}".format(e))
print(r['id'])
if b_details:
# row_event.update_meta(self.TYPE, {**r, **b_details})
# db_session.merge(row_event)
# db_session.commit()
connector.data[place_id] = b_details
db_session.merge(connector)
db_session.commit()
#
yield b_details['name'], b_details
else:
print("Unable to find {}".format(place_id))
def extract(self):
connector = self.get_connector()
i = 0
for key, res in self.data.items():
res_key = self.create_key(res['name'], res['city'])
connector_res = get_from(connector.data, [res_key], {})
res.update(connector_res)
place_id = get_from(res, ['place_id'])
if place_id:
print("Found {} | {} => {}".format(i, place_id, res_key))
if res_key not in connector.data or res != connector.data[
res_key]:
"\tUpdate"
connector.data[res_key] = res
db_session.merge(connector)
db_session.commit()
pass
else:
place_q = ", ".join([
res['name'] or "",
get_from(res, ['address'], ""), res['city'] or ""
])
print("!!!GEOCODE!!! {} | {} => {}".format(
i, res_key, place_q))
try:
results = self.client.find_place(
input=place_q,
input_type='textquery',
fields=["place_id", "name", "formatted_address"])
res_data = get_from(results, ['candidates', 0], {})
res_addr = get_from(res_data, ['formatted_address'])
if 'formatted_address' in res_data:
del res_data["formatted_address"]
if res_addr: res_data["address"] = res_addr
res.update(res_data)
connector.data[res_key] = res
db_session.merge(connector)
db_session.commit()
except Exception as e:
print("\t!!!ERROR!!! => {}".format(e))
print("\t{}".format(res))
i += 1
i = 0
for key, res in self.data.items():
place_id = get_from(res, ['place_id'])
if not place_id: continue
ev = Event.query.filter(
or_(Event.name == res['name'],
Event.alias == place_id)).first()
if ev:
print("Found Event({}) => {}".format(place_id, res['name']))
else:
print("Create Event({}) => {}".format(place_id, res['name']))
print("\t{}".format(res))
ev = Event(name=res['name'], alias=res['place_id'])
# print(i, json.dumps(res, indent=2))
i += 1
ev.primary_type = Tag.FOOD_DRINK
ev.address = get_from(res, ['address'])
ev.city = res['city']
ev.state = get_from(res, ['state'])
if not ev.meta: ev.meta = {}
ev.meta[self.TYPE] = res
db_session.merge(ev)
db_session.commit()
def extract(self, name=None, event_id=None, backfill=None, index=None):
connector = self.get_connector()
detail_fields = [
# Basic
"address_component",
"adr_address",
"business_status",
"formatted_address",
"geometry",
"icon",
"name",
"photo",
"place_id",
"plus_code",
"type",
"url",
"utc_offset",
"vicinity",
# Contact
"formatted_phone_number",
"international_phone_number",
"opening_hours",
"website",
#Atmosphere
"price_level",
"rating",
"review",
"user_ratings_total"
]
i = 0
for key, event in self.events.items():
i += 1
if index and i <= int(index):
print("Skip {}".format(i))
continue
place_id = get_from(event, ['place_id'])
place_name = get_from(event, ['name'])
if not place_id: continue
if event_id is not None and place_id != event_id: continue
if name is not None and name != place_name: continue
if backfill and place_id in connector.data:
print("Found Place ID {} => {} | {}".format(
place_id, key, connector.data[place_id]['name']))
continue
results = None
try:
print("Getting details for ({})".format(place_id))
results = self.client.place(place_id, fields=detail_fields)
except Exception as e:
print("place: {}".format(e))
results = get_from(results, ["result"])
if results:
# print(json.dumps(connector.data, indent=2))
print("**********")
# print(json.dumps(results, indent=2))
connector.data[place_id] = results
db_session.merge(connector)
db_session.commit()
yield results['name'], (results['place_id'], results['name'])