def object_from_dictionary(cls, entry):
new_media = Media(id=entry['id'])
new_media.user = User.object_from_dictionary(entry['user'])
new_media.images = {}
for version,version_info in entry['images'].iteritems():
new_media.images[version] = Image.object_from_dictionary(version_info)
if 'user_has_liked' in entry:
new_media.user_has_liked = entry['user_has_liked']
new_media.like_count = entry['likes']['count']
new_media.comment_count = entry['comments']['count']
new_media.comments = []
for comment in entry['comments']['data']:
new_media.comments.append(Comment.object_from_dictionary(comment))
new_media.created_time = timestamp_to_datetime(entry['created_time'])
if entry['location']:
new_media.location = Location.object_from_dictionary(entry['location'])
new_media.link = entry['link']
return new_media
def object_from_dictionary(cls, entry):
new_media = Media(id=entry["id"])
new_media.user = User.object_from_dictionary(entry["user"])
new_media.images = {}
for version, version_info in entry["images"].iteritems():
new_media.images[version] = Image.object_from_dictionary(version_info)
if "user_has_liked" in entry:
new_media.user_has_liked = entry["user_has_liked"]
new_media.like_count = entry["likes"]["count"]
new_media.comment_count = entry["comments"]["count"]
new_media.comments = []
for comment in entry["comments"]["data"]:
new_media.comments.append(Comment.object_from_dictionary(comment))
new_media.created_time = timestamp_to_datetime(entry["created_time"])
try:
new_media.location = Location.object_from_dictionary(entry["location"])
except KeyError:
new_media.location = None
try:
new_media.caption = entry["caption"]["text"]
except (KeyError, TypeError):
new_media.caption = None
new_media.link = entry["link"]
return new_media
def object_from_dictionary(cls, entry):
user = User.object_from_dictionary(entry['from'])
user.dict = entry
text = entry['text']
created_at = timestamp_to_datetime(entry['created_time'])
id = entry['id']
return Comment(id=id, user=user, text=text, created_at=created_at)
def object_from_dictionary(cls, entry):
new_media = Media(id=entry['id'])
new_media.user = User.object_from_dictionary(entry['user'])
new_media.images = {}
for version, version_info in entry['images'].iteritems():
new_media.images[version] = Image.object_from_dictionary(
version_info)
if 'user_has_liked' in entry:
new_media.user_has_liked = entry['user_has_liked']
new_media.like_count = entry['likes']['count']
new_media.comment_count = entry['comments']['count']
new_media.comments = []
for comment in entry['comments']['data']:
new_media.comments.append(Comment.object_from_dictionary(comment))
new_media.created_time = timestamp_to_datetime(entry['created_time'])
if entry['location']:
new_media.location = Location.object_from_dictionary(
entry['location'])
new_media.link = entry['link']
return new_media
def object_from_dictionary(cls, entry):
new_media = Media(id=entry['id'])
new_media.user = User.object_from_dictionary(entry['user'])
new_media.images = {}
for version, version_info in entry['images'].iteritems():
new_media.images[version] = Image.object_from_dictionary(
version_info)
if 'videos' in entry:
new_media.videos = {}
for version, version_info in entry['videos'].iteritems():
new_media.videos[version] = Video.object_from_dictionary(
version_info)
if 'user_has_liked' in entry:
new_media.user_has_liked = entry['user_has_liked']
new_media.like_count = entry['likes']['count']
new_media.likes = []
if 'data' in entry['likes']:
for like in entry['likes']['data']:
new_media.likes.append(User.object_from_dictionary(like))
new_media.comment_count = entry['comments']['count']
new_media.comments = []
for comment in entry['comments']['data']:
new_media.comments.append(Comment.object_from_dictionary(comment))
new_media.created_time = timestamp_to_datetime(entry['created_time'])
if entry['location'] and 'id' in entry:
new_media.location = Location.object_from_dictionary(
entry['location'])
new_media.caption = None
if entry['caption']:
new_media.caption = Comment.object_from_dictionary(
entry['caption'])
if entry['tags']:
new_media.tags = []
for tag in entry['tags']:
new_media.tags.append(Tag.object_from_dictionary({'name':
tag}))
if entry.get('users_in_photo'):
new_media.users_in_photo = []
for user in entry['users_in_photo']:
new_media.users_in_photo.append(
UserInPhoto(
User.object_from_dictionary(user['user']),
Position.object_from_dictionary(user['position'])))
new_media.link = entry['link']
new_media.filter = entry.get('filter')
return new_media
def object_from_dictionary(cls, entry):
new_media = Media(id=entry['id'])
new_media._api_dict = entry
new_media.type = entry['type']
new_media.user = User.object_from_dictionary(entry['user'])
new_media.images = {}
#2017-02-23: http://stackoverflow.com/questions/42456908/instagram-api-missing-elements-in-json-response
for version, version_info in entry.get('images',{}).iteritems():
new_media.images[version] = Image.object_from_dictionary(version_info)
if new_media.type == 'video':
new_media.videos = {}
#2017-02-23: http://stackoverflow.com/questions/42456908/instagram-api-missing-elements-in-json-response
for version, version_info in entry.get('videos',{}).iteritems():
new_media.videos[version] = Video.object_from_dictionary(version_info)
if 'user_has_liked' in entry:
new_media.user_has_liked = entry['user_has_liked']
new_media.like_count = entry['likes']['count']
new_media.likes = []
if 'data' in entry['likes']:
for like in entry['likes']['data']:
new_media.likes.append(User.object_from_dictionary(like))
new_media.comment_count = entry['comments']['count']
new_media.comments = []
#HOTFIX
#for comment in entry['comments']['data']:
for comment in entry['comments'].get('data',[]):
new_media.comments.append(Comment.object_from_dictionary(comment))
new_media.created_time = timestamp_to_datetime(entry['created_time'])
new_media.users_in_photo = []
if entry.get('users_in_photo'):
for user_in_photo in entry['users_in_photo']:
new_media.users_in_photo.append(UserInPhoto.object_from_dictionary(user_in_photo))
if entry['location'] and 'id' in entry:
new_media.location = Location.object_from_dictionary(entry['location'])
new_media.caption = None
if entry['caption']:
new_media.caption = Comment.object_from_dictionary(entry['caption'])
if entry['tags']:
new_media.tags = []
for tag in entry['tags']:
new_media.tags.append(Tag.object_from_dictionary({'name': tag}))
new_media.link = entry['link']
new_media.filter = entry.get('filter')
return new_media
def object_from_dictionary(cls, entry):
#user = User.object_from_dictionary(entry['from'])
user = None
if entry['from']:
user = User.object_from_dictionary(entry['from'])
user._dict = entry
text = entry['text']
created_at = timestamp_to_datetime(entry['created_time'])
id = entry['id']
return Comment(id=id, user=user, text=text, created_at=created_at)
def object_from_dictionary(cls, entry):
user = entry['from']
text = entry['text']
created_at = timestamp_to_datetime(entry['created_time'])
id = entry['id']
return {
'id' : id,
'user' : user,
'text' : text,
'created_at': created_at,
}
def object_from_dictionary(cls, entry):
new_media = Media(id=entry['id'])
new_media.type = entry['type']
new_media.user = User.object_from_dictionary(entry['user'])
new_media.images = {}
for version, version_info in entry['images'].iteritems():
new_media.images[version] = Image.object_from_dictionary(version_info)
if new_media.type == 'video':
new_media.videos = {}
for version, version_info in entry['videos'].iteritems():
new_media.videos[version] = Video.object_from_dictionary(version_info)
if 'user_has_liked' in entry:
new_media.user_has_liked = entry['user_has_liked']
new_media.like_count = entry['likes']['count']
new_media.likes = []
if 'data' in entry['likes']:
for like in entry['likes']['data']:
new_media.likes.append(User.object_from_dictionary(like))
new_media.comment_count = entry['comments']['count']
new_media.comments = []
for comment in entry['comments']['data']:
new_media.comments.append(Comment.object_from_dictionary(comment))
new_media.created_time = timestamp_to_datetime(entry['created_time'])
new_media.timestamp = int(entry['created_time'])
if entry['location'] and 'id' in entry:
new_media.location = Location.object_from_dictionary(entry['location'])
new_media.caption = None
if entry['caption']:
new_media.caption = Comment.object_from_dictionary(entry['caption'])
if entry['tags']:
new_media.tags = []
for tag in entry['tags']:
new_media.tags.append(Tag.object_from_dictionary({'name': tag}))
new_media.link = entry['link']
new_media.filter = entry.get('filter')
new_media.data = entry
if 'scopit_location_id' in entry:
new_media.scopit_location_id = entry['scopit_location_id']
return new_media
def process_data(data, db):
counter = 0
tt_date_time = helper.timestamp_to_datetime(data['updateTime'])
# parse out only routes
routes = data['routes']
# run through the routes for data
for route in routes:
counter += 1
route_id = route['id']
route_name = route['name']
route_from = route['fromName']
route_to = route['toName']
route_type = route['type']
length = route['length']
current_tt = route['time']
historical_tt = route['historicTime']
jam_level = route['jamLevel']
current_tt_min = helper.time_to_minutes(current_tt)
historical_tt_min = helper.time_to_minutes(historical_tt)
congested_bool = helper.check_congestion(current_tt, historical_tt,
CONGESTED_PERCENT)
congestion_table(congested_bool, route_id, tt_date_time,
current_tt_min, historical_tt_min, db)
route_details = (route_id, route_name, route_from, route_to,
route_type, length)
travel_time = (route_id, current_tt, historical_tt, current_tt_min,
historical_tt_min, congested_bool, CONGESTED_PERCENT,
jam_level, tt_date_time)
# write data
try:
write_routes(route_details, db)
write_data(travel_time, db)
except Exception as e:
logging.exception(e)
logging.info(f"Route counter: {counter}")
def object_from_dictionary(cls, entry):
new_media = Media(id=entry['id'])
new_media.user = User.object_from_dictionary(entry['user'])
new_media.images = {}
for version,version_info in entry['images'].iteritems():
new_media.images[version] = Image.object_from_dictionary(version_info)
if 'user_has_liked' in entry:
new_media.user_has_liked = entry['user_has_liked']
new_media.like_count = entry['likes']['count']
new_media.likes = []
if entry['likes'].has_key('data'):
for like in entry['likes']['data']:
new_media.likes.append(User.object_from_dictionary(like))
new_media.comment_count = entry['comments']['count']
new_media.comments = []
for comment in entry['comments']['data']:
new_media.comments.append(Comment.object_from_dictionary(comment))
new_media.caption = None
if entry['caption']:
new_media.caption = Caption.object_from_dictionary(entry['caption'])
new_media.created_time = timestamp_to_datetime(entry['created_time'])
if entry['location'] and entry.has_key('id'):
new_media.location = Location.object_from_dictionary(entry['location'])
new_media.caption = None
if entry['caption']:
new_media.caption = Comment.object_from_dictionary(entry['caption'])
new_media.tags = entry['tags']
new_media.link = entry['link']
new_media.filter = entry.get('filter')
return new_media
def object_from_dictionary(cls, entry):
new_media = Media(id=entry["id"])
new_media.user = User.object_from_dictionary(entry["user"])
new_media.images = {}
for version, version_info in entry["images"].iteritems():
new_media.images[version] = Image.object_from_dictionary(version_info)
if "user_has_liked" in entry:
new_media.user_has_liked = entry["user_has_liked"]
new_media.like_count = entry["likes"]["count"]
new_media.likes = []
if "data" in entry["likes"]:
for like in entry["likes"]["data"]:
new_media.likes.append(User.object_from_dictionary(like))
new_media.comment_count = entry["comments"]["count"]
new_media.comments = []
for comment in entry["comments"]["data"]:
new_media.comments.append(Comment.object_from_dictionary(comment))
new_media.created_time = timestamp_to_datetime(entry["created_time"])
if entry["location"] and "id" in entry:
new_media.location = Location.object_from_dictionary(entry["location"])
new_media.caption = None
if entry["caption"]:
new_media.caption = Comment.object_from_dictionary(entry["caption"])
new_media.tags = []
if entry["tags"]:
for tag in entry["tags"]:
new_media.tags.append(Tag.object_from_dictionary({"name": tag}))
new_media.link = entry["link"]
new_media.filter = entry.get("filter")
return new_media
def run(url, uid, db):
# get data from website
data = download_data.get_data_from_website(url)
timestamp = int(data['updateTime'])
tt_date_time = helper.timestamp_to_datetime(timestamp)
logging.info(f"Waze feed Epoch Time: {timestamp}")
logging.info(f'Waze feed Date/Time : {tt_date_time}')
# check to make sure data is good before proceeding
try:
helper.check_for_data_integrity(data)
logging.debug('data pass integrity check')
except Exception as e:
logging.exception(e)
raise
# check for existing pull #
persistence.check_update_time(uid, timestamp)
# run main
process_data(uid, data, db)
def process_data(uid, data, db):
counter = 0
tt_date_time = helper.timestamp_to_datetime(data['updateTime'])
# parse out only routes
routes = data['routes']
# get the feed name
feed_name = data['name']
# get current route errors
err_list = route_errors.get_route_errors()
# run through the routes for data
for route in routes:
counter += 1
route_id = route['id']
route_name = route['name']
route_from = route['fromName']
route_to = route['toName']
route_type = route['type']
length = route['length']
current_tt = route['time']
historical_tt = route['historicTime']
jam_level = route['jamLevel']
route_list.append(route_id)
if route_id in skip_routes:
logging.warning(
f'Route {route_id} is set to skip, will not process or log.')
continue
if current_tt == -1:
logging.warning(
f'Route {route_id} is showing -1, skipping for now')
route_errors.set_route_errors(route_id, route_name, add=True)
continue # move to next route, do not archive
elif route_id in err_list:
route_errors.set_route_errors(route_id, route_name, add=False)
omit = False
if route_id in omit_routes or uid in omit_feeds:
omit = True
current_tt_min = helper.time_to_minutes(current_tt)
historical_tt_min = helper.time_to_minutes(historical_tt)
congested_bool = helper.check_congestion(current_tt, historical_tt,
CONGESTED_PERCENT)
congestion_table(congested_bool, route_id, tt_date_time,
current_tt_min, historical_tt_min, omit, db)
route_details = (route_id, route_name, route_from, route_to,
route_type, length, uid, feed_name, False)
travel_time = (route_id, current_tt, historical_tt, current_tt_min,
historical_tt_min, congested_bool, CONGESTED_PERCENT,
jam_level, tt_date_time)
# write data
try:
write_routes(route_details, db)
write_data(travel_time, db)
except Exception as e:
logging.exception(e)
logging.info(f"Route counter: {counter}")
def object_from_dictionary(cls, entry):
user = User.object_from_dictionary(entry["from"])
text = entry["text"]
created_at = timestamp_to_datetime(entry["created_time"])
id = entry["id"]
return Comment(id=id, user=user, text=text, created_at=created_at)
def object_from_dictionary(cls, entry):
user = User.object_from_dictionary(entry['from'])
text = entry['text']
created_at = timestamp_to_datetime(entry['created_time'])
id = entry['id']
return cls(id=id, user=user, text=text, created_at=created_at)
