def run(dryrun=False):
logger.info("MediaOcean download started.")
try:
download(dryrun)
except Exception as ex:
logger.exception(ex, "MediaOcean download failed.")
logger.info("MediaOcean download finished.")
def denormalize(week_range):
try:
logger.info('%s fill started.' % TABLE_NAME)
sql = SQL()
sql.execute_batches(FILL_SQL, week_range)
sql.commitclose()
logger.info('%s fill finished.' % TABLE_NAME)
except Exception as ex:
logger.exception(ex, 'Critical error in denormalization: %s' % TABLE_NAME)
def download(days = 1):
logger.info('Crimson Hexagon download started.')
dat = {}
dat['is_historical'] = (days > 1)
started = time.time()
is_finished = False
while time.time() - started < 600:
try:
dat['monitors'] = monitors()
logger.info('Crimson Hexagon monitor results...')
dat['monitor_results'] = monitor_results(days)
dat['monitor_results_bycity'] = monitor_results_by_city(days)
dat['monitor_results_bystate'] = monitor_results_by_state(days)
logger.info('Crimson Hexagon facebook results...')
dat['facebook_admin_posts'] = facebook_admin_posts(days)
dat['facebook_page_likes'] = facebook_page_likes(days)
dat['facebook_total_activity'] = facebook_total_activity(days)
logger.info('Crimson Hexagon twitter results...')
dat['twitter_engagement_metrics'] = twitter_engagement_metrics(days)
dat['twitter_followers'] = twitter_followers(days)
dat['twitter_sent_posts'] = twitter_sent_posts(days)
dat['twitter_total_engagement'] = twitter_total_engagement(days)
logger.info('Crimson Hexagon instagram results...')
dat['instagram_followers'] = instagram_followers(days)
dat['instagram_total_activity'] = instagram_total_activity(days)
dat['instagram_sent_media'] = instagram_sent_media(days)
is_finished = True
break
except Exception as ex:
logger.exception(ex, 'Crimson Hexagon API failure, retrying in 10s...')
finally:
time.sleep(10)
if is_finished:
savejson(dat)
else:
logger.warn('Crimson Hexagon download failed!')
logger.info('Crimson Hexagon download finished.')