def make_request(client, job_id, end_hour): """Make a request to BigQuery to build crash stats.""" table_id = ( 'crash_stats$%s' % crash_stats.get_datetime(end_hour).strftime('%Y%m%d')) sql = SQL.format( end_hour=end_hour, end_date=(crash_stats.get_datetime(end_hour).strftime('%Y-%m-%d'))) logging.info('TableID: %s\nJobID: %s\nSQL: %s', table_id, job_id, sql) client.insert_from_query( dataset_id='main', table_id=table_id, job_id=job_id, query=sql)
def build_if_needed(): """Get the next end hour and decide whether to execute build(). If build() succeeds, then record the next end hour.""" try: end_hour = get_next_end_hour() build(end_hour) job_history = data_types.BuildCrashStatsJobHistory() job_history.end_time_in_hours = end_hour job_history.put() logging.info('CrashStatistics for end_hour=%s is built successfully', crash_stats.get_datetime(end_hour)) return end_hour except TooEarlyException: logging.info("Skip building crash stats because it's too early.")
def get_next_end_hour(): """Get the next end hour. If it's too early to compute data for the next end hour, return None.""" last_successful_hour = get_last_successful_hour_or_start_hour() if not last_successful_hour: # No crashes seen, too early to start building stats. raise TooEarlyException() next_end_hour = last_successful_hour + 1 next_datetime = crash_stats.get_datetime(next_end_hour) if (utils.utcnow() - next_datetime) <= BIGQUERY_INSERTION_DELAY: raise TooEarlyException() return next_end_hour
def build(end_hour): """Build crash stats for the end hour.""" logging.info('Started building crash stats for %s.', crash_stats.get_datetime(end_hour)) job_id = JOB_ID_TEMPLATE.format(unique_number=int(time.time())) client = big_query.Client() make_request(client, job_id, end_hour) start_time = time.time() while (time.time() - start_time) < TIMEOUT: time.sleep(10) result = client.get_job(job_id) logging.info('Checking %s', json.dumps(result)) if result['status']['state'] == 'DONE': if result['status'].get('errors'): raise Exception(json.dumps(result)) return raise Exception('Building crash stats exceeded %d seconds.' % TIMEOUT)