def get_and_store_fit_data(http_auth, cur, username, past_n_days=30): fit_service = build('fitness', 'v1', http=http_auth) n_days_ago_millis = backend.calc_n_days_ago(past_n_days) # print(help(fit_service.users().dataset().aggregate)) steps = [] activity = [] now_millis = backend.current_milli_time() print('get and store fitness data for user {}'.format(username)) try: stepsData = get_aggregate(fit_service, n_days_ago_millis, now_millis, backend.STEPS_DATASOURCE) for day in stepsData['bucket']: # store local date in the database d = datetime.fromtimestamp(int(day['startTimeMillis']) / 1000, tz=pytz.timezone( backend.DEFAULT_TIMEZONE)).strftime( backend.DATE_FORMAT) if day['dataset'][0]['point']: s = day['dataset'][0]['point'][0]['value'][0]['intVal'] steps.append([d, s]) else: steps.append([d, 0]) print("Steps:", steps) except Exception as e: print(e) print("No steps found") try: activityData = get_aggregate(fit_service, n_days_ago_millis, now_millis, backend.ACTIVITY_DATASOURCE) for day in activityData['bucket']: # store local date in the database d = datetime.fromtimestamp(int(day['startTimeMillis']) / 1000, tz=pytz.timezone( backend.DEFAULT_TIMEZONE)).strftime( backend.DATE_FORMAT) if day['dataset'][0]['point']: for a in day['dataset'][0]['point']: activity_type = a['value'][0]['intVal'] length_ms = a['value'][1]['intVal'] n_segments = a['value'][2]['intVal'] activity.append([d, activity_type, length_ms, n_segments]) else: activity.append([d, 4, 0, 0]) print("Activity:", activity) except Exception as e: print(e) print("No activity found") try: rows = cur.executemany( "REPLACE INTO steps SET username=%s, day=%s, steps=%s".format( username), [[username] + s for s in steps]) print("steps: {} rows affected".format(rows)) rows = cur.executemany( "REPLACE INTO activity SET username=%s, day=%s, activity_type=%s, length_ms=%s, n_segments=%s", [[username] + a for a in activity]) print("activity: {} rows affected".format(rows)) except Exception as e: print(e) return steps, activity
def insert_heart_rate(username): error = check_headers_apikey() if error: return error http_auth, timezone = get_google_http_auth_n_user_timezone(username) end_time_millis, start_date, error = extract_header_dates() if error: if isinstance(error, HTTPError): return error else: return HTTPResponse( { 'code': httplib.BAD_REQUEST, 'error': str(error) }, httplib.BAD_REQUEST) else: try: # end_time_millis in form data is optional if end_time_millis is None: end_time_millis = backend.current_milli_time() result = backend.get_and_insert_heart_rate(http_auth, username, start_date['year'], start_date['month'], start_date['day'], end_time_millis, local_timezone=timezone) response.content_type = 'application/json' return result except client.HttpAccessTokenRefreshError as err: return HTTPError(httplib.UNAUTHORIZED, "Refresh token invalid: " + str(err)) except googleapiclient.errors.HttpError as err: return HTTPError(err.resp.status, "Google API HttpError: " + str(err))
def insert_daily_fitness_data_thread(bucket_name, retry, username): error_reporting_client = error_reporting.Client() http_context = error_reporting.HTTPContext( method='GET', url='/v1/insert_daily_fitness', user_agent='cron job for user {}'.format(username)) storage_client = storage.Client() bucket = storage_client.get_bucket(bucket_name) http_auth, timezone = get_google_http_auth_n_user_timezone(username) # get today's local date - 1 day yesterday_local = datetime.now(pytz.timezone(timezone)) - timedelta(days=1) yesterday_local_str = yesterday_local.strftime(backend.DATE_FORMAT) df = backend.UserDataFlow(username, http_auth, yesterday_local.year, yesterday_local.month, yesterday_local.day, backend.current_milli_time(), timezone) retry[username] = {} categories = {'heartrate', 'activities', 'steps', 'calories'} for category in categories: retry[username][category] = {} # countdown is the number of retries retry[username][category]['countdown'] = 1 gs_path_get = '{}/{}/{}.json'.format(username, yesterday_local_str, category) gs_path_insert = '{}/{}/{}_inserted_count.json'.format( username, yesterday_local_str, category) get_result = None insert_result = None # start of the retry logic while retry[username][category]['countdown'] >= 0: try: if category == 'heartrate': # get and insert heart rate data insert_result = df.get_and_post_heart_rate() get_result = insert_result['heart_datasets'] elif category == 'activities': # get and insert activities data get_result = df.get_activities() insert_result = df.post_activities() elif category == 'steps': # get and insert step counts get_result = df.get_steps() insert_result = df.post_steps() elif category == 'calories': # get and insert calories get_result = df.get_calories() insert_result = df.post_calories() # set to None upon success of getting API data and inserting to BigQuery retry[username][category]['countdown'] = None except client.HttpAccessTokenRefreshError as err: http_context.responseStatusCode = httplib.UNAUTHORIZED user_token_err = '{} has invalid refresh token'.format( username) error_reporting_client.report_exception( http_context=http_context, user=user_token_err) retry[username][category]['error'] = "{}: {}".format( user_token_err, err) # can't recover; abandon retry retry[username][category]['countdown'] = -2 except googleapiclient.errors.HttpError as err: http_context.responseStatusCode = err.resp.status error_reporting_client.report_exception( http_context=http_context, user='******'.format(username)) retry[username][category]['error'] = str(err) if err.resp.status in (httplib.BAD_REQUEST, httplib.UNAUTHORIZED, httplib.NOT_FOUND, httplib.FORBIDDEN): # can't recover; abandon retry retry[username][category]['countdown'] = -2 except Exception as err: # https://googleapis.github.io/google-cloud-python/latest/error-reporting/usage.html error_reporting_client.report_exception( http_context=http_context, user='******'.format( category, username)) retry[username][category]['error'] = str(err) # if retry for user on category isn't None, recoverable failure happened, decrement the retry count if retry[username][category]['countdown'] is not None: retry[username][category]['countdown'] -= 1 else: # exiting while loop because None >= 0 is False pass # per category, putting the get, insert results on Cloud Storage upon success if retry[username][category]['countdown'] is None: retry[username][category]['gs://'] = [] blob_get_result = bucket.blob(gs_path_get) blob_get_result.upload_from_string(json.dumps(get_result)) retry[username][category]['gs://'].append("{}/{}".format( bucket_name, gs_path_get)) blob_insert_result = bucket.blob(gs_path_insert) blob_insert_result.upload_from_string(json.dumps(insert_result)) retry[username][category]['gs://'].append("{}/{}".format( bucket_name, gs_path_insert)) retry[username][category].pop('countdown')