def get_and_store_fit_data(http_auth, cur, username, past_n_days=30):
fit_service = build('fitness', 'v1', http=http_auth)
n_days_ago_millis = backend.calc_n_days_ago(past_n_days)
# print(help(fit_service.users().dataset().aggregate))
steps = []
activity = []
now_millis = backend.current_milli_time()
print('get and store fitness data for user {}'.format(username))
try:
stepsData = get_aggregate(fit_service, n_days_ago_millis, now_millis,
backend.STEPS_DATASOURCE)
for day in stepsData['bucket']:
# store local date in the database
d = datetime.fromtimestamp(int(day['startTimeMillis']) / 1000,
tz=pytz.timezone(
backend.DEFAULT_TIMEZONE)).strftime(
backend.DATE_FORMAT)
if day['dataset'][0]['point']:
s = day['dataset'][0]['point'][0]['value'][0]['intVal']
steps.append([d, s])
else:
steps.append([d, 0])
print("Steps:", steps)
except Exception as e:
print(e)
print("No steps found")
try:
activityData = get_aggregate(fit_service, n_days_ago_millis,
now_millis, backend.ACTIVITY_DATASOURCE)
for day in activityData['bucket']:
# store local date in the database
d = datetime.fromtimestamp(int(day['startTimeMillis']) / 1000,
tz=pytz.timezone(
backend.DEFAULT_TIMEZONE)).strftime(
backend.DATE_FORMAT)
if day['dataset'][0]['point']:
for a in day['dataset'][0]['point']:
activity_type = a['value'][0]['intVal']
length_ms = a['value'][1]['intVal']
n_segments = a['value'][2]['intVal']
activity.append([d, activity_type, length_ms, n_segments])
else:
activity.append([d, 4, 0, 0])
print("Activity:", activity)
except Exception as e:
print(e)
print("No activity found")
try:
rows = cur.executemany(
"REPLACE INTO steps SET username=%s, day=%s, steps=%s".format(
username), [[username] + s for s in steps])
print("steps: {} rows affected".format(rows))
rows = cur.executemany(
"REPLACE INTO activity SET username=%s, day=%s, activity_type=%s, length_ms=%s, n_segments=%s",
[[username] + a for a in activity])
print("activity: {} rows affected".format(rows))
except Exception as e:
print(e)
return steps, activity
def insert_heart_rate(username):
error = check_headers_apikey()
if error:
return error
http_auth, timezone = get_google_http_auth_n_user_timezone(username)
end_time_millis, start_date, error = extract_header_dates()
if error:
if isinstance(error, HTTPError):
return error
else:
return HTTPResponse(
{
'code': httplib.BAD_REQUEST,
'error': str(error)
}, httplib.BAD_REQUEST)
else:
try:
# end_time_millis in form data is optional
if end_time_millis is None:
end_time_millis = backend.current_milli_time()
result = backend.get_and_insert_heart_rate(http_auth,
username,
start_date['year'],
start_date['month'],
start_date['day'],
end_time_millis,
local_timezone=timezone)
response.content_type = 'application/json'
return result
except client.HttpAccessTokenRefreshError as err:
return HTTPError(httplib.UNAUTHORIZED,
"Refresh token invalid: " + str(err))
except googleapiclient.errors.HttpError as err:
return HTTPError(err.resp.status,
"Google API HttpError: " + str(err))
def insert_daily_fitness_data_thread(bucket_name, retry, username):
error_reporting_client = error_reporting.Client()
http_context = error_reporting.HTTPContext(
method='GET',
url='/v1/insert_daily_fitness',
user_agent='cron job for user {}'.format(username))
storage_client = storage.Client()
bucket = storage_client.get_bucket(bucket_name)
http_auth, timezone = get_google_http_auth_n_user_timezone(username)
# get today's local date - 1 day
yesterday_local = datetime.now(pytz.timezone(timezone)) - timedelta(days=1)
yesterday_local_str = yesterday_local.strftime(backend.DATE_FORMAT)
df = backend.UserDataFlow(username, http_auth, yesterday_local.year,
yesterday_local.month, yesterday_local.day,
backend.current_milli_time(), timezone)
retry[username] = {}
categories = {'heartrate', 'activities', 'steps', 'calories'}
for category in categories:
retry[username][category] = {}
# countdown is the number of retries
retry[username][category]['countdown'] = 1
gs_path_get = '{}/{}/{}.json'.format(username, yesterday_local_str,
category)
gs_path_insert = '{}/{}/{}_inserted_count.json'.format(
username, yesterday_local_str, category)
get_result = None
insert_result = None
# start of the retry logic
while retry[username][category]['countdown'] >= 0:
try:
if category == 'heartrate':
# get and insert heart rate data
insert_result = df.get_and_post_heart_rate()
get_result = insert_result['heart_datasets']
elif category == 'activities':
# get and insert activities data
get_result = df.get_activities()
insert_result = df.post_activities()
elif category == 'steps':
# get and insert step counts
get_result = df.get_steps()
insert_result = df.post_steps()
elif category == 'calories':
# get and insert calories
get_result = df.get_calories()
insert_result = df.post_calories()
# set to None upon success of getting API data and inserting to BigQuery
retry[username][category]['countdown'] = None
except client.HttpAccessTokenRefreshError as err:
http_context.responseStatusCode = httplib.UNAUTHORIZED
user_token_err = '{} has invalid refresh token'.format(
username)
error_reporting_client.report_exception(
http_context=http_context, user=user_token_err)
retry[username][category]['error'] = "{}: {}".format(
user_token_err, err)
# can't recover; abandon retry
retry[username][category]['countdown'] = -2
except googleapiclient.errors.HttpError as err:
http_context.responseStatusCode = err.resp.status
error_reporting_client.report_exception(
http_context=http_context,
user='******'.format(username))
retry[username][category]['error'] = str(err)
if err.resp.status in (httplib.BAD_REQUEST,
httplib.UNAUTHORIZED, httplib.NOT_FOUND,
httplib.FORBIDDEN):
# can't recover; abandon retry
retry[username][category]['countdown'] = -2
except Exception as err:
# https://googleapis.github.io/google-cloud-python/latest/error-reporting/usage.html
error_reporting_client.report_exception(
http_context=http_context,
user='******'.format(
category, username))
retry[username][category]['error'] = str(err)
# if retry for user on category isn't None, recoverable failure happened, decrement the retry count
if retry[username][category]['countdown'] is not None:
retry[username][category]['countdown'] -= 1
else:
# exiting while loop because None >= 0 is False
pass
# per category, putting the get, insert results on Cloud Storage upon success
if retry[username][category]['countdown'] is None:
retry[username][category]['gs://'] = []
blob_get_result = bucket.blob(gs_path_get)
blob_get_result.upload_from_string(json.dumps(get_result))
retry[username][category]['gs://'].append("{}/{}".format(
bucket_name, gs_path_get))
blob_insert_result = bucket.blob(gs_path_insert)
blob_insert_result.upload_from_string(json.dumps(insert_result))
retry[username][category]['gs://'].append("{}/{}".format(
bucket_name, gs_path_insert))
retry[username][category].pop('countdown')