def run_api_poller():
cfg_path = try_getenv('CONFIG_PATH')
db_host = try_getenv('DB_HOST')
db_port = try_getenv('DB_PORT')
db_user = try_getenv('DB_USER')
db_password = try_getenv('DB_PASSWORD')
db_name = try_getenv('DB_NAME')
redirect_url = try_getenv('CALLBACK_URL')
units = try_getenv('UNITS', 'it_IT')
# These are required vars, that we first try to load from file
client_id = try_load_var(cfg_path, 'client_id')
client_secret = try_load_var(cfg_path, 'client_secret')
access_token = try_load_var(cfg_path, 'access_token')
refresh_token = try_load_var(cfg_path, 'refresh_token')
expires_at = try_load_var(cfg_path, 'expires_at')
# If any of the required vars is not in file, try to read from env
# If read, save
if not client_id:
client_id = try_getenv('CLIENT_ID')
save_var(cfg_path, 'client_id', client_id)
if not client_secret:
client_secret = try_getenv('CLIENT_SECRET')
save_var(cfg_path, 'client_secret', client_secret)
if not access_token:
access_token = try_getenv('ACCESS_TOKEN')
save_var(cfg_path, 'access_token', access_token)
if not refresh_token:
refresh_token = try_getenv('REFRESH_TOKEN')
save_var(cfg_path, 'refresh_token', refresh_token)
if not expires_at:
expires_at = try_cast_to_int(try_getenv('EXPIRES_AT'))
save_var(cfg_path, 'expires_at', str(expires_at))
logger.debug(
"client_id: %s, client_secret: %s, access_token: %s, refresh_token: %s, expires_at: %s",
client_id, client_secret, access_token, refresh_token, expires_at)
if not client_id:
logging.critical("client_id missing, aborting!")
sys.exit(1)
if not client_secret:
logging.critical("client_secret missing, aborting!")
sys.exit(1)
if not access_token:
logging.critical("access_token missing, aborting!")
sys.exit(1)
if not refresh_token:
logging.critical("refresh_token missing, aborting!")
sys.exit(1)
api_client = Fitbit(client_id=client_id,
client_secret=client_secret,
access_token=access_token,
refresh_token=refresh_token,
redirect_uri=redirect_url,
refresh_cb=partial(write_updated_credentials,
cfg_path),
system=Fitbit.METRIC)
user_profile = None
while True:
try:
user_profile = api_client.user_profile_get()
break
except Timeout as ex:
logger.warning('Request timed out, retrying in 15 seconds...')
time.sleep(15)
except HTTPServerError as ex:
logger.warning(
'Server returned exception (5xx), retrying in 15 seconds (%s)',
ex)
time.sleep(15)
except HTTPTooManyRequests as ex:
# 150 API calls done, and python-fitbit doesn't provide the retry-after header, so stop trying
# and allow the limit to reset, even if it costs us one hour
logger.info('API limit reached, sleeping for 3610 seconds!')
time.sleep(3610)
except Exception as ex:
logger.exception('Got some unexpected exception')
raise
member_since = user_profile.get('user', {}).get('memberSince',
'1970-01-01')
member_since_dt = parse(member_since, ignoretz=True)
member_since_ts = parse(member_since, ignoretz=True).timestamp()
logger.info('User is member since: %s (ts: %s)', member_since,
member_since_ts)
cur_day = datetime.utcnow()
db_client = InfluxDBClient(db_host, db_port, db_user, db_password, db_name)
for one_db in db_client.get_list_database():
if one_db['name'] == db_name:
break
else:
db_client.create_database(db_name)
db_client.close()
# First try to fill any gaps: between User_member_since and first_ts,
# and then between last_ts and cur_day
while True:
for meas, series_list in BASE_SERIES.items():
for series in series_list:
resource = '{}/{}'.format(meas, series)
if '_' in meas:
resource = resource.replace('_', '/', 1)
if resource == 'sleep/sleep':
# Sleep is special, is its own main category
resource = 'sleep'
db_client = InfluxDBClient(db_host, db_port, db_user,
db_password, db_name)
key_series = series
if isinstance(series_list, dict) and series_list.get(series):
# Datapoints are retrieved with all keys in the same dict, so makes no sense to retrieve individual
# series names. Use one series as the key series.
key_series = series_list[series]['key_series']
first_ts = get_first_timestamp_for_measurement(db_client,
meas,
key_series,
min_ts=cur_day)
last_ts = get_last_timestamp_for_measurement(db_client,
meas,
key_series,
min_ts=cur_day)
profile_to_first = int(
(first_ts - member_since_dt) / timedelta(days=1))
last_to_current = int((cur_day - last_ts) / timedelta(days=1))
logger.debug(
'key_series: %s, first_ts: %s, last_ts: %s, profile_to_first: %s, last_to_current: %s',
key_series, first_ts, last_ts, profile_to_first,
last_to_current)
db_client.close()
intervals_to_fetch = []
if profile_to_first > 1:
append_between_day_series(intervals_to_fetch,
member_since_dt, first_ts)
if last_to_current > 1:
append_between_day_series(intervals_to_fetch, last_ts,
cur_day)
if not intervals_to_fetch:
logger.info(
'No gaps to fetch for %s, %s: fetching last day only',
meas, series)
intervals_to_fetch.append((
cur_day,
cur_day,
))
# DB can't be open here, because fitbit_fetch_datapoints can hang for a long time
if meas == 'sleep':
api_client.API_VERSION = '1.2'
datapoints = fitbit_fetch_datapoints(api_client, meas, series,
resource,
intervals_to_fetch)
if meas == 'sleep':
api_client.API_ENDPOINT = '1'
converted_dps = []
for one_d in datapoints:
if not one_d:
continue
if isinstance(series_list,
dict) and series_list.get(series):
new_dps = series_list[series]['transform'](one_d)
for one_dd in new_dps:
converted_dps.append(
create_api_datapoint_meas_series(
one_dd['meas'], one_dd['series'],
one_dd['value'], one_dd['dateTime']))
else:
converted_dps.append(
create_api_datapoint_meas_series(
meas, series, one_d.get('value'),
one_d.get('dateTime')))
db_client = InfluxDBClient(db_host, db_port, db_user,
db_password, db_name)
precision = 'h'
if meas == 'sleep':
precision = 's'
logger.debug(
'Going to write %s points, key_series: %s, first_ts: %s, last_ts: %s, profile_to_first: %s, last_to_current: %s',
len(converted_dps), key_series, first_ts, last_ts,
profile_to_first, last_to_current)
logger.debug('First 3: %s', converted_dps[:3])
logger.debug('Last 3: %s', converted_dps[-3:])
if not db_client.write_points(converted_dps,
time_precision=precision,
batch_size=2500):
logger.critical(
'key_series: %s, first_ts: %s, last_ts: %s, profile_to_first: %s, last_to_current: %s',
key_series, first_ts, last_ts, profile_to_first,
last_to_current)
raise Exception('Unable to write points!')
db_client.close()
logger.info('All series processed, sleeping for 4h')
time.sleep(3610 * 4)
sys.exit(0)
