def render(self, request):
logging.info("Fitbit App, request args: {0}".format(request.args))
# if "init" in request.args:
# self.indx = IndxClient("http://{0}".format(request.args['host'][0]), request.args['box'], request.args['username'], request.args["password"], "FitbitConnector")
# print self.indx
# logging.info("Fitbit App, connected to the box {0}".format(box))
# self.return_ok(request, data = {"init": "ok"})
if "gotourl" in request.args:
gotourl = self.fitbit.get_token_url()
logging.info("Fitbit App, the gotourl is {0}".format(gotourl))
self.return_ok(request, data = {"url": gotourl})
elif "pin" in request.args:
pin = request.args['pin'][0]
logging.info("Fitbit App, the pin is {0}".format(pin))
token = self.fitbit.get_token_with_pin(pin)
self.return_ok(request, data = {"token": json.dumps({"token_key": "{0}".format(token.key), "token_secret": "{0}".format(token.secret)})})
elif "token" in request.args:
token = json.loads(request.args["token"][0])
self.fitbit = Fitbit(self.consumer_key, self.consumer_secret, token['token_key'], token['token_secret'])
self.return_ok(request, data={})
elif "download" in request.args:
self.fitbit_min = FitbitIntraDay(self.fitbit)
start = None
if ("start" in request.args):
start = datetime.fromtimestamp(int(request.args["start"][0])/1000)
response = self.download_data(start)
self.return_ok(request, data = response)
else:
logging.info("Fitbit App, returning 404")
self.return_not_found(request)
return NOT_DONE_YET
def fitbit_sleep(user_id):
""" 创建或更新今天的 sleep 记录
API: https://dev.fitbit.com/build/reference/web-api/sleep/
"""
user = load_user(user_id)
oauth = get_user_oauth(user=user, provider='fitbit')
if not oauth:
return jsonify({'msg': 'Unauthorized'})
fitbit = Fitbit(client_id=os.getenv('FITBIT_APP_ID'),
client_secret=os.getenv('FITBIT_SECRET'),
access_token=oauth.token['access_token'],
refresh_token=oauth.token['refresh_token'])
# TODO: 如果没有 datetime 参数处理
dt_str = request.args.get('datetime')
dt = pendulum.parse(dt_str, strict=False)
sleep_data = fitbit.sleep(date=dt.date(), user_id=oauth.provider_user_id)
query = models.Sleep.query.filter_by(user=user, date=dt.date())
try:
sleep = query.one()
sleep.data = sleep_data
db.session.add(sleep)
db.session.commit()
except NoResultFound:
sleep = models.Sleep(user=user, data=sleep_data, date=dt.date())
db.session.add(sleep)
db.session.commit()
return jsonify(sleep.data['summary'])
def test_recent_activities(self):
user_id = "LukeSkywalker"
with mock.patch("fitbit.api.Fitbit.activity_stats") as act_stats:
fb = Fitbit("x", "y")
retval = fb.recent_activities(user_id=user_id)
args, kwargs = act_stats.call_args
self.assertEqual((), args)
self.assertEqual({"user_id": user_id, "qualifier": "recent"}, kwargs)
def test_recent_activities(self):
user_id = "LukeSkywalker"
with mock.patch('fitbit.api.Fitbit.activity_stats') as act_stats:
fb = Fitbit('x', 'y')
retval = fb.recent_activities(user_id=user_id)
args, kwargs = act_stats.call_args
self.assertEqual((), args)
self.assertEqual({'user_id': user_id, 'qualifier': 'recent'}, kwargs)
def test_delete_water(self):
log_id = "fake_log_id"
# We need to mock _DELETE_COLLECTION_RESOURCE before we create the Fitbit object,
# since the __init__ is going to set up references to it
with mock.patch("fitbit.api.Fitbit._DELETE_COLLECTION_RESOURCE") as delete_resource:
delete_resource.return_value = 999
fb = Fitbit("x", "y")
retval = fb.delete_foods_log(log_id=log_id)
args, kwargs = delete_resource.call_args
self.assertEqual(("foods/log",), args)
self.assertEqual({"log_id": log_id}, kwargs)
self.assertEqual(999, retval)
def test_delete_water(self):
log_id = "OmarKhayyam"
# We need to mock _DELETE_COLLECTION_RESOURCE before we create the Fitbit object,
# since the __init__ is going to set up references to it
with mock.patch('fitbit.api.Fitbit._DELETE_COLLECTION_RESOURCE') as delete_resource:
delete_resource.return_value = 999
fb = Fitbit('x', 'y')
retval = fb.delete_foods_log_water(log_id=log_id)
args, kwargs = delete_resource.call_args
self.assertEqual(('foods/log/water',), args)
self.assertEqual({'log_id': log_id}, kwargs)
self.assertEqual(999, retval)
def __init__(self):
log_handler = logging.FileHandler("fitbit_harvester.log", "a")
log_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(name)s\t%(levelname)s\t%(asctime)s\t%(message)s')
log_handler.setFormatter(formatter)
self.logger = logging.getLogger()
self.logger.setLevel(logging.DEBUG)
for handler in self.logger.handlers: # remove default handler
self.logger.removeHandler(handler)
self.logger.addHandler(log_handler)
data_root = keyring.util.platform_.data_root()
if not os.path.exists(data_root):
os.mkdir(data_root)
keyring.set_keyring(PlaintextKeyring())
self.parser = argparse.ArgumentParser(prog="run")
self.parser.add_argument(
'--config', help="Set config (input requires JSON) and exit.")
self.parser.add_argument(
'--get-config',
action="store_true",
help="Output current config as JSON and exit.")
self.parser.add_argument('--server',
help="The server URL to connect to.")
# init fitbit
consumer_key = "9cc7928d03fa4e1a92eda0d01ede2297"
consumer_secret = "340ea36a974e47738a335c0cccfe1fcf"
self.fitbit = Fitbit(consumer_key, consumer_secret)
self.fitbit_intraday = None
self.box_version = 0
self.config_overwrite = False
self.config_start = self.today()
self.config_box = None
self.config_indx_user = None
self.config_indx_pass = None
self.config_fetched_days = []
self.config_zeros_from = self.today()
self.harvester_id = "fitbit_harvester"
self.steps_ts_id = "fitbit_steps_ts"
self.calories_ts_id = "fitbit_calories_ts"
self.distance_ts_id = "fitbit_distance_ts"
self.floors_ts_id = "fitbit_floors_ts"
self.elevation_ts_id = "fitbit_elevation_ts"
self.ts_count = 0
self.ts_error = None
def fitbit_activity(user_id):
""" 存储今天的 activity 记录
API: https://dev.fitbit.com/build/reference/web-api/activity/
"""
user = load_user(user_id)
oauth = get_user_oauth(user=user, provider='fitbit')
if not oauth:
return jsonify({'msg': 'Unauthorized'})
fitbit = Fitbit(client_id=os.getenv('FITBIT_APP_ID'),
client_secret=os.getenv('FITBIT_SECRET'),
access_token=oauth.token['access_token'],
refresh_token=oauth.token['refresh_token'],
refresh_cb=actions.fitbit_refresh_cb)
dt_str = request.args.get('datetime')
dt = pendulum.parse(dt_str, strict=False)
# 如果 token 失效更新 token
if is_token_expired(dt, oauth.token):
new_token = fitbit.client.refresh_token()
app.logger.info('=== {} new_token: {}'.format(user.username,
new_token))
actions.update_oauth_token(oauth, new_token)
# 获取 activity
data = fitbit.activities(date=dt.date(), user_id=oauth.provider_user_id)
print('==== activities from fitbit')
print(data)
query = models.Activity.query.filter_by(user=user, date=dt.date())
try:
activities = query.one()
activities.data = data
db.session.add(activities)
db.session.commit()
except NoResultFound:
# 存储 activity
activities = models.Activity(user=user, data=data, date=dt.date())
db.session.add(activities)
db.session.commit()
result = activities.data['summary']
for distance in result['distances']:
if distance['activity'] == 'total':
result['distances'] = distance['distance']
break
return jsonify(result)
def test_body(self):
# Test the first method defined in __init__ to see if it calls
# _COLLECTION_RESOURCE okay - if it does, they should all since
# they're all built the same way
# We need to mock _COLLECTION_RESOURCE before we create the Fitbit object,
# since the __init__ is going to set up references to it
with mock.patch('fitbit.api.Fitbit._COLLECTION_RESOURCE') as coll_resource:
coll_resource.return_value = 999
fb = Fitbit('x', 'y')
retval = fb.body(date=1, user_id=2, data=3)
args, kwargs = coll_resource.call_args
self.assertEqual(('body',), args)
self.assertEqual({'date': 1, 'user_id': 2, 'data': 3}, kwargs)
self.assertEqual(999, retval)
def get_profile(self, token):
""" Returns a user profile from a token """
fb = Fitbit(self.client_id,
self.client_secret,
access_token=token['access_token'],
refresh_token=token['refresh_token'],
expires_at=token['expires_at'],
refresh_cb=update_token)
profile = fb.user_profile_get()
print("got profile")
return dict(first_name=profile['user']['firstName'],
last_name=profile['user']['lastName'],
username=token['user_id'],
token=token)
def oauth_callback(provider):
next_page = session.get('next_page', 'index')
if not current_user.is_anonymous:
return redirect(url_for(next_page))
oauth = OAuthSignIn.get_provider(provider)
token = oauth.callback()
session['token'] = token
client = Fitbit(app.config['OAUTH_CREDENTIALS'][provider]['id'],
app.config['OAUTH_CREDENTIALS'][provider]['secret'],
access_token=token['access_token'], refresh_token=token['refresh_token'])
fitbit_client = None
if provider == 'fitbit':
fitbit_client = client
if fitbit_client is not None:
me = fitbit_client.get_user_profile()
social_id = 'fitbit_' + me['user']['encodedId']
full_name = me['user']['fullName']
nickname = me['user'].get('nickname')
if social_id is None:
flash('Authentication failed.')
return redirect(url_for(next_page))
user = User.query.filter_by(social_id=social_id).first()
if not user:
user = User(social_id=social_id, full_name=full_name, nickname=nickname)
db.session.add(user)
db.session.commit()
login_user(user, remember=True)
return redirect(url_for(next_page))
def test_auto_refresh_token_non_exception(self):
"""Test of auto_refersh when the exception doesn't fire"""
# 1. first call to _request causes a 401 expired token response
# 2. the token_refresh call is faked
# 3. the second call to _request returns a valid value
refresh_cb = mock.MagicMock()
kwargs = self.client_kwargs
kwargs['access_token'] = 'fake_access_token'
kwargs['refresh_token'] = 'fake_refresh_token'
kwargs['refresh_cb'] = refresh_cb
fb = Fitbit(**kwargs)
with mock.patch.object(FitbitOauth2Client, '_request') as r:
r.side_effect = [
fake_response(
401,
b'{"errors": [{"message": "Access token expired: some_token_goes_here", "errorType": "expired_token", "fieldName": "access_token"}]}'
),
fake_response(200, 'correct_response')
]
with mock.patch.object(OAuth2Session, 'refresh_token') as rt:
rt.return_value = {
'access_token': 'fake_return_access_token',
'refresh_token': 'fake_return_refresh_token'
}
retval = fb.client.make_request(Fitbit.API_ENDPOINT +
'/1/user/-/profile.json')
self.assertEqual("correct_response", retval.text)
self.assertEqual("fake_return_access_token",
fb.client.token['access_token'])
self.assertEqual("fake_return_refresh_token",
fb.client.token['refresh_token'])
self.assertEqual(1, rt.call_count)
self.assertEqual(2, r.call_count)
refresh_cb.assert_called_once_with(rt.return_value)
def test_auto_refresh_token_exception(self):
"""Test of auto_refresh with Unauthorized exception"""
# 1. first call to _request causes a HTTPUnauthorized
# 2. the token_refresh call is faked
# 3. the second call to _request returns a valid value
kwargs = self.client_kwargs
kwargs['access_token'] = 'fake_access_token'
kwargs['refresh_token'] = 'fake_refresh_token'
fb = Fitbit(**kwargs)
with mock.patch.object(FitbitOauth2Client, '_request') as r:
r.side_effect = [
HTTPUnauthorized(fake_response(401, b'correct_response')),
fake_response(200, 'correct_response')
]
with mock.patch.object(OAuth2Session, 'refresh_token') as rt:
rt.return_value = {
'access_token': 'fake_return_access_token',
'refresh_token': 'fake_return_refresh_token'
}
retval = fb.client.make_request(Fitbit.API_ENDPOINT +
'/1/user/-/profile.json')
self.assertEqual("correct_response", retval.text)
self.assertEqual("fake_return_access_token",
fb.client.token['access_token'])
self.assertEqual("fake_return_refresh_token",
fb.client.token['refresh_token'])
self.assertEqual(1, rt.call_count)
self.assertEqual(2, r.call_count)
def test_fetch_access_token_error(self):
fb = Fitbit(**self.client_kwargs)
with mock.patch('requests.sessions.Session.post') as post:
post.return_value = mock.Mock(text="not a url encoded string")
fake_token = mock.Mock(key="FAKEKEY", secret="FAKESECRET")
self.assertRaises(ValueError, fb.client.fetch_access_token,
fake_token, "fake_verifier")
def test_response_auth(self):
"""
This test checks how the client handles different auth responses, and
the exceptions raised by the client.
"""
r = mock.Mock(spec=requests.Response)
r.status_code = 401
json_response = {
"errors": [{
"errorType": "unauthorized",
"message": "Unknown auth error"}
],
"normal": "resource"
}
r.content = json.dumps(json_response).encode('utf8')
f = Fitbit(**self.client_kwargs)
f.client._request = lambda *args, **kwargs: r
self.assertRaises(exceptions.HTTPUnauthorized, f.user_profile_get)
r.status_code = 403
json_response['errors'][0].update({
"errorType": "forbidden",
"message": "Forbidden"
})
r.content = json.dumps(json_response).encode('utf8')
self.assertRaises(exceptions.HTTPForbidden, f.user_profile_get)
def test_auto_refresh_token_nonException(self):
# test of auto_refersh when the exception doesn't fire
# 1. first call to _request causes a 401 expired token response
# 2. the token_refresh call is faked
# 3. the second call to _request returns a valid value
kwargs = self.client_kwargs
kwargs['access_token'] = 'fake_access_token'
kwargs['refresh_token'] = 'fake_refresh_token'
fb = Fitbit(**kwargs)
with mock.patch.object(FitbitOauth2Client, '_request') as r:
r.side_effect = [
fake_response(
401,
b'{"errors": [{"message": "Access token invalid or expired: some_token_goes_here", "errorType": "oauth", "fieldName": "access_token"}]}'
),
fake_response(200, 'correct_response')
]
with mock.patch.object(OAuth2Session, 'post') as auth:
auth.return_value = fake_response(
200,
'{"access_token": "fake_return_access_token", "scope": "fake_scope", "token_type": "Bearer", "refresh_token": "fake_return_refresh_token"}'
)
retval = fb.client.make_request(Fitbit.API_ENDPOINT +
'/1/user/-/profile.json')
self.assertEqual("correct_response", retval.text)
self.assertEqual("fake_return_access_token",
fb.client.token['access_token'])
self.assertEqual("fake_return_refresh_token",
fb.client.token['refresh_token'])
self.assertEqual(1, auth.call_count)
self.assertEqual(2, r.call_count)
def auth1(self,
user=None,
userData=None,
token=None,
callerToken=None,
**kwargs):
"""OAuth2 entry point 1 - start with authentication sequence, redirect user's browser to Fitbit site"""
if not user:
self.igor.app.raiseHTTPError(
"401 fitbitplugin/auth1 requires 'user' argument")
if not isinstance(userData, dict):
self.igor.app.raiseHTTPError(
'401 Element /data/identities/%s/plugindata/%s/token is missing'
% (self.user, self.pluginName))
oauthSettings = {}
for k in KEYS_PER_APP:
if not k in self.pluginData:
self.igor.app.raiseHTTPError(
"401 fitbitplugin/auth1 requires global plugindata '%s'" %
k)
oauthSettings[k] = self.pluginData[k]
fb = Fitbit(**oauthSettings)
step2url = self.igor.databaseAccessor.get_key('services/igor/url',
'text/plain', 'content',
token)
step2url = urllib.parse.urljoin(step2url,
'/plugin/%s/auth2' % self.pluginName)
#step2url += '?' + urllib.urlencode(dict(user=user))
redirectUrl, _ = fb.client.authorize_token_url(redirect_uri=step2url,
state=user)
return self.igor.app.raiseSeeother(redirectUrl)
def auth2(self,
code=None,
state=None,
token=None,
callerToken=None,
**kwargs):
"""Oatth2 entry point 2 - return data from Fitbit site via the user's browser"""
oauthSettings = {}
self.user = state
self.token = token
for k in KEYS_PER_APP:
if not k in self.pluginData:
self.igor.app.raiseHTTPError(
"401 fitbitplugin/auth2 requires global plugindata '%s'" %
k)
oauthSettings[k] = self.pluginData[k]
if not state:
self.igor.app.raiseHTTPError(
"401 fitbitplugin/auth2 requires 'state' argument")
if not code:
self.igor.app.raiseHTTPError(
"401 fitbitplugin/auth2 requires 'code' argument")
step2url = self.igor.databaseAccessor.get_key('services/igor/url',
'text/plain', 'content',
token)
step2url = urllib.parse.urljoin(step2url,
'/plugin/%s/auth2' % self.pluginName)
fb = Fitbit(state=state, redirect_uri=step2url, **oauthSettings)
fbToken = fb.client.fetch_access_token(code)
self._refresh(fbToken)
return 'ok\n'
class FitBitHelper(object):
""" Wraps around Fitbit api and contains useful methods for interacting with FitBit data """
def __init__(self, user_id):
self.user_id = user_id
# look up the user in the database
self.user = current.db(
current.db.auth_user.id == user_id).select().first()
if self.user is None:
raise ValueError(
"User {user_id} wasn't found in the database!".format(
user_id=user_id))
self.fitbit = Fitbit(current.client_id,
current.client_secret,
access_token=self.user.token['access_token'],
refresh_token=self.user.token['refresh_token'],
expires_at=self.user.token['expires_at'],
refresh_cb=_update_token)
def get_sleep_history(self, period=datetime.timedelta(days=14)):
"""
Gets a slice of sleep history ending with today
The default slice is two weeks.
"""
# get a two week sleep history
today = datetime.datetime.today()
old = today - datetime.timedelta(days=14)
return self.fitbit.time_series("sleep",
user_id=self.user.username,
base_date=old,
end_date=today)
def test_fetch_access_token_error(self):
fb = Fitbit(**self.client_kwargs)
fake_token = "FAKETOKEN"
fake_verifier = "FAKEVERIFIER"
with mock.patch.object(
oauth.Request,
'from_consumer_and_token') as from_consumer_and_token:
mock_request = mock.Mock()
mock_request.to_header.return_value = "FAKEHEADERS"
from_consumer_and_token.return_value = mock_request
with mock.patch(
'fitbit.api.FitbitOauthClient._request') as _request:
fake_response = mock.Mock()
fake_response.content = "FAKECONTENT"
fake_response.status_code = 999
_request.return_value = fake_response
with mock.patch.object(oauth.Token,
'from_string') as from_string:
from_string.return_value = "FAKERETURNVALUE"
with mock.patch('fitbit.api.urlparse') as urlparse:
urlparse.parse_qs.return_value = {
'encoded_user_id': ['foo']
}
self.assertRaises(Exception,
fb.client.fetch_access_token,
fake_token, fake_verifier)
async def setup_fitbit(hass, config):
_LOGGER.debug('Initializing Fitbit fitness push services')
from fitbit import Fitbit
from homeassistant.components.fitbit.sensor import (
FITBIT_CONFIG_FILE, ATTR_ACCESS_TOKEN as FITBIT_ATTR_ACCESS_TOKEN,
ATTR_REFRESH_TOKEN as FITBIT_ATTR_REFRESH_TOKEN, ATTR_CLIENT_ID as
FITBIT_ATTR_CLIENT_ID, ATTR_CLIENT_SECRET as FITBIT_ATTR_CLIENT_SECRET,
ATTR_LAST_SAVED_AT as FITBIT_ATTR_LAST_SAVED_AT)
config_path = hass.config.path(FITBIT_CONFIG_FILE)
if os.path.isfile(config_path):
config_file = load_json(config_path)
else:
_LOGGER.warn("No Fitbit config file found")
return None
access_token = config_file.get(FITBIT_ATTR_ACCESS_TOKEN)
refresh_token = config_file.get(FITBIT_ATTR_REFRESH_TOKEN)
expires_at = config_file.get(FITBIT_ATTR_LAST_SAVED_AT)
if None in (access_token, refresh_token):
_LOGGER.warn("Fitbit config file is not initialized")
return None
return Fitbit(
config_file.get(FITBIT_ATTR_CLIENT_ID),
config_file.get(FITBIT_ATTR_CLIENT_SECRET),
access_token=access_token,
refresh_token=refresh_token,
expires_at=expires_at,
refresh_cb=lambda x: None,
)
def test_auto_refresh_token_exception(self):
# test of auto_refersh with tokenExpired exception
# 1. first call to _request causes a TokenExpired
# 2. the token_refresh call is faked
# 3. the second call to _request returns a valid value
kwargs = self.client_kwargs
kwargs['access_token'] = 'fake_access_token'
kwargs['refresh_token'] = 'fake_refresh_token'
fb = Fitbit(**kwargs)
with mock.patch.object(FitbitOauth2Client, '_request') as r:
r.side_effect = [
TokenExpiredError,
fake_response(200, 'correct_response')
]
with mock.patch.object(OAuth2Session, 'post') as auth:
auth.return_value = fake_response(
200,
'{"access_token": "fake_return_access_token", "scope": "fake_scope", "token_type": "Bearer", "refresh_token": "fake_return_refresh_token"}'
)
retval = fb.client.make_request(Fitbit.API_ENDPOINT +
'/1/user/-/profile.json')
self.assertEqual("correct_response", retval.text)
self.assertEqual("fake_return_access_token",
fb.client.token['access_token'])
self.assertEqual("fake_return_refresh_token",
fb.client.token['refresh_token'])
self.assertEqual(1, auth.call_count)
self.assertEqual(2, r.call_count)
def fitbit_auth():
token = None
error = None
client_id = '227XNF'
client_secret = '1a53508ac0bd0aa5ffa3a9f6de07cb9d'
redirect_uri = ('https://%s/project/default/fitbit_auth' %
request.env.http_host)
code = request.vars.code
oauth = FitbitOauth2Client(client_id, client_secret)
print code
try:
token = oauth.fetch_access_token(code, redirect_uri)
except MissingTokenError:
error = 'Missing access token parameter.</br>Please check that you are using the correct client_secret'
except MismatchingStateError:
error ='CSRF Warning! Mismatching state'
print token
client = Fitbit(client_id, client_secret, access_token=token["access_token"], refresh_token=token["refresh_token"])
f_id = db.fitbit_user_t.insert(
user_email=auth.user.email,
fitbit_user_id=token["user_id"],
access_token=token["access_token"],
refresh_token=token["refresh_token"],
expires_at=token["expires_at"],
)
user = client.user_profile_get()
goals = client.activities_daily_goal()
wt_goal = client.body_weight_goal()
print wt_goal
u_id = db.user_t.insert(
user_email=auth.user.email,
dob=user["user"]["dateOfBirth"],
sex=user["user"]["gender"],
height=user["user"]["height"],
image=user["user"]["avatar150"],
steps_target=goals["goals"]["steps"],
weight_target=wt_goal["goal"]["weight"],
weight=user["user"]["weight"],
)
redirect(URL('index'))
def test_auto_refresh_token_exception(self):
"""Test of auto_refresh with Unauthorized exception"""
# 1. first call to _request causes a HTTPUnauthorized
# 2. the token_refresh call is faked
# 3. the second call to _request returns a valid value
refresh_cb = mock.MagicMock()
kwargs = copy.copy(self.client_kwargs)
kwargs.update({
'access_token': 'fake_access_token',
'refresh_token': 'fake_refresh_token',
'refresh_cb': refresh_cb,
})
fb = Fitbit(**kwargs)
profile_url = Fitbit.API_ENDPOINT + '/1/user/-/profile.json'
with requests_mock.mock() as m:
m.get(profile_url, [{
'text': json.dumps({
"errors": [{
"errorType": "expired_token",
"message": "Access token expired:"
}]
}),
'status_code': 401
}, {
'text': '{"user":{"aboutMe": "python-fitbit developer"}}',
'status_code': 200
}])
token = {
'access_token': 'fake_return_access_token',
'refresh_token': 'fake_return_refresh_token'
}
m.post(fb.client.refresh_token_url, text=json.dumps(token))
retval = fb.make_request(profile_url)
self.assertEqual(m.request_history[1].path, '/oauth2/token')
self.assertEqual(
m.request_history[1].headers['Authorization'],
_basic_auth_str(
self.client_kwargs['client_id'],
self.client_kwargs['client_secret']
)
)
self.assertEqual(retval['user']['aboutMe'], "python-fitbit developer")
self.assertEqual("fake_return_access_token", token['access_token'])
self.assertEqual("fake_return_refresh_token", token['refresh_token'])
refresh_cb.assert_called_once_with(token)
def test_authorize_token_url(self):
# authorize_token_url calls oauth and returns a URL
fb = Fitbit(**self.client_kwargs)
with mock.patch.object(OAuth1Session, 'authorization_url') as au:
au.return_value = 'FAKEURL'
retval = fb.client.authorize_token_url()
self.assertEqual(1, au.call_count)
self.assertEqual("FAKEURL", retval)
def test_authorize_token_url(self):
# authorize_token_url calls oauth and returns a URL
fb = Fitbit(**self.client_kwargs)
retval = fb.client.authorize_token_url()
self.assertEqual(
retval[0],
'https://www.fitbit.com/oauth2/authorize?response_type=code&client_id=fake_id&scope=activity+nutrition+heartrate+location+nutrition+profile+settings+sleep+social+weight&state='
+ retval[1])
def __init__(self, user_id):
self.user_id = user_id
# look up the user in the database
self.user = current.db(
current.db.auth_user.id == user_id).select().first()
if self.user is None:
raise ValueError(
"User {user_id} wasn't found in the database!".format(
user_id=user_id))
self.fitbit = Fitbit(current.client_id,
current.client_secret,
access_token=self.user.token['access_token'],
refresh_token=self.user.token['refresh_token'],
expires_at=self.user.token['expires_at'],
refresh_cb=_update_token)
def test_auto_refresh_token_exception(self):
"""Test of auto_refresh with Unauthorized exception"""
# 1. first call to _request causes a HTTPUnauthorized
# 2. the token_refresh call is faked
# 3. the second call to _request returns a valid value
refresh_cb = mock.MagicMock()
kwargs = copy.copy(self.client_kwargs)
kwargs.update({
'access_token': 'fake_access_token',
'refresh_token': 'fake_refresh_token',
'refresh_cb': refresh_cb,
})
fb = Fitbit(**kwargs)
profile_url = Fitbit.API_ENDPOINT + '/1/user/-/profile.json'
with requests_mock.mock() as m:
m.get(profile_url, [{
'text':
json.dumps({
"errors": [{
"errorType": "expired_token",
"message": "Access token expired:"
}]
}),
'status_code':
401
}, {
'text': '{"user":{"aboutMe": "python-fitbit developer"}}',
'status_code': 200
}])
token = {
'access_token': 'fake_return_access_token',
'refresh_token': 'fake_return_refresh_token'
}
m.post(fb.client.refresh_token_url, text=json.dumps(token))
retval = fb.make_request(profile_url)
self.assertEqual(m.request_history[1].path, '/oauth2/token')
self.assertEqual(
m.request_history[1].headers['Authorization'],
_basic_auth_str(self.client_kwargs['client_id'],
self.client_kwargs['client_secret']))
self.assertEqual(retval['user']['aboutMe'], "python-fitbit developer")
self.assertEqual("fake_return_access_token", token['access_token'])
self.assertEqual("fake_return_refresh_token", token['refresh_token'])
refresh_cb.assert_called_once_with(token)
def create_fitbit(**kwargs):
consumer_key = None
consumer_secret = None
try:
consumer_key = settings.FITBIT_CONSUMER_KEY
consumer_secret = settings.FITBIT_CONSUMER_SECRET
except:
raise ImproperlyConfigured('Missing Fitbit API credentials')
return Fitbit(consumer_key, consumer_secret, **kwargs)
def test_too_many_requests(self):
"""
Tests the 429 response, given in case of exceeding the rate limit
"""
r = mock.Mock(spec=requests.Response)
r.content = b"{'normal': 'resource'}"
r.headers = {'Retry-After': '10'}
f = Fitbit(**self.client_kwargs)
f.client._request = lambda *args, **kwargs: r
r.status_code = 429
try:
f.user_profile_get()
self.assertEqual(True, False) # Won't run if an exception's raised
except exceptions.HTTPTooManyRequests:
e = sys.exc_info()[1]
self.assertEqual(e.retry_after_secs, 10)
async def sync():
fit_client = Fitbit(user=session["user"], access_token=app.secret_key)
result = fit_client.init_sync()
# Create a producer client to send messages to the event hub.
# Specify a connection string to your event hubs namespace and
# the event hub name.
producer = EventHubProducerClient.from_connection_string(
conn_str=app.config["EVENT_HUB_CONN_STR"])
async with producer:
# Create a batch.
event_data_batch = await producer.create_batch()
for item in result:
print(item)
event_data_batch.add(EventData(json.dumps(item, indent=4)))
# Send the batch of events to the event hub.
await producer.send_batch(event_data_batch)
def test_authorize_token_url_with_scope(self):
# authorize_token_url calls oauth and returns a URL
fb = Fitbit(**self.client_kwargs)
retval = fb.client.authorize_token_url(
scope=self.client_kwargs['scope'])
self.assertEqual(
retval[0],
'https://www.fitbit.com/oauth2/authorize?response_type=code&client_id=fake_id&redirect_uri=http%3A%2F%2F127.0.0.1%3A8080&scope='
+ str(self.client_kwargs['scope'][0]) + '&state=' + retval[1])
def test_too_many_requests(self):
"""
Tests the 429 response, given in case of exceeding the rate limit
"""
r = mock.Mock(spec=requests.Response)
r.content = b"{'normal': 'resource'}"
r.headers = {'Retry-After': '10'}
f = Fitbit(**self.client_kwargs)
f.client._request = lambda *args, **kwargs: r
r.status_code = 429
try:
f.user_profile_get()
self.assertEqual(True, False) # Won't run if an exception's raised
except exceptions.HTTPTooManyRequests:
e = sys.exc_info()[1]
self.assertEqual(e.retry_after_secs, 10)
def login():
fitbit = Fitbit(client_id=os.environ['FITBIT_CLIENT_ID'],
client_secret=os.environ['FITBIT_CLIENT_SECRET'],
redirect_uri=urljoin(flask.request.host_url,
redirect_path))
print(urljoin(flask.request.host_url, redirect_path))
url, state = fitbit.client.authorize_token_url(scope=None)
flask.session['login_state'] = state
print(f'redirect url is {url}')
return flask.redirect(url)
class TimeoutTest(TestCase):
def setUp(self):
self.fb = Fitbit('x', 'y')
self.fb_timeout = Fitbit('x', 'y', timeout=10)
self.test_url = 'invalid://do.not.connect'
def test_fb_without_timeout(self):
with mock.patch.object(self.fb.client.session, 'request') as request:
mock_response = mock.Mock()
mock_response.status_code = 200
mock_response.content = b'{}'
request.return_value = mock_response
result = self.fb.make_request(self.test_url)
request.assert_called_once()
self.assertNotIn('timeout', request.call_args[1])
self.assertEqual({}, result)
def test_fb_with_timeout__timing_out(self):
with mock.patch.object(self.fb_timeout.client.session, 'request') as request:
request.side_effect = requests.Timeout('Timed out')
with self.assertRaisesRegexp(Timeout, 'Timed out'):
self.fb_timeout.make_request(self.test_url)
request.assert_called_once()
self.assertEqual(10, request.call_args[1]['timeout'])
def test_fb_with_timeout__not_timing_out(self):
with mock.patch.object(self.fb_timeout.client.session, 'request') as request:
mock_response = mock.Mock()
mock_response.status_code = 200
mock_response.content = b'{}'
request.return_value = mock_response
result = self.fb_timeout.make_request(self.test_url)
request.assert_called_once()
self.assertEqual(10, request.call_args[1]['timeout'])
self.assertEqual({}, result)
def __init__(self):
log_handler = logging.FileHandler("fitbit_harvester.log", "a")
log_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(name)s\t%(levelname)s\t%(asctime)s\t%(message)s')
log_handler.setFormatter(formatter)
self.logger = logging.getLogger()
self.logger.setLevel(logging.DEBUG)
for handler in self.logger.handlers: # remove default handler
self.logger.removeHandler(handler)
self.logger.addHandler(log_handler)
data_root = keyring.util.platform_.data_root()
if not os.path.exists(data_root):
os.mkdir(data_root)
keyring.set_keyring(PlaintextKeyring())
self.parser = argparse.ArgumentParser(prog="run")
self.parser.add_argument('--config', help="Set config (input requires JSON) and exit.")
self.parser.add_argument('--get-config', action="store_true", help="Output current config as JSON and exit.")
self.parser.add_argument('--server', help="The server URL to connect to.")
# init fitbit
consumer_key = "9cc7928d03fa4e1a92eda0d01ede2297"
consumer_secret = "340ea36a974e47738a335c0cccfe1fcf"
self.fitbit = Fitbit(consumer_key, consumer_secret)
self.fitbit_intraday = None
self.box_version = 0
self.config_overwrite = False;
self.config_start = self.today()
self.config_box = None
self.config_indx_user = None
self.config_indx_pass = None
self.config_fetched_days = []
self.config_zeros_from = self.today()
self.harvester_id = "fitbit_harvester"
self.steps_ts_id = "fitbit_steps_ts"
self.calories_ts_id = "fitbit_calories_ts"
self.distance_ts_id = "fitbit_distance_ts"
self.floors_ts_id = "fitbit_floors_ts"
self.elevation_ts_id = "fitbit_elevation_ts"
self.ts_count = 0
self.ts_error = None
def test_response_ok(self):
"""
This mocks a pretty normal resource, that the request was authenticated,
and data was returned. This test should just run and not raise any
exceptions
"""
r = mock.Mock(spec=requests.Response)
r.status_code = 200
r.content = b'{"normal": "resource"}'
f = Fitbit(**self.client_kwargs)
f.client._request = lambda *args, **kwargs: r
f.user_profile_get()
r.status_code = 202
f.user_profile_get()
r.status_code = 204
f.user_profile_get()
def index():
if 'fitbit_access_token' not in session:
return redirect(url_for('login'))
client = Fitbit(CONSUMER_KEY, CONSUMER_SECRET, resource_owner_key=session['fitbit_access_token']['oauth_token'], resource_owner_secret=session['fitbit_access_token']['oauth_token_secret'])
return str(client.user_profile_get())
# if args['debug']:
# logging.basicConfig(level = logging.DEBUG)
# else:
# logging.basicConfig(level=logging.INFO)
# set up connection to INDX
# password = getpass.getpass()
# indx = IndxClient(args['address'], args['box'], args['user'], args['passwd'], "Fitbit Connector")
# set up connection to Fitbit
consumer_key = "9cc7928d03fa4e1a92eda0d01ede2297"
consumer_secret = "340ea36a974e47738a335c0cccfe1fcf"
# fitbit = Fitbit(consumer_key, consumer_secret, access_token_key, access_token_secret)
fitbit = Fitbit(consumer_key, consumer_secret)
if fitbit.token == None:
gotourl = fitbit.get_token_url()
pin = raw_input("Please input you PIN: ")
fitbit.get_token_with_pin(pin)
fitbit_min = FitbitIntraDay(fitbit)
# def get_fitbit_data():
# from_date = args['from_date']
# to_date = args['to_date']
# print '\nget activities tracker steps:\n'
# return fitbit_ts.get_activities_tracker_steps(to_date, from_date)
def transform_fitbit_response(response):
class FitbitApp(BaseHandler):
def __init__(self, server):
BaseHandler.__init__(self, server)
self.isLeaf = True
self.consumer_key = "9cc7928d03fa4e1a92eda0d01ede2297"
self.consumer_secret = "340ea36a974e47738a335c0cccfe1fcf"
self.fitbit = Fitbit(self.consumer_key, self.consumer_secret)
def render(self, request):
logging.info("Fitbit App, request args: {0}".format(request.args))
# if "init" in request.args:
# self.indx = IndxClient("http://{0}".format(request.args['host'][0]), request.args['box'], request.args['username'], request.args["password"], "FitbitConnector")
# print self.indx
# logging.info("Fitbit App, connected to the box {0}".format(box))
# self.return_ok(request, data = {"init": "ok"})
if "gotourl" in request.args:
gotourl = self.fitbit.get_token_url()
logging.info("Fitbit App, the gotourl is {0}".format(gotourl))
self.return_ok(request, data = {"url": gotourl})
elif "pin" in request.args:
pin = request.args['pin'][0]
logging.info("Fitbit App, the pin is {0}".format(pin))
token = self.fitbit.get_token_with_pin(pin)
self.return_ok(request, data = {"token": json.dumps({"token_key": "{0}".format(token.key), "token_secret": "{0}".format(token.secret)})})
elif "token" in request.args:
token = json.loads(request.args["token"][0])
self.fitbit = Fitbit(self.consumer_key, self.consumer_secret, token['token_key'], token['token_secret'])
self.return_ok(request, data={})
elif "download" in request.args:
self.fitbit_min = FitbitIntraDay(self.fitbit)
start = None
if ("start" in request.args):
start = datetime.fromtimestamp(int(request.args["start"][0])/1000)
response = self.download_data(start)
self.return_ok(request, data = response)
else:
logging.info("Fitbit App, returning 404")
self.return_not_found(request)
return NOT_DONE_YET
def download_data(self, start):
# end time is end of yesterday
end = datetime.combine((datetime.now()+timedelta(days=-1)).date(), time(23,59,59))
response = {}
if (start == None):
d = timedelta(days=0)
start = datetime.combine(end.date()+d, time(0,0,0))
response["from_date"] = start.isoformat()
steps = self.fitbit_min.get_steps(start, end)
calories = self.fitbit_min.get_calories(start, end)
distance = self.fitbit_min.get_distance(start, end)
floors = self.fitbit_min.get_floors(start, end)
elevation = self.fitbit_min.get_elevation(start, end)
compact_steps = self.compact_data(steps)
compact_calories = self.compact_data(calories)
compact_distance = self.compact_data(distance)
compact_floors = self.compact_data(floors)
compact_elevation = self.compact_data(elevation)
observations = self.create_observation_points({"step_count":compact_steps, "calories_burned": compact_calories, "distance": compact_distance, "floors_climbed": compact_floors, "elevation": compact_elevation})
response["up_to_date"] = end.isoformat()
response["observations"] = "{0}".format(json.dumps(observations))
return response
def compact_data(self, observations):
out = {}
for day_data in observations:
out = dict(out.items() + self.compact_day_data(day_data).items())
return out
def compact_day_data(self, observations):
out = {}
day = None
for key in observations.keys():
if (not key.endswith('-intraday')):
day = datetime.strptime(observations[key][0]["dateTime"], "%Y-%m-%d").date()
for key in observations.keys():
if (key.endswith('-intraday')):
for obs in observations[key]["dataset"]:
if (obs["value"] != 0):
t = datetime.strptime(obs["time"], "%H:%M:%S").time()
out[datetime.combine(day, t)] = obs["value"]
return out
def create_observation_points(self, lists):
data_points = {}
for key in lists.keys():
lst = lists[key]
for d in lst.keys():
if (d in data_points):
data_points[d] = dict(data_points[d].items() + {key: lst[d]}.items())
else:
data_points[d] = {key: lst[d]}
observations = []
for data_point in data_points.items():
obs = self.create_observation_point(data_point[0], data_point[1])
observations.append(obs)
return observations
def create_observation_point(self, ts, data):
obs = {}
# obs['@id'] = 'fitbit_{0}'.format(ts.strftime('%Y_%m_%d_%H_%M')) # put the id and type in the js code to transfer less data
# obs['@type'] = 'http://purl.org/linked-data/cube#Observation'
obs['start'] = ts.isoformat()
obs['end'] = (ts+timedelta(seconds=59)).isoformat()
# obs['device'] = [ { '@type': 'http://www.w3.org/2001/XMLSchema#string', '@value': 'Fitbit Connector' } ] # put the device in the js code to transfer less data
for key in data.keys():
obs[key] = '{0}'.format(data[key])
return obs
def __init__(self, server):
BaseHandler.__init__(self, server)
self.isLeaf = True
self.consumer_key = "9cc7928d03fa4e1a92eda0d01ede2297"
self.consumer_secret = "340ea36a974e47738a335c0cccfe1fcf"
self.fitbit = Fitbit(self.consumer_key, self.consumer_secret)
class FitbitHarvester:
def __init__(self):
log_handler = logging.FileHandler("fitbit_harvester.log", "a")
log_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(name)s\t%(levelname)s\t%(asctime)s\t%(message)s')
log_handler.setFormatter(formatter)
self.logger = logging.getLogger()
self.logger.setLevel(logging.DEBUG)
for handler in self.logger.handlers: # remove default handler
self.logger.removeHandler(handler)
self.logger.addHandler(log_handler)
data_root = keyring.util.platform_.data_root()
if not os.path.exists(data_root):
os.mkdir(data_root)
keyring.set_keyring(PlaintextKeyring())
self.parser = argparse.ArgumentParser(prog="run")
self.parser.add_argument('--config', help="Set config (input requires JSON) and exit.")
self.parser.add_argument('--get-config', action="store_true", help="Output current config as JSON and exit.")
self.parser.add_argument('--server', help="The server URL to connect to.")
# init fitbit
consumer_key = "9cc7928d03fa4e1a92eda0d01ede2297"
consumer_secret = "340ea36a974e47738a335c0cccfe1fcf"
self.fitbit = Fitbit(consumer_key, consumer_secret)
self.fitbit_intraday = None
self.box_version = 0
self.config_overwrite = False;
self.config_start = self.today()
self.config_box = None
self.config_indx_user = None
self.config_indx_pass = None
self.config_fetched_days = []
self.config_zeros_from = self.today()
self.harvester_id = "fitbit_harvester"
self.steps_ts_id = "fitbit_steps_ts"
self.calories_ts_id = "fitbit_calories_ts"
self.distance_ts_id = "fitbit_distance_ts"
self.floors_ts_id = "fitbit_floors_ts"
self.elevation_ts_id = "fitbit_elevation_ts"
self.ts_count = 0
self.ts_error = None
def set_config(self, args):
stored_config_harvester = keyring.get_password("INDX", "INDX_Fitbit_Harvester")
if stored_config_harvester is not None:
stored_config_harvester = json.loads(stored_config_harvester)
stored_config_fitbit = keyring.get_password("Fitbit.com", "Fitbit")
if stored_config_fitbit is not None:
stored_config_fitbit = json.loads(stored_config_fitbit)
received_config = json.loads(args['config'])
if (type(received_config) != dict):
received_config = json.loads(received_config)
self.logger.debug("Received config ({0}): {1}".format(type(received_config), received_config))
config = {}
if 'fitbit' in received_config:
fitbit_config = received_config['fitbit']
if fitbit_config and ('pin' in fitbit_config) and ('req_token' in fitbit_config): # this should check for the req_token in the stored config!
self.logger.debug("Received pin: {0}".format(fitbit_config['pin']))
try:
token = self.fitbit.get_token_with_pin(fitbit_config['pin'], fitbit_config['req_token'])
self.logger.debug("Got auth token {0}, of type {1}".format(token, type(token)))
if token:
config['token']=token
keyring.set_password("Fitbit.com", "Fitbit", json.dumps(config))
except Exception as exc:
self.logger.error("Could not authorise to fitbit, with pin {0}, error: {1}".format(fitbit_config['pin'], exc))
if 'harvester' in received_config:
harvester_config = received_config['harvester']
if harvester_config != stored_config_harvester:
keyring.set_password("INDX", "INDX_Fitbit_Harvester", json.dumps(harvester_config))
def get_config(self, args):
stored_config_harvester = keyring.get_password("INDX", "INDX_Fitbit_Harvester")
stored_config_fitbit = keyring.get_password("Fitbit.com", "Fitbit")
self.logger.debug("Loaded harvester config from keyring: {0}".format(stored_config_harvester))
self.logger.debug("Loaded fitbit config from keyring: {0}".format(stored_config_fitbit))
if stored_config_fitbit is None :
token_url = self.fitbit.get_token_url()
config_fitbit = {}
config_fitbit["url"] = token_url['url']
config_fitbit["req_token"] = token_url['req_token']
else :
if (type(stored_config_fitbit) != dict):
config_fitbit = json.loads(stored_config_fitbit)
# if (type(config_fitbit) != dict):
# config_fitbit = json.loads(config_fitbit)
if 'token' not in config_fitbit :
token_url = self.fitbit.get_token_url()
config_fitbit["url"] = token_url['url']
config_fitbit["req_token"] = token_url['req_token']
keyring.set_password("Fitbit.com", "Fitbit", json.dumps(config_fitbit))
if stored_config_harvester is None:
return json.dumps({"fitbit":config_fitbit}) # don't send the req_token
return json.dumps({"fitbit":config_fitbit, "harvester":json.loads(stored_config_harvester)}) # don't send the req_token
def load_configuration(self):
stored_config_harvester = keyring.get_password("INDX", "INDX_Fitbit_Harvester")
self.logger.debug("Loaded harvester config from keyring: {0}".format(stored_config_harvester))
if stored_config_harvester is None :
self.logger.error("Harvester not configured. Please configure before use.")
return
if (type(stored_config_harvester) != dict):
stored_config_harvester = json.loads(stored_config_harvester)
if self.fitbit.get_token() is None:
stored_config_fitbit = keyring.get_password("Fitbit.com", "Fitbit")
self.logger.debug("Loaded fitbit config from keyring: {0}".format(stored_config_fitbit))
token = None
if stored_config_fitbit is None :
self.logger.error("Not authenticated to Fitbit.com. Please configure before use.")
return
else :
if (type(stored_config_fitbit) != dict):
stored_config_fitbit = json.loads(stored_config_fitbit)
if 'token' not in stored_config_fitbit :
self.logger.debug("Could not find Fitbit auth token in keyring");
if ('pin' in stored_config_fitbit) and ('req_token' in stored_config_fitbit):
self.logger.debug("Found pin {0} and req_token in keyring, attempting authorization to Fitbit.com".format(stored_config_fitbit['pin']))
try:
fitbit_token_config = {}
token = self.fitbit.get_token_with_pin(stored_fitbit_config['pin'], stored_fitbit_config['req_token'])
if token:
self.logger.debug("Got auth token {0}".format(token))
# self.logger.debug("Got auth token of type {0}".format(type(token)))
fitbit_token_config['token']=token
keyring.set_password("Fitbit.com", "Fitbit", json.dumps(fitbit_token_config))
except Exception as exc:
self.logger.error("Could not authorise to fitbit, error: {1}".format(exc))
return
else :
self.logger.debug("Could not find pin or req_token in keyring. Cannot attempt authorization to Fitbit.com.")
return
else:
token = stored_config_fitbit['token']
if token is not None :
self.fitbit.set_token(token)
self.config_start = datetime.strptime(stored_config_harvester['start'], "%Y-%m-%d")
self.config_box = stored_config_harvester['box']
self.config_indx_user = stored_config_harvester['user']
self.config_indx_pass = stored_config_harvester['password']
if 'overwrite' in stored_config_harvester :
self.config_overwrite = stored_config_harvester['overwrite']
def run(self):
args = vars(self.parser.parse_args())
self.logger.debug("Received arguments: {0}".format(args))
if args['config']:
self.set_config(args)
elif args['get_config']:
print self.get_config(args)
else:
self.logger.debug("Starting the harvester. ")
self.work(args['server'])
reactor.run()
def yesterday(self):
return datetime.combine((datetime.now()+timedelta(days=-1)).date(), time(00,00,00))
def today(self):
return datetime.combine(datetime.now().date(), time(00,00,00))
def get_indx(self, server_url):
indx_d = Deferred()
def authed_cb():
def token_cb(token):
indx = IndxClient(server_url, self.config_box, "INDX_Fitbit_Harvester", token = token, client = authclient.client)
indx_d.callback(indx)
authclient.get_token(self.config_box).addCallbacks(token_cb, indx_d.errback)
authclient = IndxClientAuth(server_url, "INDX_Fitbit_Harvester")
authclient.auth_plain(self.config_indx_user, self.config_indx_pass).addCallbacks(lambda response: authed_cb(), indx_d.errback)
return indx_d
def work(self, server_url):
self.load_configuration()
self.logger.debug("Loaded configuration: box: {0}, user: {1}, pass: {2}, overwrite: {3}, start: {4}".format(self.config_box, self.config_indx_user, self.config_indx_pass, self.config_overwrite, self.config_start.isoformat()))
self.fitbit_intraday = FitbitIntraDay(self.fitbit)
self.logger.debug("Created FitbitIntraDay.")
def indx_cb(indx):
self.logger.debug("Created INDXClient.")
prep_d = Deferred()
def objects_cb(harvester, indx=indx):
# self.logger.debug("Results for object creation: {0}".format(objs))
self.logger.debug("Found or created all 5 time series.")
self.logger.debug("Found or created harvester. {0}".format(harvester))
def wait(x):
self.logger.debug("Harvested! Suspending execution for 6 hours at {0}.".format(datetime.now().isoformat()))
sleep(21600)
prep_d.callback(None)
# harvester = objs[5]
if harvester:
if "zeros_from" in harvester :
self.config_zeros_from = datetime.strptime(harvester["zeros_from"][0]["@value"], "%Y-%m-%dT%H:%M:%S")
if self.config_start < self.config_zeros_from :
if self.config_overwrite :
self.config_zeros_from = self.today()
self.config_overwrite = False; # will only overwrite the first time if the flag is set
else:
self.config_start = stored_zeros_from+timedelta(days=-1)
if "fetched_days" in harvester :
self.config_fetched_days = self.parse_list(harvester["fetched_days"])
self.harvest(indx, harvester).addCallbacks(wait, prep_d.errback)
self.find_create(indx, self.steps_ts_id, {"http://www.w3.org/2000/01/rdf-schema#label":"Fitbit Steps Time Series", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type":"http://purl.org/linked-data/cube#Dataset"}).addCallbacks(
lambda x: self.find_create(indx, self.calories_ts_id, {"http://www.w3.org/2000/01/rdf-schema#label":"Fitbit Calories Time Series", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type":"http://purl.org/linked-data/cube#Dataset"}), prep_d.errback).addCallbacks(
lambda x: self.find_create(indx, self.distance_ts_id, {"http://www.w3.org/2000/01/rdf-schema#label":"Fitbit Distance Time Series", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type":"http://purl.org/linked-data/cube#Dataset"}), prep_d.errback).addCallbacks(
lambda x: self.find_create(indx, self.floors_ts_id, {"http://www.w3.org/2000/01/rdf-schema#label":"Fitbit Floors Time Series", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type":"http://purl.org/linked-data/cube#Dataset"}), prep_d.errback).addCallbacks(
lambda x: self.find_create(indx, self.elevation_ts_id, {"http://www.w3.org/2000/01/rdf-schema#label":"Fitbit Elevation Time Series", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type":"http://purl.org/linked-data/cube#Dataset"}), prep_d.errback).addCallbacks(
lambda x: self.find_create(indx, self.harvester_id, {"http://www.w3.org/2000/01/rdf-schema#label":"INDX Fitbit Harvester extra info"}), prep_d.errback).addCallbacks(objects_cb, prep_d.errback)
return prep_d
def indx_fail_cb(f):
self.logger.error("Error getting indx! Exiting... {0}".format(f))
sys.exit(1)
def loop_fail_cb(f):
self.logger.error("Error in indx_cb! Exiting... {0}".format(f))
sys.exit(1)
def loop(x=None):
self.get_indx(server_url).addCallbacks(indx_cb, indx_fail_cb).addCallbacks(loop, loop_fail_cb)
threads.deferToThread(loop)
def harvest(self, indx, harvester):
self.logger.debug("Starting to harvest!")
harvest_d = Deferred()
def process_cb(day_points, day):
self.logger.debug("Process day {0}".format(day.isoformat()))
process_d = Deferred()
# processing steps
steps = self.download_steps(day)
zeros = False
zeros = self.check_all_zero(steps)
if zeros :
self.logger.debug("Step data points are all 0. ")
if self.config_zeros_from > day:
self.config_zeros_from = day
else :
self.config_zeros_from = self.today()
steps_points = self.prepare_points(steps, day_points, day, self.steps_ts_id, "http://sociam.org/ontology/health/StepCount")
# processing calories
calories = self.download_calories(day)
calories_points = self.prepare_points(calories, day_points, day, self.calories_ts_id, "http://sociam.org/ontology/health/CaloriesBurned")
# processing distance
distance = self.download_distance(day)
distance_points = self.prepare_points(distance, day_points, day, self.distance_ts_id, "http://sociam.org/ontology/health/Distance")
# processing floors
floors = self.download_floors(day)
floors_points = self.prepare_points(floors, day_points, day, self.floors_ts_id, "http://sociam.org/ontology/health/FloorsClimbed")
# processing elevation
elevation = self.download_elevation(day)
elevation_points = self.prepare_points(elevation, day_points, day, self.elevation_ts_id, "http://sociam.org/ontology/health/Elevation")
self.safe_update(indx, steps_points).addCallbacks(
lambda x: self.safe_update(indx, calories_points), process_d.errback).addCallbacks(
lambda x: self.safe_update(indx, distance_points), process_d.errback).addCallbacks(
lambda x: self.safe_update(indx, floors_points), process_d.errback).addCallbacks(
lambda x: self.safe_update(indx, elevation_points), process_d.errback).addCallbacks(process_d.callback, process_d.errback)
return process_d
def step_day(x, day, indx=indx, harvester=harvester):
self.logger.debug("Current day: {0}".format(day))
if day < self.today():
next_day = day + timedelta(days=+1)
self.get_day_points(indx, day).addCallbacks(process_cb, harvest_d.errback, callbackArgs=[day]).addCallbacks(step_day, harvest_d.errback, callbackArgs=[next_day, indx])
else:
self.logger.debug("Finished harvesting round. Saving harvester data .. ")
harvester["fetched_days"] = self.config_fetched_days
harvester["zeros_from"] = self.config_zeros_from.isoformat()
self.safe_update(indx, harvester).addCallbacks(harvest_d.callback, harvest_d.errback)
self.logger.debug("Fetched days : {0}, Start from : {1}, Zeros from : {2}".format(self.config_fetched_days, self.config_start.isoformat(), self.config_zeros_from.isoformat()))
step_day(None, self.config_start, indx)
return harvest_d
def prepare_points(self, raw_points, day_points, day, ts_id, rdf_type=None):
points = []
if day_points and ts_id in day_points and day_points[ts_id] :
points = self.replace_data_points(day, raw_points, day_points[ts_id], ts_id, rdf_type)
else:
points = self.create_data_points(day, raw_points, ts_id, rdf_type)
return points
def create_data_points(self, day, data, ts_id, rdf_type=None):
self.logger.debug("Started creating data points.")
data_points = []
for key in data.keys():
if key.endswith('-intraday'):
for point in data[key]["dataset"]:
interval_start = datetime.combine(day, datetime.strptime(point["time"], "%H:%M:%S").time())
interval_end = interval_start+timedelta(minutes=1)
value = point["value"]
data_point = { "@id": "fitbit_dp_{0}".format(uuid.uuid4()),
"tstart": interval_start.isoformat(),
"tend": interval_end.isoformat(),
"value": value,
"timeseries": { "@id": ts_id } }
if rdf_type:
data_point["http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] = [ { "@id": rdf_type } ]
data_points.append(data_point)
self.logger.debug("Finished creating data points.")
return data_points
def replace_data_points(self, day, new_data, old_data, ts_id, rdf_type=None):
self.logger.debug("Started replacing values in data points.")
data_points = []
replaced = 0
kept = 0
created = 0
for key in new_data.keys():
if key.endswith('-intraday'):
for point in new_data[key]["dataset"]:
data_point = None
interval_start = datetime.combine(day, datetime.strptime(point["time"], "%H:%M:%S").time())
interval_end = interval_start+timedelta(minutes=1)
value = point["value"]
if interval_start in old_data:
if len(old_data[interval_start]) > 1 :
self.logger.error("There are {0} points for the same start time {1} in the same dataset {2}!!".format(len(old_data[interval_start]), interval_start, ts_id))
old_point = old_data[interval_start][0] # there shouldn't be more than 1 point here!!!
if old_point['value'][0]['@value'] == str(value):
kept = kept+1
else:
replaced = replaced+1
# if all_other_fields_match:
data_point = { "@id": old_point['@id'] }
else:
created = created+1
data_point = { "@id": "fitbit_dp_{0}".format(uuid.uuid4()) }
if data_point :
self.logger.debug("Making a data point for {0}".format(interval_start.isoformat()))
data_point["tstart"] = interval_start.isoformat()
data_point["tend"] = interval_end.isoformat()
data_point["value"] = value
data_point["timeseries"] = { "@id": ts_id }
if rdf_type:
data_point["http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] = [ { "@id": rdf_type } ]
data_points.append(data_point)
self.logger.debug("Finished replacing values in data points - replaced: {0}, kept: {1}, created: {2}".format(replaced, kept, created))
return data_points
def download_steps(self, day):
if (day == None):
self.logger.error("No date given for download.")
return []
steps = self.fitbit_intraday.get_steps(day)[0]
self.logger.debug("Retrieved steps data for {0}.".format(day.date()))
return steps
def download_calories(self, day):
if (day == None):
self.logger.error("No date given for download.")
return []
calories = self.fitbit_intraday.get_calories(day)[0]
self.logger.debug("Retrieved calories data for {0}.".format(day.date()))
return calories
def download_distance(self, day):
if (day == None):
self.logger.error("No date given for download.")
return []
distance = self.fitbit_intraday.get_distance(day)[0]
self.logger.debug("Retrieved distance data for {0}.".format(day.date()))
return distance
def download_floors(self, day):
if (day == None):
self.logger.error("No date given for download.")
return []
floors = self.fitbit_intraday.get_floors(day)[0]
self.logger.debug("Retrieved floors data for {0}.".format(day.date()))
return floors
def download_elevation(self, day):
if (day == None):
self.logger.error("No date given for download.")
return []
elevation = self.fitbit_intraday.get_elevation(day)[0]
self.logger.debug("Retrieved elevation data for {0}.".format(day.date()))
return elevation
def get_day_points(self, indx, day):
self.logger.debug("getting day points for {0}".format(day.isoformat()))
points_d = Deferred()
d = day.date()
def points_cb(pts):
points_d.callback(pts)
if d.isoformat() in self.config_fetched_days :
self.logger.debug("Data for {0} was already fetched, rechecking!".format(d.isoformat()))
self.find_day_points(indx, day).addCallbacks(points_cb, points_d.errback)
else:
self.logger.debug("Data for {0} was not yet fetched. Getting it now.".format(d.isoformat()))
self.config_fetched_days.append(d.isoformat())
points_d.callback(None)
return points_d
def find_day_points(self, indx, day) :
self.logger.debug("Find day data points from {0}".format(day.date().isoformat()))
points_d = Deferred()
def found_cb(results):
out = {self.steps_ts_id:{}, self.calories_ts_id:{}, self.distance_ts_id:{}, self.floors_ts_id:{}, self.elevation_ts_id:{}}
for resp in results:
if resp and 'code' in resp and resp['code']==200 and 'data' in resp:
for pt in resp['data'] :
obj = resp['data'][pt]
if 'timeseries' in obj and obj['timeseries'][0]['@id'] in [self.steps_ts_id, self.calories_ts_id, self.distance_ts_id, self.floors_ts_id, self.elevation_ts_id] :
objs_date_hash = out[obj['timeseries'][0]['@id']]
if find_start in objs_date_hash:
objs_list = objs_date_hash[find_start]
else:
objs_list = []
objs_list.append(obj)
objs_date_hash[find_start] = objs_list
out[obj['timeseries'][0]['@id']] = objs_date_hash
self.logger.debug("Found points with start time {0}: {1}".format(find_start.isoformat(),objs_list))
else:
self.logger.error("Didn't find any time points: {0}".format(resp))
points_d.callback(out)
deferreds = []
for h in range(24):
for m in range(60):
find_start = datetime.combine(day.date(), time(h,m,0))
self.logger.debug("Searching data points with start time: {0}".format(find_start.isoformat()))
deferreds.append(indx.query(json.dumps({"tstart":find_start.isoformat()})))
dl = defer.gatherResults(deferreds)
dl.addCallbacks(found_cb, points_d.errback)
return points_d
def safe_update(self, indx, obj) :
self.logger.debug("Updating objects at box version {0}".format(self.box_version))
update_d = Deferred()
def update_cb(resp):
self.logger.debug("safe_update: received response: {0}".format(resp))
if resp and "code" in resp and "data" in resp:
if resp["code"] == 201 or resp["code"] == 200:
self.box_version = resp["data"]["@version"]
self.logger.debug("Updated objects! new box version: {0}".format(self.box_version))
update_d.callback(self.box_version)
else:
self.logger.debug("Received unknown response code {0}".format(resp))
update_d.errback(resp)
else:
self.logger.debug("Received unknown or no response {0}".format(resp))
update_d.errback(resp)
def exception_cb(e, obj=obj, indx=indx):
self.logger.error("Exception in safe update: {0}".format(e))
if isinstance(e.value, urllib2.HTTPError): # handle a version incorrect error, and update the version
if e.value.code == 409: # 409 Obsolete
response = e.value.read()
json_response = json.loads(response)
self.box_version = json_response['@version']
# self.safe_update(indx, obj).addCallbacks(reupdate_cb, update_d.errback) # try updating again now the version is correct
indx.update(self.box_version, obj).addCallbacks(update_cb, exception_cb)
else:
self.logger.error("HTTPError updating INDX: {0}".format(e.value))
update_d.errback(e.value)
else:
self.logger.error("Error updating INDX: {0}".format(e.value))
update_d.errback(e.value)
indx.update(self.box_version, obj).addCallbacks(update_cb, exception_cb)
return update_d
def find_by_id(self, indx, oid) :
self.logger.debug("Searching object by id {0}".format(oid))
find_d = Deferred()
def find_cb(resp):
self.logger.debug("find_by_id: received response: {0}".format(resp))
if resp and "code" in resp and resp["code"] == 200 and "data" in resp:
resp_data = resp["data"]
if oid in resp_data:
self.logger.debug("Object {0} found!".format(oid))
find_d.callback(resp_data[oid])
else:
self.logger.debug("Object {0} not found!".format(oid))
find_d.callback(None)
else:
self.logger.debug("Response code is not 200! respose: {0}".format(resp))
find_d.errback(resp) # put a useful error here!
indx.query(json.dumps({"@id": oid})).addCallbacks(find_cb, find_d.errback)
return find_d
def find_create(self, indx, oid, attrs={}):
self.logger.debug("Searching for id: {0} and attrs: {1}".format(oid, attrs))
create_d = Deferred()
def find_cb(obj):
if obj is None:
self.logger.debug("Object {0} not found! Creating it.".format(oid))
attrs.update({"@id":oid})
self.safe_update(indx, attrs).addCallbacks(lambda x: create_d.callback(attrs), create_d.errback)
else:
create_d.callback(obj)
self.find_by_id(indx, oid).addCallbacks(find_cb, create_d.errback)
return create_d
def check_all_zero(self, data):
for key in data.keys():
if key.endswith('-intraday'):
for pt in data[key]["dataset"]:
if pt["value"] > 0 :
return False
return True
def parse_list(self, vlist):
out = [x['@value'] for x in vlist]
self.logger.debug("Parsed value list: {0}".format(out))
return out
def setUp(self):
self.fb = Fitbit('x', 'y')
self.fb_timeout = Fitbit('x', 'y', timeout=10)
self.test_url = 'invalid://do.not.connect'
