def __init__(self):
log_handler = logging.FileHandler("nikeplus_harvester.log", "a")
log_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(name)s\t%(levelname)s\t%(asctime)s\t%(message)s')
log_handler.setFormatter(formatter)
self.logger = logging.getLogger()
self.logger.setLevel(logging.DEBUG)
for handler in self.logger.handlers:
self.logger.removeHandler(handler)
self.logger.addHandler(log_handler)
data_root = keyring.util.platform_.data_root()
if not os.path.exists(data_root):
os.mkdir(data_root)
keyring.set_keyring(PlaintextKeyring())
self.parser = argparse.ArgumentParser(prog="run")
self.parser.add_argument('--config', help="Set config (input requires JSON) and exit.")
self.parser.add_argument('--get-config', action="store_true", help="Output current config as JSON and exit.")
self.parser.add_argument('--server', help="The server URL to connect to.")
self.nike = NikePlus()
self.box_version = 0
self.harvester_id = "nikeplus_harvester"
self.fuel_ts_id = "nikeplus_fuel_ts"
self.calories_ts_id = "nikeplus_calories_ts"
self.steps_ts_id = "nikeplus_steps_ts"
self.stars_ts_id = "nikeplus_stars_ts"
self.ts_ids_by_type = {'FUEL':self.fuel_ts_id, 'CALORIES':self.calories_ts_id, 'STEPS':self.steps_ts_id, 'STARS':self.stars_ts_id }
self.ts_count = 0
self.ts_error = None
self.rdf_types = {
'FUEL':'http://sociam.org/ontology/health/NikeFuel',
'CALORIES':'http://sociam.org/ontology/health/CaloriesBurned',
'STEPS':'http://sociam.org/ontology/health/StepCount',
'STARS':'http://sociam.org/ontology/health/Intensity'
}
self.zeros = self.today()
self.new_zeros = self.today()
self.retrieved = self.today()
self.config_box = None
self.config_indx_user = None
self.config_indx_pass = None
self.config_max_date = datetime(2013, 7, 1)
class NikeHarvester:
def __init__(self):
log_handler = logging.FileHandler("nikeplus_harvester.log", "a")
log_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(name)s\t%(levelname)s\t%(asctime)s\t%(message)s')
log_handler.setFormatter(formatter)
self.logger = logging.getLogger()
self.logger.setLevel(logging.DEBUG)
for handler in self.logger.handlers:
self.logger.removeHandler(handler)
self.logger.addHandler(log_handler)
data_root = keyring.util.platform_.data_root()
if not os.path.exists(data_root):
os.mkdir(data_root)
keyring.set_keyring(PlaintextKeyring())
self.parser = argparse.ArgumentParser(prog="run")
self.parser.add_argument('--config', help="Set config (input requires JSON) and exit.")
self.parser.add_argument('--get-config', action="store_true", help="Output current config as JSON and exit.")
self.parser.add_argument('--server', help="The server URL to connect to.")
self.nike = NikePlus()
self.box_version = 0
self.harvester_id = "nikeplus_harvester"
self.fuel_ts_id = "nikeplus_fuel_ts"
self.calories_ts_id = "nikeplus_calories_ts"
self.steps_ts_id = "nikeplus_steps_ts"
self.stars_ts_id = "nikeplus_stars_ts"
self.ts_ids_by_type = {'FUEL':self.fuel_ts_id, 'CALORIES':self.calories_ts_id, 'STEPS':self.steps_ts_id, 'STARS':self.stars_ts_id }
self.ts_count = 0
self.ts_error = None
self.rdf_types = {
'FUEL':'http://sociam.org/ontology/health/NikeFuel',
'CALORIES':'http://sociam.org/ontology/health/CaloriesBurned',
'STEPS':'http://sociam.org/ontology/health/StepCount',
'STARS':'http://sociam.org/ontology/health/Intensity'
}
self.zeros = self.today()
self.new_zeros = self.today()
self.retrieved = self.today()
self.config_box = None
self.config_indx_user = None
self.config_indx_pass = None
self.config_max_date = datetime(2013, 7, 1)
def set_config(self, args):
stored_config_harvester = keyring.get_password("INDX", "INDX_Nike_Harvester")
if stored_config_harvester is not None:
stored_config_harvester = json.loads(stored_config_harvester)
stored_config_nike = keyring.get_password("Nike.com", "Nike+")
if stored_config_nike is not None:
stored_config_nike = json.loads(stored_config_nike)
received_config = json.loads(args['config'])
if (type(received_config) != dict):
received_config = json.loads(received_config)
self.logger.debug("Received config: {0}".format(received_config))
if 'nike' in received_config:
nike_config = received_config['nike']
if nike_config and ('user' in nike_config) and ('password' in nike_config):
self.logger.debug("Received user: {0} and password: {1}".format(nike_config['user'], nike_config['password']))
try:
self.nike.login(nike_config['user'], nike_config['password'])
self.logger.debug("Logged in with username {0} and password {1}".format(nike_config['user'], nike_config['password']))
token = self.nike.get_token()
self.logger.debug("Got token {0}".format(token))
if token:
nike_config['token'] = token
if 'error' in nike_config:
del nike_config['error']
except Exception as exc:
self.logger.error("Could not authorise to Nike, with username {0} and password {1}, error: {2}".format(nike_config['user'], nike_config['password'], exc))
nike_config['error'] = str(exc)
del nike_config['password']
keyring.set_password("Nike.com", "Nike+", json.dumps(nike_config))
if 'harvester' in received_config:
harvester_config = received_config['harvester']
if harvester_config != stored_config_harvester:
keyring.set_password("INDX", "INDX_Nike_Harvester", json.dumps(harvester_config))
def get_config(self, args):
stored_config_harvester = keyring.get_password("INDX", "INDX_Nike_Harvester")
stored_config_nike = keyring.get_password("Nike.com", "Nike+")
self.logger.debug("Loaded harvester config from keyring: {0}".format(stored_config_harvester))
self.logger.debug("Loaded nike config from keyring: {0}".format(stored_config_nike))
if stored_config_nike is not None :
if stored_config_harvester is None :
return json.dumps({"nike":json.loads(stored_config_nike)})
return json.dumps({"nike":json.loads(stored_config_nike), "harvester":json.loads(stored_config_harvester)})
if stored_config_harvester is not None :
return json.dumps({"harvester":json.loads(stored_config_harvester)})
return json.dumps({})
def load_configuration(self):
stored_config_harvester = keyring.get_password("INDX", "INDX_Nike_Harvester")
self.logger.debug("Loaded harvester config from keyring: {0}".format(stored_config_harvester))
if stored_config_harvester is None :
self.logger.error("Harvester not configured. Please configure before use.")
return
if (type(stored_config_harvester) != dict):
stored_config_harvester = json.loads(stored_config_harvester)
stored_config_nike = keyring.get_password("Nike.com", "Nike+")
self.logger.debug("Loaded nike config from keyring: {0}".format(stored_config_nike))
if stored_config_nike is None :
self.logger.error("No credentials for Nike.com. Please configure before use.")
return
else :
if (type(stored_config_nike) != dict):
stored_config_nike = json.loads(stored_config_nike)
if ('password' in stored_config_nike) and ('user' in stored_config_nike):
self.config_nike_user = stored_config_nike['user']
self.config_nike_pass = stored_config_nike['password']
if not self.try_nike_login():
return
self.config_box = stored_config_harvester['box']
self.config_indx_user = stored_config_harvester['user']
self.config_indx_pass = stored_config_harvester['password']
def try_nike_login(self):
try:
self.nike.login(self.config_nike_user, self.config_nike_pass)
self.logger.debug("Logged in with username {0} and password {1}".format(self.config_nike_user, self.config_nike_pass))
token = self.nike.get_token()
if token :
self.logger.debug("Got token {0}".format(token))
return True
else :
return False
except Exception as exc:
self.logger.error("Could not authorise to nike, error: {0}".format(exc))
return False
def run(self):
args = vars(self.parser.parse_args())
self.logger.debug("Received arguments: {0}".format(args))
if args['config']:
self.set_config(args)
elif args['get_config']:
print self.get_config(args)
else:
self.logger.debug("Starting the harvester. ")
self.work(args['server'])
reactor.run()
def yesterday(self):
return datetime.combine((datetime.now()+timedelta(days=-1)).date(), time(00,00,00))
def today(self):
return datetime.combine(datetime.now().date(), time(00,00,00))
def get_indx(self, server_url):
indx_d = Deferred()
def authed_cb():
def token_cb(token):
indx = IndxClient(server_url, self.config_box, "INDX_Nike_Harvester", token = token, client = authclient.client)
indx_d.callback(indx)
authclient.get_token(self.config_box).addCallbacks(token_cb, indx_d.errback)
authclient = IndxClientAuth(server_url, "INDX_Nike_Harvester")
authclient.auth_plain(self.config_indx_user, self.config_indx_pass).addCallbacks(lambda response: authed_cb(), indx_d.errback)
return indx_d
def work(self, server_url):
self.load_configuration()
def indx_cb(indx):
self.logger.debug("Created INDXClient.")
prep_d = Deferred()
if self.try_nike_login():
self.logger.debug("Successfully logged in to Nike.com.")
else:
self.logger.error("Failed to log in to Nike.com.")
prep_d.errback("Failed to log in to Nike.com.")
def objects_cb(harvester, indx=indx):
self.logger.debug("Found or created all 4 time series.")
self.logger.debug("Found or created harvester. {0}".format(harvester))
def wait(x):
self.logger.debug("Harvested! Suspending execution for 6 hours at {0}.".format(datetime.now().isoformat()))
sleep(21600)
prep_d.callback(None)
if harvester:
if "zeros_from" in harvester :
self.zeros = datetime.strptime(harvester["zeros_from"][0]["@value"], "%Y-%m-%dT%H:%M:%S")
if "retrieved_to" in harvester :
self.retrieved = datetime.strptime(harvester["retrieved_to"][0]["@value"], "%Y-%m-%dT%H:%M:%S")
self.harvest(indx, harvester).addCallbacks(wait, prep_d.errback)
self.find_create(indx, self.steps_ts_id, {"http://www.w3.org/2000/01/rdf-schema#label":"Nike+ Steps Time Series", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type":"http://purl.org/linked-data/cube#Dataset"}).addCallbacks(
lambda x: self.find_create(indx, self.calories_ts_id, {"http://www.w3.org/2000/01/rdf-schema#label":"Nike+ Calories Time Series", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type":"http://purl.org/linked-data/cube#Dataset"}), prep_d.errback).addCallbacks(
lambda x: self.find_create(indx, self.stars_ts_id, {"http://www.w3.org/2000/01/rdf-schema#label":"Nike+ Stars Time Series", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type":"http://purl.org/linked-data/cube#Dataset"}), prep_d.errback).addCallbacks(
lambda x: self.find_create(indx, self.fuel_ts_id, {"http://www.w3.org/2000/01/rdf-schema#label":"Nike+ Fuel Time Series", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type":"http://purl.org/linked-data/cube#Dataset"}), prep_d.errback).addCallbacks(
lambda x: self.find_create(indx, self.harvester_id, {"http://www.w3.org/2000/01/rdf-schema#label":"INDX Nike+ Harvester extra info"}), prep_d.errback).addCallbacks(objects_cb, prep_d.errback)
return prep_d
def indx_fail_cb(f):
self.logger.error("Error getting indx! Exiting... {0}".format(f))
sys.exit(1)
def loop_fail_cb(f):
self.logger.error("Error in indx_cb! Exiting... {0}".format(f))
sys.exit(1)
def loop(x=None):
self.get_indx(server_url).addCallbacks(indx_cb, indx_fail_cb).addCallbacks(loop, loop_fail_cb)
threads.deferToThread(loop)
def harvest(self, indx, harvester):
self.logger.debug("Starting to harvest!")
harvest_d = Deferred()
def step_day(x, day, indx=indx, harvester=harvester):
self.logger.debug("Current day: {0}".format(day))
if day < self.config_max_date :
self.logger.debug("Finished harvesting round. Saving harvester data .. ")
harvester["retrieved_to"] = self.retrieved.isoformat()
harvester["zeros_from"] = self.new_zeros.isoformat()
self.safe_update(indx, harvester).addCallbacks(harvest_d.callback, harvest_d.errback)
else:
prev_day = day + timedelta(days=-1)
next_day = day + timedelta(days=+1)
if self.retrieved <= day and next_day < self.zeros :
self.step_day(None, prev_day, indx, harvester)
else:
self.get_day_points(indx, day).addCallbacks(step_day, harvest_d.errback, callbackArgs=[prev_day, indx])
self.logger.debug("Retrieved to : {0}, Zeros from : {1}".format(self.retrieved.isoformat(), self.zeros.isoformat()))
step_day(None, self.yesterday(), indx)
return harvest_d
def get_day_points(self, indx, day):
self.logger.debug("Getting data for {0}".format(day.isoformat()))
process_d = Deferred()
activities = self.nike.get_day_activities(day.strftime("%Y-%m-%d"))
next_day = day + timedelta(days=+1)
points = []
if len(activities) == 0 :
self.logger.debug("No data for activities, nothing saved to indx.")
if next_day == self.new_zeros :
self.new_zeros = day
else :
# best case, just take the start date and count from there ..
self.logger.debug("Single activity. Getting details.")
activity_id = activities[0]['activityId']
self.logger.debug("Got activity id: {0}".format(activity_id))
activity = self.nike.get_activity_detail(activity_id)
self.logger.debug("Got activity details: {0}".format(activity))
points = self.prepare_points(activity)
if next_day == self.retrieved :
self.retrieved = day
if len(points) > 0 :
self.safe_update(indx, points).addCallbacks(process_d.callback, process_d.errback)
else :
process_d.callback(0)
return process_d
def prepare_points(self, activity):
self.logger.debug("Started creating data points.")
data_points = []
metrics = activity['metrics']
start = datetime.strptime(activity['startTime'],"%Y-%m-%dT%H:%M:%SZ")
for metric in metrics :
values = metric['values']
mtype = metric['metricType']
minute = 0
for value in values :
interval_start = start + timedelta(minutes=minute)
interval_end = interval_start+timedelta(minutes=1)
data_point = { "@id": "nikeplus_dp_{0}".format(uuid.uuid4()),
"tstart": interval_start.isoformat(),
"tend": interval_end.isoformat(),
"value": value,
"timeseries": { "@id": self.ts_ids_by_type[mtype] },
"http://www.w3.org/1999/02/22-rdf-syntax-ns#type": { "@id": self.rdf_types[mtype] } }
data_points.append(data_point)
# self.logger.debug("Created data point ({0}): {1}".format(minute, data_point))
minute = minute+1
self.logger.debug("Finished creating data points.")
return data_points
def safe_update(self, indx, obj) :
self.logger.debug("Updating object {0} at box version {1}".format(obj, self.box_version))
update_d = Deferred()
def update_cb(resp):
self.logger.debug("safe_update: received response: {0}".format(resp))
if resp and "code" in resp and "data" in resp:
if resp["code"] == 201 or resp["code"] == 200:
self.box_version = resp["data"]["@version"]
self.logger.debug("Updated objects! new box version: {0}".format(self.box_version))
update_d.callback(self.box_version)
else:
self.logger.debug("Received unknown response code {0}".format(resp))
update_d.errback(resp)
else:
self.logger.debug("Received unknown or no response {0}".format(resp))
update_d.errback(resp)
def exception_cb(e, obj=obj, indx=indx):
self.logger.error("EXCEPTION ? {0}".format(e))
if isinstance(e.value, urllib2.HTTPError): # handle a version incorrect error, and update the version
if e.value.code == 409: # 409 Obsolete
response = e.value.read()
json_response = json.loads(response)
self.box_version = json_response['@version']
# self.safe_update(indx, obj).addCallbacks(reupdate_cb, update_d.errback) # try updating again now the version is correct
indx.update(self.box_version, obj).addCallbacks(update_cb, exception_cb)
else:
self.logger.error("HTTPError updating INDX: {0}".format(e.value))
update_d.errback(e.value)
else:
self.logger.error("Error updating INDX: {0}".format(e.value))
update_d.errback(e.value)
indx.update(self.box_version, obj).addCallbacks(update_cb, exception_cb)
return update_d
def find_by_id(self, indx, oid) :
self.logger.debug("Searching object by id {0}".format(oid))
find_d = Deferred()
def find_cb(resp):
self.logger.debug("find_by_id: received response: {0}".format(resp))
if resp and "code" in resp and resp["code"] == 200 and "data" in resp:
resp_data = resp["data"]
if oid in resp_data:
self.logger.debug("Object {0} found!".format(oid))
find_d.callback(resp_data[oid])
else:
self.logger.debug("Object {0} not found!".format(oid))
find_d.callback(None)
else:
self.logger.debug("Response code is not 200! respose: {0}".format(resp))
find_d.errback(resp) # put a useful error here!
indx.query(json.dumps({"@id": oid})).addCallbacks(find_cb, find_d.errback)
return find_d
def find_create(self, indx, oid, attrs={}):
self.logger.debug("Searching for id: {0} and attrs: {1}".format(oid, attrs))
create_d = Deferred()
def find_cb(obj):
if obj is None:
self.logger.debug("Object {0} not found! Creating it.".format(oid))
attrs.update({"@id":oid})
self.safe_update(indx, attrs).addCallbacks(lambda x: create_d.callback(attrs), create_d.errback)
else:
create_d.callback(obj)
self.find_by_id(indx, oid).addCallbacks(find_cb, create_d.errback)
return create_d
