def regular_import(self):
"""
Import new checkins, beginning from the last checkin found in the local database
:return:
"""
# todo: check request limit and handle exhausting
while True:
#+1 to get new checkins, not the same we have already
latest_local_checkin = self._get_latest_checkin()
latest_local_checkin_time = latest_local_checkin['createdAt'] + 1
logger.debug("latest local stored checkin entry is from %s" % datetime.datetime.fromtimestamp(
latest_local_checkin['createdAt']).isoformat())
res = self.client.users.checkins(
params={'limit': 250, 'sort': 'oldestfirst', 'afterTimestamp': latest_local_checkin_time})
if len(res['checkins']['items']) == 0:
logger.debug("no further checkins to import - finishing")
break
start_time = datetime.datetime.fromtimestamp(latest_local_checkin['createdAt']).isoformat()
end_time = datetime.datetime.fromtimestamp(res['checkins']['items'][-1]['createdAt']).isoformat()
logger.debug("store checkins from %s to %s" % (start_time, end_time))
store(self.database, res['checkins']['items'])
return True
def __init__(self):
try:
Module.__init__(self, "moves")
except:
return
self.docs = [x for x in self.get_data()]
logger.debug("read %d new entries from the Moves API" % len(self.docs))
store(self.database, self.docs)
def __init__(self):
try:
Module.__init__(self, "moves")
except:
return
self.docs = [x for x in self.get_data()]
logger.debug("read %d new entries from the Moves API" % len(self.docs))
store(self.database, self.docs)
def __init__(self):
try:
Module.__init__(self, "xbox")
except:
return
self.user_id = self.get_user_id()
self.docs = [x for x in self.get_achievements()]
logger.debug("read %d achievements from Xbox live" % len(self.docs))
store(self.database, self.docs)
def __init__(self):
try:
Module.__init__(self, "imessage")
except:
return
last_id = self.get_last_id()
self.docs = [x for x in self.get_docs(last_id)]
logger.debug("read %d new entries from iMessage's local database" % len(self.docs))
store(self.database, self.docs)
def __init__(self):
try:
Module.__init__(self, "xbox")
except:
return
self.user_id = self.get_user_id()
self.docs = [x for x in self.get_achievements()]
logger.debug("read %d achievements from Xbox live" % len(self.docs))
store(self.database, self.docs)
def __init__(self):
try:
Module.__init__(self, "twitter")
except:
return
self.screen_name = None
self.api = self.get_api()
self.docs = self.get_tweets()
logger.debug("read the most recent %d tweets" % len(self.docs))
store(self.database, self.docs)
def __init__(self):
try:
Module.__init__(self, "skype")
except:
return
last_id = self.get_last_id()
self.docs = [x for x in self.get_docs(last_id)]
logger.debug("read %d new entries from Skype's local database" %
len(self.docs))
store(self.database, self.docs)
def _run_fetch_store(self, param, opt_params):
"""
fetches available tracks from lastfm page by page and stores inside the database
:param params: parameter to narrow the API result
:return: True
"""
next_page = 1
while True:
# build parameter set to get step by step all data
opt_params['extended'] = 1
opt_params['page'] = next_page
opt_params['limit'] = 200
data = self.api_call(param, opt_params)
if 'recenttracks' in data:
if '@attr' in data['recenttracks']:
attr = data['recenttracks']['@attr']
else:
attr = data['recenttracks']
else:
logger.debug("Finished import to early?")
break
# stop import process if we do not have any import!
if int(attr['total']) == 0:
break
# when we get a single track it is not a list so wie have fix this manually
tracks = data['recenttracks']['track']
if not isinstance(tracks, list):
tracks = list(data['recenttracks']['track'])
store(self.database, tracks)
logger.debug("Stored page %s with %s tracks" %
(attr['page'], len(data['recenttracks']['track'])))
# calculate next iteration
cur_page = int(attr['page'])
if cur_page < int(attr['totalPages']):
next_page = cur_page + 1
else:
logger.debug("All tracks fetched.")
break
return True
def _run_fetch_store(self, param, opt_params):
"""
fetches available tracks from lastfm page by page and stores inside the database
:param params: parameter to narrow the API result
:return: True
"""
next_page = 1
while True:
# build parameter set to get step by step all data
opt_params['extended'] = 1
opt_params['page'] = next_page
opt_params['limit'] = 200
data = self.api_call(param, opt_params)
if 'recenttracks' in data:
if '@attr' in data['recenttracks']:
attr = data['recenttracks']['@attr']
else:
attr = data['recenttracks']
else:
logger.debug("Finished import to early?")
break
# stop import process if we do not have any import!
if int(attr['total']) == 0:
break
# when we get a single track it is not a list so wie have fix this manually
tracks = data['recenttracks']['track']
if not isinstance(tracks, list):
tracks = list(data['recenttracks']['track'])
store(self.database, tracks)
logger.debug("Stored page %s with %s tracks" % (attr['page'], len(data['recenttracks']['track'])))
# calculate next iteration
cur_page = int(attr['page'])
if cur_page < int(attr['totalPages']):
next_page = cur_page + 1
else:
logger.debug("All tracks fetched.")
break
return True
def initial_import(self):
"""
Import all checkins at once if no import before were executed
:return: Boolean
"""
if not self._get_latest_checkin():
logger.debug("no local stored data found - inital import needed")
# import all checkins because we imported nothing before
checkins = [checkin for checkin in self.client.users.all_checkins()]
store(self.database, checkins)
return True
return False
def initial_import(self):
"""
Import all checkins at once if no import before were executed
:return: Boolean
"""
if not self._get_latest_checkin():
logger.debug("no local stored data found - inital import needed")
# import all checkins because we imported nothing before
checkins = [
checkin for checkin in self.client.users.all_checkins()
]
store(self.database, checkins)
return True
return False
def regular_import(self):
"""
Import new checkins, beginning from the last checkin found in the local database
:return:
"""
# todo: check request limit and handle exhausting
while True:
#+1 to get new checkins, not the same we have already
latest_local_checkin = self._get_latest_checkin()
latest_local_checkin_time = latest_local_checkin['createdAt'] + 1
logger.debug("latest local stored checkin entry is from %s" %
datetime.datetime.fromtimestamp(
latest_local_checkin['createdAt']).isoformat())
res = self.client.users.checkins(
params={
'limit': 250,
'sort': 'oldestfirst',
'afterTimestamp': latest_local_checkin_time
})
if len(res['checkins']['items']) == 0:
logger.debug("no further checkins to import - finishing")
break
start_time = datetime.datetime.fromtimestamp(
latest_local_checkin['createdAt']).isoformat()
end_time = datetime.datetime.fromtimestamp(
res['checkins']['items'][-1]['createdAt']).isoformat()
logger.debug("store checkins from %s to %s" %
(start_time, end_time))
store(self.database, res['checkins']['items'])
return True
