def update_moves(oh_member, moves_access_token, moves_data):
try:
start_date = get_start_date(moves_data, moves_access_token)
start_date = datetime.strptime(start_date, "%Y%m%d")
start_date_iso = start_date.isocalendar()[:2]
moves_data = remove_partial_data(moves_data, start_date_iso)
stop_date_iso = (datetime.utcnow() +
timedelta(days=7)).isocalendar()[:2]
while start_date_iso != stop_date_iso:
print('processing {}-{} for member {}'.format(
start_date_iso[0], start_date_iso[1], oh_member.oh_id))
query = MOVES_API_STORY + \
'/{0}-W{1}?trackPoints=true&access_token={2}'.format(
start_date_iso[0],
start_date_iso[1],
moves_access_token
)
response = rr.get(query, realms=['moves'])
moves_data += response.json()
start_date = start_date + timedelta(days=7)
start_date_iso = start_date.isocalendar()[:2]
print('successfully finished update for {}'.format(oh_member.oh_id))
moves_member = oh_member.datasourcemember
moves_member.last_updated = arrow.now().format()
moves_member.save()
except RequestsRespectfulRateLimitedError:
logger.debug('requeued processing for {} with 60 secs delay'.format(
oh_member.oh_id))
process_moves.apply_async(args=[oh_member.oh_id], countdown=61)
finally:
replace_moves(oh_member, moves_data)
def get_start_date(moves_data, moves_access_token):
if moves_data == []:
url = MOVES_API_BASE + "/user/profile?access_token={}".format(
moves_access_token)
response = rr.get(url, wait=True, realms=['moves'])
return response.json()['profile']['firstDate']
else:
return moves_data[-1]['date']
def get_start_date(twitter_data, twitter_access_token):
if not twitter_data:
url = TWITTER_API_BASE + "/user?access_token={}".format(
twitter_access_token)
response = rr.get(url, wait=True, realms=['twitter'])
reso = response.json()
print(reso)
return reso['created_at']
else:
return twitter_data[-1]['date']
def get_start_date(github_data, github_access_token):
if not github_data:
url = GITHUB_API_BASE + "/user?access_token={}".format(
github_access_token
)
response = rr.get(url, wait=True, realms=['github'])
reso = response.json()
print(reso)
return reso['created_at']
else:
return github_data[-1]['date']
def get_repo_commits_for_user(github_access_token, repo, username,
sync_after_date):
results = []
cnt = 0
url = GITHUB_REPO_COMMITS_ENDPOINT.format(repo, username)
# commits are fetched chronologically
latest_commit_date = None
reached_previous_data = False
while not reached_previous_data:
cnt += 1
response = rr.get(url,
headers=get_auth_header(github_access_token),
realms=['github'])
commits = json.loads(response.content)
if not isinstance(commits, list):
print(
"Got unexpected response from API - repo may be empty, will skip"
)
print(commits)
return [], None
if latest_commit_date is None and len(commits) > 0:
# github returns the data in descending chronological order
# date is in the format 2014-05-09T15:14:07Z
latest_commit_date = get_commit_date(commits[0])
if sync_after_date is not None:
for idx, commit in enumerate(commits):
commit_date = get_commit_date(commit)
# this may not handle rebases/history rewrites 100% correctly
# is good effort for gathering our commit histories <:o)
if commit_date <= sync_after_date:
print("reached existing data at idx {}".format(idx))
print("will only add {} to existing".format(commits[:idx]))
reached_previous_data = True
commits = commits[:idx]
break
results += commits
# if results['type'] == 'PushEvent'
# results[0]['payload']['commits'][0]['message']
next = response.links.get('next')
if not next:
break
else:
url = next['url']
#print("Called the api {} times".format(cnt))
return results, latest_commit_date
def get_rate_limit_remaining(github_access_token, type="core"):
""" Get the rate limit remaining and the reset time (in seconds from epoch format) as a tuple.
The query itself does not decrement the available
credit remaining.
https://developer.github.com/v3/rate_limit/
"""
auth_header = get_auth_header(github_access_token)
response = rr.get(GITHUB_RATE_LIMIT_ENDPOINT,
headers=auth_header,
realms=['github'])
rate_limit_info = json.loads(response.content)['resources'][type]
return rate_limit_info['remaining'], rate_limit_info['reset']
def get_user_repos(github_access_token):
results = []
cnt = 0
url = GITHUB_REPOS_ENDPOINT
while True:
cnt += 1
response = rr.get(url,
headers=get_auth_header(github_access_token),
realms=['github'])
results += json.loads(response.content)
next = response.links.get('next')
if not next:
break
else:
url = next['url']
#print("Called the api {} times".format(cnt))
return results
def make_request_respectful_get(url, realms, **kwargs):
r = rr.get(url=url, realms=realms, **kwargs)
logger.debug('Request completed. Response: {}'.format(r.text))
def get_user_info(github_access_token):
auth_header = get_auth_header(github_access_token)
response = rr.get(GITHUB_USER_INFO_ENDPOINT,
headers=auth_header,
realms=['github'])
return json.loads(response.content)