def make_gitiles_json_call(url, n=1000):
"""Make a JSON call to gitiles and decode the result.
This makes a gitiles call with exponential backoff in the case of 429s. When
it gets a result, it decodes and returns the resulting object. If the
exponential backoff times out, it raises pipeline.PipelineUserError.
Args:
url (str): the url to query
n (int): the number of items to get (this is appended to the query string)
"""
full_url = url + '?format=json&n=%d' % n
backoff = 10
attempts = 4
for i in range(attempts):
logging.info('scanning %s', full_url)
result = urlfetch.fetch(full_url, deadline=180)
if result.status_code == 200:
# Gitiles serves JSONP, so we strip it out here.
assert result.content[0:5] == ')]}\'\n'
return json.loads(result.content[5:])
elif result.status_code == 404:
return {'status': 404}
elif result.status_code != 429:
raise pipeline.PipelineUserError('urlfetch returned %d' %
result.status_code)
sleep = backoff * (2**i)
logging.info('got 429, sleeping %d secs...', sleep)
time.sleep(sleep + random.random())
raise pipeline.PipelineUserError(
'urlfetch returned 429 after %d attempts, timing out!' % attempts)
def obtain_repo_scan_lock(project, repo, pipeline_id): # pragma: no cover
client = memcache.Client()
key = MEMCACHE_REPO_SCAN_LOCK % models.Repo.repo_id(project, repo)
logging.info('obtaining lock on %s' % key)
counter = client.gets(key)
if counter:
if counter['counter'] != 0 and counter['pipeline_id'] != pipeline_id:
logging.info('pipeline_id %s doesn\'t match %s' %
(counter['pipeline_id'], pipeline_id))
return False
while True:
new_counter = {
'pipeline_id': pipeline_id,
'counter': 1,
}
if client.cas(key, new_counter,
time=MEMCACHE_REPO_SCAN_EXPIRATION):
logging.info('cas succeeded')
return True
counter = client.gets(key)
if counter is not None:
raise pipeline.PipelineUserError('Uninitialized counter')
else:
new_counter = {
'pipeline_id': pipeline_id,
'counter': 1,
}
return client.add(MEMCACHE_REPO_SCAN_LOCK %
models.Repo.repo_id(project, repo),
new_counter,
time=MEMCACHE_REPO_SCAN_EXPIRATION)
