def upload(urls, source, token, retries=10):
unavailable = []
for url, response in MultiPut(urls, source, token).perform():
status = response.status
if str(status).startswith('2'):
yield response.read()
elif status == httplib.SERVICE_UNAVAILABLE:
if not retries:
raise CommError("Maximum number of retries reached", url)
unavailable.append(url)
else:
raise CommError("Upload failed: %s" % response.read(), url, status)
if unavailable:
for response in upload(unavailable, source, token,
retries=retries - 1):
yield response
def upload(urls, source, token, retries=10):
unavailable = []
for url, response in MultiPut(urls, source, token).perform():
status = response.status
if str(status).startswith('2'):
yield response.read()
elif status == httplib.SERVICE_UNAVAILABLE:
if not retries:
raise CommError("Maximum number of retries reached", url)
unavailable.append(url)
elif status == httplib.TEMPORARY_REDIRECT:
new_url = response.getheader('location')
for item in upload([new_url], source, token, retries):
yield item
else:
raise CommError("Upload failed: {0}".format(response.read()), url,
status)
if unavailable:
for response in upload(unavailable, source, token,
retries=retries - 1):
yield response
def request(method, url, data=None, headers={}, sleep=0):
scheme, netloc, path = urlsplit(urlresolve(url))
try:
conn = HTTPConnection(str(netloc))
conn.request(method, '/{0}'.format(path), body=data, headers=headers)
response = conn.getresponse()
status = response.status
errmsg = response.reason
except httplib.HTTPException as e:
status = None
errmsg = str(e) or repr(e)
except (httplib.socket.error, socket.error) as e:
status = None
errmsg = e if isinstance(e, basestring) else str(e) or repr(e)
if not status or isunavailable(status):
if sleep == 9:
raise CommError(errmsg, url, status)
time.sleep(random.randint(1, 2**sleep))
return request(method,
url,
data=data,
headers=headers,
sleep=sleep + 1)
elif isredirection(status):
loc = response.getheader('location')
return request(
method,
loc if loc.startswith('http:') else resolveuri(url, loc),
data=data,
headers=headers,
sleep=sleep)
elif not issuccessful(status):
raise CommError(response.read(), url, status)
return response
try:
conn = HTTPConnection(str(netloc))
conn.request(method, '/%s' % path, body=data, headers=headers)
response = conn.getresponse()
status = response.status
errmsg = response.reason
except httplib.HTTPException, e:
status = None
errmsg = str(e) or repr(e)
except (httplib.socket.error, socket.error), e:
status = None
errmsg = e if isinstance(e, basestring) else e[1]
if not status or isunavailable(status):
if sleep == 9:
raise CommError(errmsg, url, status)
time.sleep(random.randint(1, 2**sleep))
return request(method,
url,
data=data,
headers=headers,
sleep=sleep + 1)
elif isredirection(status):
loc = response.getheader('location')
return request(
method,
loc if loc.startswith('http:') else resolveuri(url, loc),
data=data,
headers=headers,
sleep=sleep)
elif not issuccessful(status):