def upload_hash_content_to_blobstore(
generate_upload_url, data, hash_key, content):
"""Uploads the given hash contents directly to the blobstore via a generated
url.
Arguments:
generate_upload_url: The url to get the new upload url from.
data: extra POST data.
hash_key: sha1 of the uncompressed version of content.
content: The contents to upload. Must fit in memory for now.
"""
logging.debug('Generating url to directly upload file to blobstore')
assert isinstance(hash_key, str), hash_key
assert isinstance(content, str), (hash_key, content)
# TODO(maruel): Support large files. This would require streaming support.
content_type, body = encode_multipart_formdata(
data, [('content', hash_key, content)])
for attempt in xrange(run_isolated.URL_OPEN_MAX_ATTEMPTS):
# Retry HTTP 50x here.
upload_url = run_isolated.url_read(generate_upload_url, data=data)
if not upload_url:
raise run_isolated.MappingError(
'Unable to connect to server %s' % generate_upload_url)
# Do not retry this request on HTTP 50x. Regenerate an upload url each time
# since uploading "consumes" the upload url.
result = run_isolated.url_read(
upload_url, data=body, content_type=content_type, retry_50x=False)
if result is not None:
return result
if attempt != run_isolated.URL_OPEN_MAX_ATTEMPTS - 1:
run_isolated.HttpService.sleep_before_retry(attempt, None)
raise run_isolated.MappingError(
'Unable to connect to server %s' % generate_upload_url)
def url_read(url, **kwargs):
result = run_isolated.url_read(url, **kwargs)
if result is None:
# If we get no response from the server, assume it is down and raise an
# exception.
raise run_isolated.MappingError('Unable to connect to server %s' % url)
return result
def test_url_read(self):
# Successfully reads the data.
self.mock(run_isolated, 'url_open',
lambda url, **_kwargs: fake_http_response('111', url))
self.assertEqual(run_isolated.url_read('https://fake_url.com/test'), '111')
# Respects url_open connection errors.
self.mock(run_isolated, 'url_open', lambda _url, **_kwargs: None)
self.assertIsNone(run_isolated.url_read('https://fake_url.com/test'))
# Respects read timeout errors.
def timeouting_http_response(url):
def read_mock(_size=None):
raise run_isolated.TimeoutError()
stream = StringIO.StringIO('')
stream.headers = {'content-length': 0}
response = run_isolated.HttpResponse(stream, url)
response.read = read_mock
return response
self.mock(run_isolated, 'url_open',
lambda url, **_kwargs: timeouting_http_response(url))
self.assertIsNone(run_isolated.url_read('https://fake_url.com/test'))
