def test_retrieve_gs_url_ok(self):
"""Assert that URL retrieval works for GS entities."""
# get URL via preupload
content = pad_string('Lycidas')
collection = generate_collection([content])
preupload_status = self.call_api(
'preupload', self.message_to_dict(collection), 200)
message = preupload_status.json.get(u'items', [{}])[0]
# finalize GS upload
request = preupload_status_to_request(message, content)
embedded = validate(
request.upload_ticket, handlers_endpoints_v1.UPLOAD_MESSAGES[1])
self.mock(gcs, 'get_file_info', get_file_info_factory(content))
self.call_api(
'finalize_gs_upload', self.message_to_dict(request), 200)
# retrieve the upload URL
retrieve_request = handlers_endpoints_v1.RetrieveRequest(
digest=embedded['d'], namespace=handlers_endpoints_v1.Namespace())
retrieved_response = self.call_api(
'retrieve', self.message_to_dict(retrieve_request), 200)
retrieved = retrieved_response.json
self.assertNotEqual(message.get(u'gs_upload_url', ''), '')
self.assertNotEqual(retrieved.get(u'url', ''), '')
self.assertTrue(retrieved.get(u'url', '').startswith(self.store_prefix))
# clear the taskqueue
self.assertEqual(1, self.execute_tasks())
def _round_trip_gcs(self, namespace, algo, is_compressed):
"""Does a roundtrip of uploading large content and downloading back."""
content = 'foo' * 10000
data = zlib.compress(content) if is_compressed else content
# Lookup
collection = generate_collection(namespace, [content])
res = self.call_api('preupload', message_to_dict(collection), 200)
message = res.json.get(u'items', [{}])[0]
# Upload
# Simulate that the file is now on GCS.
self.mock(gcs, 'get_file_info', get_file_info_factory(data))
self.mock(gcs, 'read_file', lambda _bucket, _key: data)
request = handlers_endpoints_v1.FinalizeRequest(
upload_ticket=message['upload_ticket'])
self.call_api('finalize_gs_upload', message_to_dict(request), 200)
self.assertEqual(1, self.execute_tasks())
# Download
digest = algo(content).hexdigest()
request = handlers_endpoints_v1.RetrieveRequest(
digest=digest,
namespace=handlers_endpoints_v1.Namespace(namespace=namespace))
resp = self.call_api('retrieve', message_to_dict(request), 200)
prefix = (
'https://storage.googleapis.com/sample-app/%s/%s?GoogleAccessId=&'
'Expires=') % (namespace, digest)
self.assertTrue(resp.json['url'].startswith(prefix))
def _round_trip_inline(self, namespace, algo, is_compressed):
"""Does a roundtrip of uploading small content and downloading back."""
content = 'foo' * 10
data = zlib.compress(content) if is_compressed else content
# Lookup
collection = generate_collection(namespace, [content])
res = self.call_api('preupload', message_to_dict(collection), 200)
message = res.json.get(u'items', [{}])[0]
self.assertNotIn('gs_upload_url', message)
# Upload
request = handlers_endpoints_v1.StorageRequest(
upload_ticket=message['upload_ticket'], content=data)
self.call_api('store_inline', message_to_dict(request), 200)
# Download
request = handlers_endpoints_v1.RetrieveRequest(
digest=algo(content).hexdigest(),
namespace=handlers_endpoints_v1.Namespace(namespace=namespace))
resp = self.call_api('retrieve', message_to_dict(request), 200)
actual = base64.b64decode(resp.json['content'])
# It always returns the data as-is but base64 encoded.
self.assertEqual(actual, data)
if is_compressed:
self.assertEqual(content, zlib.decompress(actual))
else:
self.assertEqual(content, actual)
def test_pre_upload_invalid_namespace(self):
"""Assert that status 400 is returned when the namespace is invalid."""
bad_collection = handlers_endpoints_v1.DigestCollection(
namespace=handlers_endpoints_v1.Namespace(namespace='~tildewhatevs'))
bad_collection.items.append(generate_digest('pangolin'))
with self.call_should_fail('400'):
self.call_api(
'preupload', self.message_to_dict(bad_collection), 200)
def test_pre_upload_invalid_hash(self):
"""Assert that status 400 is returned when the digest is invalid."""
bad_collection = handlers_endpoints_v1.DigestCollection(
namespace=handlers_endpoints_v1.Namespace())
bad_digest = hash_content('some stuff')
bad_digest = 'g' + bad_digest[1:] # that's not hexadecimal!
bad_collection.items.append(
handlers_endpoints_v1.Digest(digest=bad_digest, size=10))
with self.call_should_fail('400'):
self.call_api(
'preupload', self.message_to_dict(bad_collection), 200)
def test_retrieve_memcache_ok(self):
"""Assert that content retrieval goes off swimmingly in the normal case."""
content = 'Grecian Urn'
request = self.store_request(content)
embedded = validate(
request.upload_ticket, handlers_endpoints_v1.UPLOAD_MESSAGES[0])
self.call_api(
'store_inline', self.message_to_dict(request), 200)
retrieve_request = handlers_endpoints_v1.RetrieveRequest(
digest=embedded['d'], namespace=handlers_endpoints_v1.Namespace())
response = self.call_api(
'retrieve', self.message_to_dict(retrieve_request), 200)
retrieved = response.json
self.assertEqual(content, base64.b64decode(retrieved.get(u'content', '')))
def test_retrieve_partial_bad_offset_fails(self):
"""Assert that retrieval fails with status 416 when offset is invalid."""
content = 'Of Man\'s first Disobedience, and the Fruit'
request = self.store_request(content)
embedded = validate(
request.upload_ticket, handlers_endpoints_v1.UPLOAD_MESSAGES[0])
self.call_api(
'store_inline', self.message_to_dict(request), 200)
requests = [handlers_endpoints_v1.RetrieveRequest(
digest=embedded['d'],
namespace=handlers_endpoints_v1.Namespace(),
offset=offset) for offset in [-1, len(content) + 1]]
for request in requests:
with self.call_should_fail('400'):
self.call_api('retrieve', self.message_to_dict(request), 200)
def test_retrieve_db_ok(self):
"""Assert that content retrieval works for non-memcached DB entities."""
content = 'Isabella, or the Pot of Basil'
request = self.store_request(content)
embedded = validate(
request.upload_ticket, handlers_endpoints_v1.UPLOAD_MESSAGES[0])
self.call_api(
'store_inline', self.message_to_dict(request), 200)
retrieve_request = handlers_endpoints_v1.RetrieveRequest(
digest=embedded['d'], namespace=handlers_endpoints_v1.Namespace())
memcache.flush_all()
response = self.call_api(
'retrieve', self.message_to_dict(retrieve_request), 200)
retrieved = response.json
self.assertEqual(content, base64.b64decode(retrieved.get(u'content', '')))
def test_check_existing_enqueues_tasks(self):
"""Assert that existent entities are enqueued."""
collection = handlers_endpoints_v1.DigestCollection(
namespace=handlers_endpoints_v1.Namespace())
collection.items.append(
generate_digest(collection.namespace.namespace, 'some content'))
key = model.get_entry_key(collection.namespace.namespace,
collection.items[0].digest)
# guarantee that one digest already exists in the datastore
model.new_content_entry(key).put()
self.call_api('preupload', message_to_dict(collection), 200)
# find enqueued tasks
self.assertEqual(1, self.execute_tasks())
def test_retrieve_partial_ok(self):
"""Assert that content retrieval works when a range is specified."""
content = 'Song of the Andoumboulou'
offset = 5
request = self.store_request(content)
embedded = validate(request.upload_ticket,
handlers_endpoints_v1.UPLOAD_MESSAGES[0])
self.call_api('store_inline', self.message_to_dict(request), 200)
retrieve_request = handlers_endpoints_v1.RetrieveRequest(
digest=embedded['d'],
namespace=handlers_endpoints_v1.Namespace(),
offset=offset) # TODO(cmassaro): determine where offsets come from
response = self.call_api('retrieve',
self.message_to_dict(retrieve_request), 200)
retrieved = response.json
self.assertEqual(content[offset:],
base64.b64decode(retrieved.get(u'content', '')))
def test_retrieve_not_found(self):
"""Assert that HTTP 404 response is served when content is absent."""
# get a valid digest
content = """\xe1\xbc\x84\xce\xbd\xce\xb4\xcf\x81\xce\xb1
\xce\xbc\xce\xbf\xce\xb9
\xe1\xbc\x94\xce\xbd\xce\xbd\xce\xb5\xcf\x80\xce\xb5"""
collection = generate_collection([content])
preupload_status = self.call_api(
'preupload', self.message_to_dict(collection), 200)
message = preupload_status.json.get(u'items', [{}])[0]
# get the digest
request = preupload_status_to_request(message, content)
embedded = validate(
request.upload_ticket, handlers_endpoints_v1.UPLOAD_MESSAGES[0])
# don't upload data; try to retrieve
retrieve_request = handlers_endpoints_v1.RetrieveRequest(
digest=embedded['d'], namespace=handlers_endpoints_v1.Namespace())
with self.call_should_fail('404'):
self.call_api(
'retrieve', self.message_to_dict(retrieve_request), 200)
def generate_collection(contents, namespace=None):
if namespace is None:
namespace = handlers_endpoints_v1.Namespace()
return handlers_endpoints_v1.DigestCollection(
namespace=namespace,
items=[generate_digest(content) for content in contents])
def generate_collection(namespace, contents):
return handlers_endpoints_v1.DigestCollection(
namespace=handlers_endpoints_v1.Namespace(namespace=namespace),
items=[generate_digest(namespace, content) for content in contents])
