def test_retrieve_gs_url_ok(self): """Assert that URL retrieval works for GS entities.""" # get URL via preupload content = pad_string('Lycidas') collection = generate_collection([content]) preupload_status = self.call_api( 'preupload', self.message_to_dict(collection), 200) message = preupload_status.json.get(u'items', [{}])[0] # finalize GS upload request = preupload_status_to_request(message, content) embedded = validate( request.upload_ticket, handlers_endpoints_v1.UPLOAD_MESSAGES[1]) self.mock(gcs, 'get_file_info', get_file_info_factory(content)) self.call_api( 'finalize_gs_upload', self.message_to_dict(request), 200) # retrieve the upload URL retrieve_request = handlers_endpoints_v1.RetrieveRequest( digest=embedded['d'], namespace=handlers_endpoints_v1.Namespace()) retrieved_response = self.call_api( 'retrieve', self.message_to_dict(retrieve_request), 200) retrieved = retrieved_response.json self.assertNotEqual(message.get(u'gs_upload_url', ''), '') self.assertNotEqual(retrieved.get(u'url', ''), '') self.assertTrue(retrieved.get(u'url', '').startswith(self.store_prefix)) # clear the taskqueue self.assertEqual(1, self.execute_tasks())
def _round_trip_gcs(self, namespace, algo, is_compressed): """Does a roundtrip of uploading large content and downloading back.""" content = 'foo' * 10000 data = zlib.compress(content) if is_compressed else content # Lookup collection = generate_collection(namespace, [content]) res = self.call_api('preupload', message_to_dict(collection), 200) message = res.json.get(u'items', [{}])[0] # Upload # Simulate that the file is now on GCS. self.mock(gcs, 'get_file_info', get_file_info_factory(data)) self.mock(gcs, 'read_file', lambda _bucket, _key: data) request = handlers_endpoints_v1.FinalizeRequest( upload_ticket=message['upload_ticket']) self.call_api('finalize_gs_upload', message_to_dict(request), 200) self.assertEqual(1, self.execute_tasks()) # Download digest = algo(content).hexdigest() request = handlers_endpoints_v1.RetrieveRequest( digest=digest, namespace=handlers_endpoints_v1.Namespace(namespace=namespace)) resp = self.call_api('retrieve', message_to_dict(request), 200) prefix = ( 'https://storage.googleapis.com/sample-app/%s/%s?GoogleAccessId=&' 'Expires=') % (namespace, digest) self.assertTrue(resp.json['url'].startswith(prefix))
def _round_trip_inline(self, namespace, algo, is_compressed): """Does a roundtrip of uploading small content and downloading back.""" content = 'foo' * 10 data = zlib.compress(content) if is_compressed else content # Lookup collection = generate_collection(namespace, [content]) res = self.call_api('preupload', message_to_dict(collection), 200) message = res.json.get(u'items', [{}])[0] self.assertNotIn('gs_upload_url', message) # Upload request = handlers_endpoints_v1.StorageRequest( upload_ticket=message['upload_ticket'], content=data) self.call_api('store_inline', message_to_dict(request), 200) # Download request = handlers_endpoints_v1.RetrieveRequest( digest=algo(content).hexdigest(), namespace=handlers_endpoints_v1.Namespace(namespace=namespace)) resp = self.call_api('retrieve', message_to_dict(request), 200) actual = base64.b64decode(resp.json['content']) # It always returns the data as-is but base64 encoded. self.assertEqual(actual, data) if is_compressed: self.assertEqual(content, zlib.decompress(actual)) else: self.assertEqual(content, actual)
def test_pre_upload_invalid_namespace(self): """Assert that status 400 is returned when the namespace is invalid.""" bad_collection = handlers_endpoints_v1.DigestCollection( namespace=handlers_endpoints_v1.Namespace(namespace='~tildewhatevs')) bad_collection.items.append(generate_digest('pangolin')) with self.call_should_fail('400'): self.call_api( 'preupload', self.message_to_dict(bad_collection), 200)
def test_pre_upload_invalid_hash(self): """Assert that status 400 is returned when the digest is invalid.""" bad_collection = handlers_endpoints_v1.DigestCollection( namespace=handlers_endpoints_v1.Namespace()) bad_digest = hash_content('some stuff') bad_digest = 'g' + bad_digest[1:] # that's not hexadecimal! bad_collection.items.append( handlers_endpoints_v1.Digest(digest=bad_digest, size=10)) with self.call_should_fail('400'): self.call_api( 'preupload', self.message_to_dict(bad_collection), 200)
def test_retrieve_memcache_ok(self): """Assert that content retrieval goes off swimmingly in the normal case.""" content = 'Grecian Urn' request = self.store_request(content) embedded = validate( request.upload_ticket, handlers_endpoints_v1.UPLOAD_MESSAGES[0]) self.call_api( 'store_inline', self.message_to_dict(request), 200) retrieve_request = handlers_endpoints_v1.RetrieveRequest( digest=embedded['d'], namespace=handlers_endpoints_v1.Namespace()) response = self.call_api( 'retrieve', self.message_to_dict(retrieve_request), 200) retrieved = response.json self.assertEqual(content, base64.b64decode(retrieved.get(u'content', '')))
def test_retrieve_partial_bad_offset_fails(self): """Assert that retrieval fails with status 416 when offset is invalid.""" content = 'Of Man\'s first Disobedience, and the Fruit' request = self.store_request(content) embedded = validate( request.upload_ticket, handlers_endpoints_v1.UPLOAD_MESSAGES[0]) self.call_api( 'store_inline', self.message_to_dict(request), 200) requests = [handlers_endpoints_v1.RetrieveRequest( digest=embedded['d'], namespace=handlers_endpoints_v1.Namespace(), offset=offset) for offset in [-1, len(content) + 1]] for request in requests: with self.call_should_fail('400'): self.call_api('retrieve', self.message_to_dict(request), 200)
def test_retrieve_db_ok(self): """Assert that content retrieval works for non-memcached DB entities.""" content = 'Isabella, or the Pot of Basil' request = self.store_request(content) embedded = validate( request.upload_ticket, handlers_endpoints_v1.UPLOAD_MESSAGES[0]) self.call_api( 'store_inline', self.message_to_dict(request), 200) retrieve_request = handlers_endpoints_v1.RetrieveRequest( digest=embedded['d'], namespace=handlers_endpoints_v1.Namespace()) memcache.flush_all() response = self.call_api( 'retrieve', self.message_to_dict(retrieve_request), 200) retrieved = response.json self.assertEqual(content, base64.b64decode(retrieved.get(u'content', '')))
def test_check_existing_enqueues_tasks(self): """Assert that existent entities are enqueued.""" collection = handlers_endpoints_v1.DigestCollection( namespace=handlers_endpoints_v1.Namespace()) collection.items.append( generate_digest(collection.namespace.namespace, 'some content')) key = model.get_entry_key(collection.namespace.namespace, collection.items[0].digest) # guarantee that one digest already exists in the datastore model.new_content_entry(key).put() self.call_api('preupload', message_to_dict(collection), 200) # find enqueued tasks self.assertEqual(1, self.execute_tasks())
def test_retrieve_partial_ok(self): """Assert that content retrieval works when a range is specified.""" content = 'Song of the Andoumboulou' offset = 5 request = self.store_request(content) embedded = validate(request.upload_ticket, handlers_endpoints_v1.UPLOAD_MESSAGES[0]) self.call_api('store_inline', self.message_to_dict(request), 200) retrieve_request = handlers_endpoints_v1.RetrieveRequest( digest=embedded['d'], namespace=handlers_endpoints_v1.Namespace(), offset=offset) # TODO(cmassaro): determine where offsets come from response = self.call_api('retrieve', self.message_to_dict(retrieve_request), 200) retrieved = response.json self.assertEqual(content[offset:], base64.b64decode(retrieved.get(u'content', '')))
def test_retrieve_not_found(self): """Assert that HTTP 404 response is served when content is absent.""" # get a valid digest content = """\xe1\xbc\x84\xce\xbd\xce\xb4\xcf\x81\xce\xb1 \xce\xbc\xce\xbf\xce\xb9 \xe1\xbc\x94\xce\xbd\xce\xbd\xce\xb5\xcf\x80\xce\xb5""" collection = generate_collection([content]) preupload_status = self.call_api( 'preupload', self.message_to_dict(collection), 200) message = preupload_status.json.get(u'items', [{}])[0] # get the digest request = preupload_status_to_request(message, content) embedded = validate( request.upload_ticket, handlers_endpoints_v1.UPLOAD_MESSAGES[0]) # don't upload data; try to retrieve retrieve_request = handlers_endpoints_v1.RetrieveRequest( digest=embedded['d'], namespace=handlers_endpoints_v1.Namespace()) with self.call_should_fail('404'): self.call_api( 'retrieve', self.message_to_dict(retrieve_request), 200)
def generate_collection(contents, namespace=None): if namespace is None: namespace = handlers_endpoints_v1.Namespace() return handlers_endpoints_v1.DigestCollection( namespace=namespace, items=[generate_digest(content) for content in contents])
def generate_collection(namespace, contents): return handlers_endpoints_v1.DigestCollection( namespace=handlers_endpoints_v1.Namespace(namespace=namespace), items=[generate_digest(namespace, content) for content in contents])