def test_metadata_from_resp_headers_missing_object_name(
self, meta_file_raw):
resp_headers = utils.get_multi_dict_from_python_dict(
dict(json.loads(meta_file_raw)))
with pytest.raises(exceptions.MetadataError):
GoogleCloudFileMetadata.new_from_resp_headers('', resp_headers)
def test_metadata_from_resp_headers_invalid_resp_headers(
self, file_obj_name, meta_file_raw):
resp_headers = json.loads(meta_file_raw)
with pytest.raises(exceptions.MetadataError):
GoogleCloudFileMetadata.new_from_resp_headers(
file_obj_name, resp_headers)
def test_metadata_from_resp_headers(self, file_obj_name, meta_file_raw):
resp_headers = utils.get_multi_dict_from_python_dict(dict(json.loads(meta_file_raw)))
metadata = GoogleCloudFileMetadata.new_from_resp_headers(file_obj_name, resp_headers)
assert metadata
assert metadata.etag == '9a46947c9c622d7792125d8ea44c4638'
async def test_upload_file(self, mock_time, mock_provider, file_wb_path,
meta_file_raw, meta_file_parsed,
meta_file_upload_raw, file_stream_file):
file_obj_name = utils.get_obj_name(file_wb_path, is_folder=False)
signed_url_upload = mock_provider._build_and_sign_url(
'PUT', file_obj_name, **{})
resp_headers = utils.get_multi_dict_from_python_dict(
dict(json.loads(meta_file_upload_raw)))
aiohttpretty.register_uri('PUT',
signed_url_upload,
headers=resp_headers,
status=HTTPStatus.OK)
signed_url_metadata = mock_provider._build_and_sign_url(
'HEAD', file_obj_name, **{})
resp_headers = utils.get_multi_dict_from_python_dict(
dict(json.loads(meta_file_raw)))
aiohttpretty.register_uri('HEAD',
signed_url_metadata,
headers=resp_headers,
status=HTTPStatus.OK)
metadata_json = json.loads(meta_file_parsed)
metadata_expected = GoogleCloudFileMetadata(metadata_json)
metadata, _ = await mock_provider.upload(file_stream_file,
file_wb_path)
assert metadata == metadata_expected
assert aiohttpretty.has_call(method='PUT', uri=signed_url_upload)
assert aiohttpretty.has_call(method='HEAD', uri=signed_url_metadata)
def test_metadata_from_dict(self, meta_file_parsed):
resp_headers = dict(json.loads(meta_file_parsed))
metadata = GoogleCloudFileMetadata(resp_headers)
assert metadata
assert metadata.etag == '9a46947c9c622d7792125d8ea44c4638'
def test_metadata_from_resp_headers(self, file_obj_name, meta_file_raw):
resp_headers = utils.get_multi_dict_from_python_dict(dict(json.loads(meta_file_raw)))
metadata = GoogleCloudFileMetadata.new_from_resp_headers(file_obj_name, resp_headers)
assert metadata
assert metadata.etag == '9a46947c9c622d7792125d8ea44c4638'
async def _metadata_object(self, path: WaterButlerPath,
is_folder: bool=False) \
-> typing.Union[GoogleCloudFileMetadata, GoogleCloudFolderMetadata]:
"""Get the metadata about the object with the given WaterButlerPath.
API docs:
GET Object: https://cloud.google.com/storage/docs/xml-api/get-object
HEAD Object: https://cloud.google.com/storage/docs/xml-api/head-object
.. note::
Use ``HEAD`` instead of ``GET`` to retrieve the metadata of an object. Google points
out that: "You should not use a ``GET`` object request to retrieve only non-ACL
metadata, because doing so incurs egress charges associated with downloading the entire
object. Instead use a ``HEAD`` object request to retrieve non-ACL metadata for the
object."
.. note::
The flag ``is_folder`` is explicitly used. Providing the wrong type will always fail.
This is the case for many internal/private methods of and helper/utility functions for
this class. They are not exposed to any outside usage, including the parent classes.
:param path: the WaterButlerPath of the object
:type path: :class:`.WaterButlerPath`
:param bool is_folder: whether the object is a file or folder
:rtype: :class:`.GoogleCloudFileMetadata`
:rtype: :class:`.GoogleCloudFolderMetadata`
"""
req_method = 'HEAD'
obj_name = utils.get_obj_name(path, is_folder=is_folder)
signed_url = functools.partial(self._build_and_sign_url, req_method,
obj_name, **{})
resp = await self.make_request(req_method,
signed_url,
expects=(HTTPStatus.OK, ),
throws=MetadataError)
await resp.release()
if is_folder:
return GoogleCloudFolderMetadata.new_from_resp_headers(
obj_name, resp.headers)
else:
return GoogleCloudFileMetadata.new_from_resp_headers(
obj_name, resp.headers)
async def _metadata_object(self, path: WaterButlerPath,
is_folder: bool=False) \
-> typing.Union[GoogleCloudFileMetadata, GoogleCloudFolderMetadata]:
"""Get the metadata about the object with the given WaterButlerPath.
API docs:
GET Object: https://cloud.google.com/storage/docs/xml-api/get-object
HEAD Object: https://cloud.google.com/storage/docs/xml-api/head-object
.. note::
Use ``HEAD`` instead of ``GET`` to retrieve the metadata of an object. Google points
out that: "You should not use a ``GET`` object request to retrieve only non-ACL
metadata, because doing so incurs egress charges associated with downloading the entire
object. Instead use a ``HEAD`` object request to retrieve non-ACL metadata for the
object."
.. note::
The flag ``is_folder`` is explicitly used. Providing the wrong type will always fail.
This is the case for many internal/private methods of and helper/utility functions for
this class. They are not exposed to any outside usage, including the parent classes.
:param path: the WaterButlerPath of the object
:type path: :class:`.WaterButlerPath`
:param bool is_folder: whether the object is a file or folder
:rtype: :class:`.GoogleCloudFileMetadata`
:rtype: :class:`.GoogleCloudFolderMetadata`
"""
req_method = 'HEAD'
obj_name = utils.get_obj_name(path, is_folder=is_folder)
signed_url = functools.partial(self._build_and_sign_url, req_method, obj_name, **{})
resp = await self.make_request(
req_method,
signed_url,
expects=(HTTPStatus.OK,),
throws=MetadataError
)
await resp.release()
if is_folder:
return GoogleCloudFolderMetadata.new_from_resp_headers(obj_name, resp.headers)
else:
return GoogleCloudFileMetadata.new_from_resp_headers(obj_name, resp.headers)
async def test_intra_copy_file(self, mock_time, mock_provider,
file_wb_path, file_2_wb_path, meta_file_raw,
meta_file_parsed, meta_file_copy_raw):
src_file_path = file_2_wb_path
dest_file_path = file_wb_path
src_file_obj_name = utils.get_obj_name(src_file_path, is_folder=False)
dest_file_obj_name = utils.get_obj_name(dest_file_path,
is_folder=False)
object_name_with_bucket = '{}/{}'.format(mock_provider.bucket,
src_file_obj_name)
canonical_ext_headers = {'x-goog-copy-source': object_name_with_bucket}
signed_url_intra_copy = mock_provider._build_and_sign_url(
'PUT',
dest_file_obj_name,
canonical_ext_headers=canonical_ext_headers,
**{})
resp_headers = utils.get_multi_dict_from_python_dict(
dict(json.loads(meta_file_copy_raw)))
aiohttpretty.register_uri('PUT',
signed_url_intra_copy,
headers=resp_headers,
status=HTTPStatus.OK)
signed_url_metadata = mock_provider._build_and_sign_url(
'HEAD', dest_file_obj_name, **{})
resp_headers = utils.get_multi_dict_from_python_dict(
dict(json.loads(meta_file_raw)))
aiohttpretty.register_uri('HEAD',
signed_url_metadata,
headers=resp_headers,
status=HTTPStatus.OK)
metadata_json = json.loads(meta_file_parsed)
metadata_expected = GoogleCloudFileMetadata(metadata_json)
metadata, _ = await mock_provider.intra_copy(mock_provider,
src_file_path,
dest_file_path)
assert metadata == metadata_expected
assert aiohttpretty.has_call(method='PUT', uri=signed_url_intra_copy)
assert aiohttpretty.has_call(method='HEAD', uri=signed_url_metadata)
def test_file_metadata(self, file_name, file_obj_name, meta_file_parsed, meta_file_extra):
metadata_json = json.loads(meta_file_parsed)
metadata_extra = json.loads(meta_file_extra)
metadata = GoogleCloudFileMetadata(metadata_json)
assert isinstance(metadata, BaseGoogleCloudMetadata)
assert metadata.provider == 'googlecloud'
assert metadata.path == '/{}'.format(file_obj_name)
assert metadata.name == file_name
assert metadata.kind == 'file'
assert metadata.content_type == 'text/plain'
assert metadata.modified == 'Thu, 01 Mar 2018 19:04:45 GMT'
assert metadata.modified_utc == '2018-03-01T19:04:45+00:00'
assert metadata.created_utc is None
assert metadata.etag == '9a46947c9c622d7792125d8ea44c4638'
assert metadata.size == 85
assert metadata.size_as_int == 85
assert metadata.extra == dict(metadata_extra)
async def test_metadata_file(self, mock_time, mock_provider, file_wb_path,
meta_file_raw, meta_file_parsed):
file_obj_name = utils.get_obj_name(file_wb_path, is_folder=False)
signed_url = mock_provider._build_and_sign_url('HEAD', file_obj_name,
**{})
resp_headers = utils.get_multi_dict_from_python_dict(
dict(json.loads(meta_file_raw)))
aiohttpretty.register_uri('HEAD',
signed_url,
headers=resp_headers,
status=HTTPStatus.OK)
metadata_json = json.loads(meta_file_parsed)
metadata_expected = GoogleCloudFileMetadata(metadata_json)
metadata = await mock_provider._metadata_object(file_wb_path,
is_folder=False)
assert isinstance(metadata, GoogleCloudFileMetadata)
assert metadata == metadata_expected
def test_metadata_from_resp_headers_missing_resp_headers(self, file_obj_name):
with pytest.raises(exceptions.MetadataError):
GoogleCloudFileMetadata.new_from_resp_headers(file_obj_name, MultiDict({}))
def test_metadata_from_resp_headers_invalid_resp_headers(self, file_obj_name, meta_file_raw):
resp_headers = json.loads(meta_file_raw)
with pytest.raises(exceptions.MetadataError):
GoogleCloudFileMetadata.new_from_resp_headers(file_obj_name, resp_headers)
def test_metadata_from_resp_headers_missing_object_name(self, meta_file_raw):
resp_headers = utils.get_multi_dict_from_python_dict(dict(json.loads(meta_file_raw)))
with pytest.raises(exceptions.MetadataError):
GoogleCloudFileMetadata.new_from_resp_headers('', resp_headers)
def test_metadata_from_resp_headers_missing_resp_headers(
self, file_obj_name):
with pytest.raises(exceptions.MetadataError):
GoogleCloudFileMetadata.new_from_resp_headers(
file_obj_name, MultiDict({}))