def test_object_client_create_directory_default(mocker):
mocker.patch.object(ObjectClient, "_put_raw")
client = ObjectClient(mocker.Mock())
client.create_directory(PROJECT_ID, "test-path")
ObjectClient._put_raw.assert_called_once_with(
"/project/{}/directory/{}".format(PROJECT_ID, "test-path"),
params={"parents": 0},
)
def test_object_client_create_directory_url_encoding(mocker):
mocker.patch.object(ObjectClient, "_put_raw")
client = ObjectClient(mocker.Mock())
client.create_directory(PROJECT_ID, "[1]")
ObjectClient._put_raw.assert_called_once_with(
"/project/{}/directory/%5B1%5D".format(PROJECT_ID),
params={"parents": 0},
)
def test_object_client_delete_url_encoding(mocker):
path = "[1]"
mocker.patch.object(ObjectClient, "_delete_raw")
client = ObjectClient(mocker.Mock(), mocker.Mock())
client.delete(PROJECT_ID, path)
ObjectClient._delete_raw.assert_called_once_with(
"/project/{}/object/%5B1%5D".format(PROJECT_ID),
params={"recursive": 0},
)
def test_object_client_delete(mocker, recursive, expected_recursive):
path = "test-path"
mocker.patch.object(ObjectClient, "_delete_raw")
client = ObjectClient(mocker.Mock(), mocker.Mock())
client.delete(PROJECT_ID, path, recursive=recursive)
ObjectClient._delete_raw.assert_called_once_with(
"/project/{}/object/{}".format(PROJECT_ID, path),
params={"recursive": expected_recursive},
)
def test_object_client_get(mocker, path):
mocker.patch.object(ObjectClient, "_get", return_value=OBJECT)
schema_mock = mocker.patch("faculty.clients.object._ObjectSchema")
client = ObjectClient(mocker.Mock(), mocker.Mock())
assert client.get(PROJECT_ID, path) == OBJECT
schema_mock.assert_called_once_with()
ObjectClient._get.assert_called_once_with(
"/project/{}/object/test/path".format(PROJECT_ID),
schema_mock.return_value,
)
def test_object_client_copy(mocker, recursive, expected_recursive):
mocker.patch.object(ObjectClient, "_put_raw")
client = ObjectClient(mocker.Mock(), mocker.Mock())
client.copy(PROJECT_ID, "source", "destination", recursive=recursive)
ObjectClient._put_raw.assert_called_once_with(
"/project/{}/object/{}".format(PROJECT_ID, "destination"),
params={
"sourcePath": "source",
"recursive": expected_recursive
},
)
def test_object_client_copy_url_encoding(mocker):
mocker.patch.object(ObjectClient, "_put_raw")
client = ObjectClient(mocker.Mock(), mocker.Mock())
client.copy(PROJECT_ID, "source", "/[1]/")
ObjectClient._put_raw.assert_called_once_with(
"/project/{}/object/%5B1%5D/".format(PROJECT_ID),
params={
"sourcePath": "source",
"recursive": 0
},
)
def test_object_client_create_directory_already_exists(mocker):
error_code = "object_already_exists"
exception = Conflict(mocker.Mock(), mocker.Mock(), error_code)
mocker.patch.object(ObjectClient, "_put_raw", side_effect=exception)
client = ObjectClient(mocker.Mock(), mocker.Mock())
with pytest.raises(PathAlreadyExists, match="'test-path' already exists"):
client.create_directory(PROJECT_ID, "test-path")
ObjectClient._put_raw.assert_called_once_with(
"/project/{}/directory/{}".format(PROJECT_ID, "test-path"),
params={"parents": 0},
)
def test_object_client_get_url_encoding(mocker):
path = "/test/[1].txt"
mocker.patch.object(ObjectClient, "_get", return_value=OBJECT)
schema_mock = mocker.patch("faculty.clients.object.ObjectSchema")
client = ObjectClient(mocker.Mock())
assert client.get(PROJECT_ID, path) == OBJECT
schema_mock.assert_called_once_with()
ObjectClient._get.assert_called_once_with(
"/project/{}/object/test/%5B1%5D.txt".format(PROJECT_ID),
schema_mock.return_value,
)
def test_object_client_list_defaults(mocker):
mocker.patch.object(ObjectClient,
"_get",
return_value=LIST_OBJECTS_RESPONSE)
schema_mock = mocker.patch(
"faculty.clients.object._ListObjectsResponseSchema")
client = ObjectClient(mocker.Mock(), mocker.Mock())
assert client.list(PROJECT_ID) == LIST_OBJECTS_RESPONSE
schema_mock.assert_called_once_with()
ObjectClient._get.assert_called_once_with(
"/project/{}/object-list/".format(PROJECT_ID),
schema_mock.return_value,
params={},
)
def etag(project_path, project_id=None, object_client=None):
"""Get a unique identifier for the current version of a file.
Parameters
----------
project_path : str
The path in the project datasets.
project_id : str, optional
The project to get files from. You need to have access to this project
for it to work. Defaults to the project set by FACULTY_PROJECT_ID in
your environment.
object_client : faculty.clients.object.ObjectClient, optional
Advanced - can be used to benefit from caching in chain interactions
with datasets.
Returns
-------
str
"""
project_id = project_id or get_context().project_id
object_client = object_client or ObjectClient(get_session())
object = object_client.get(project_id, project_path)
return object.etag
def put(local_path, project_path, project_id=None, object_client=None):
"""Copy from the local filesystem to a project's datasets.
Parameters
----------
local_path : str or os.PathLike
The source path in the local filesystem to copy.
project_path : str
The destination path in the project directory.
project_id : str, optional
The project to put files in. You need to have access to this project
for it to work. Defaults to the project set by FACULTY_PROJECT_ID in
your environment.
object_client : faculty.clients.object.ObjectClient, optional
Advanced - can be used to benefit from caching in chain interactions
with datasets.
"""
project_id = project_id or get_context().project_id
object_client = object_client or ObjectClient(get_session())
if hasattr(os, "fspath"):
local_path = os.fspath(local_path)
_create_parent_directories(project_path, project_id, object_client)
_put_recursive(local_path, project_path, project_id, object_client)
def cp(
source_path,
destination_path,
project_id=None,
recursive=False,
object_client=None,
):
"""Copy a file or directory within a project's datasets.
Parameters
----------
source_path : str
The source path in the project datasets to copy.
destination_path : str
The destination path in the project datasets.
project_id : str, optional
The project to get files from. You need to have access to this project
for it to work. Defaults to the project set by FACULTY_PROJECT_ID in
your environment.
recursive : bool, optional
If True, allows copying directories
like a recursive copy in a filesystem. By default the action
is not recursive.
object_client : faculty.clients.object.ObjectClient, optional
Advanced - can be used to benefit from caching in chain interactions
with datasets.
"""
project_id = project_id or get_context().project_id
object_client = object_client or ObjectClient(get_session())
_create_parent_directories(destination_path, project_id, object_client)
object_client.copy(
project_id, source_path, destination_path, recursive=recursive
)
def get(project_path, local_path, project_id=None, object_client=None):
"""Copy from a project's datasets to the local filesystem.
Parameters
----------
project_path : str
The source path in the project datasets to copy.
local_path : str or os.PathLike
The destination path in the local filesystem.
project_id : str, optional
The project to get files from. You need to have access to this project
for it to work. Defaults to the project set by FACULTY_PROJECT_ID in
your environment.
object_client : faculty.clients.object.ObjectClient, optional
Advanced - can be used to benefit from caching in chain interactions
with datasets.
"""
project_id = project_id or get_context().project_id
object_client = object_client or ObjectClient(get_session())
if hasattr(os, "fspath"):
local_path = os.fspath(local_path)
if _isdir(project_path, project_id, object_client):
_get_directory(project_path, local_path, project_id, object_client)
else:
_get_file(project_path, local_path, project_id, object_client)
def mv(source_path, destination_path, project_id=None, object_client=None):
"""Move a file or directory within a project's datasets.
Parameters
----------
source_path : str
The source path in the project datasets to move.
destination_path : str
The destination path in the project datasets.
project_id : str, optional
The project to get files from. You need to have access to this project
for it to work. Defaults to the project set by FACULTY_PROJECT_ID in
your environment.
object_client : faculty.clients.object.ObjectClient, optional
Advanced - can be used to benefit from caching in chain interactions
with datasets.
"""
project_id = project_id or get_context().project_id
object_client = object_client or ObjectClient(get_session())
cp(
source_path,
destination_path,
project_id=project_id,
recursive=True,
object_client=object_client,
)
rm(
source_path,
project_id=project_id,
recursive=True,
object_client=object_client,
)
def test_object_client_presign_upload(mocker):
mocker.patch.object(ObjectClient,
"_post",
return_value=PRESIGN_UPLOAD_RESPONSE_S3)
schema_mock = mocker.patch(
"faculty.clients.object._PresignUploadResponseSchema")
client = ObjectClient(mocker.Mock(), mocker.Mock())
returned = client.presign_upload(PROJECT_ID, "/path")
assert returned == PRESIGN_UPLOAD_RESPONSE_S3
schema_mock.assert_called_once_with()
ObjectClient._post.assert_called_once_with(
"/project/{}/presign/upload".format(PROJECT_ID),
schema_mock.return_value,
json={"path": "/path"},
)
def test_object_client_presign_download_defaults(mocker):
mocker.patch.object(ObjectClient,
"_post",
return_value=SIMPLE_PRESIGN_RESPONSE)
schema_mock = mocker.patch(
"faculty.clients.object._SimplePresignResponseSchema")
client = ObjectClient(mocker.Mock(), mocker.Mock())
returned = client.presign_download(PROJECT_ID, "/path")
assert returned == SIMPLE_PRESIGN_RESPONSE.url
schema_mock.assert_called_once_with()
ObjectClient._post.assert_called_once_with(
"/project/{}/presign/download".format(PROJECT_ID),
schema_mock.return_value,
json={"path": "/path"},
)
def test_object_client_list(mocker, path):
mocker.patch.object(ObjectClient,
"_get",
return_value=LIST_OBJECTS_RESPONSE)
schema_mock = mocker.patch(
"faculty.clients.object._ListObjectsResponseSchema")
client = ObjectClient(mocker.Mock(), mocker.Mock())
response = client.list(PROJECT_ID, path, page_token="token")
assert response == LIST_OBJECTS_RESPONSE
schema_mock.assert_called_once_with()
ObjectClient._get.assert_called_once_with(
"/project/{}/object-list/test/path".format(PROJECT_ID),
schema_mock.return_value,
params={"pageToken": "token"},
)
def test_object_client_complete_multipart_upload(mocker):
mocker.patch.object(ObjectClient, "_put_raw")
payload_schema_mock = mocker.patch(
"faculty.clients.object._CompleteMultipartUploadSchema")
client = ObjectClient(mocker.Mock(), mocker.Mock())
client.complete_multipart_upload(PROJECT_ID, "/path", "upload-id",
[COMPLETED_UPLOAD_PART])
payload_schema_mock.assert_called_once_with()
dump_mock = payload_schema_mock.return_value.dump
dump_mock.assert_called_once_with({
"path": "/path",
"upload_id": "upload-id",
"parts": [COMPLETED_UPLOAD_PART],
})
ObjectClient._put_raw.assert_called_once_with(
"/project/{}/presign/upload/complete".format(PROJECT_ID),
json=dump_mock.return_value,
)
def ls(prefix="/", project_id=None, show_hidden=False, object_client=None):
"""List contents of project datasets.
Parameters
----------
prefix : str, optional
List only files in the datasets matching this prefix. Default behaviour
is to list all files.
project_id : str, optional
The project to list files from. You need to have access to this project
for it to work. Defaults to the project set by FACULTY_PROJECT_ID in
your environment.
show_hidden : bool, optional
Include hidden files in the output. Defaults to False.
object_client : faculty.clients.object.ObjectClient, optional
Advanced - can be used to benefit from caching in chain interactions
with datasets.
Returns
-------
list
The list of files from the project datasets.
"""
project_id = project_id or get_context().project_id
object_client = object_client or ObjectClient(get_session())
list_response = object_client.list(project_id, prefix)
paths = [obj.path for obj in list_response.objects]
while list_response.next_page_token is not None:
list_response = object_client.list(
project_id, prefix, list_response.next_page_token
)
paths += [obj.path for obj in list_response.objects]
if show_hidden:
return paths
else:
non_hidden_paths = [
path
for path in paths
if not any(element.startswith(".") for element in path.split("/"))
]
return non_hidden_paths
def rm(project_path, project_id=None, recursive=False, object_client=None):
"""Remove a file or directory from the project directory.
Parameters
----------
project_path : str
The path in the project datasets to remove.
project_id : str, optional
The project to get files from. You need to have access to this project
for it to work. Defaults to the project set by FACULTY_PROJECT_ID in
your environment.
recursive : bool, optional
If True, allows deleting directories
like a recursive delete in a filesystem. By default the action
is not recursive.
object_client : faculty.clients.object.ObjectClient, optional
Advanced - can be used to benefit from caching in chain interactions
with datasets.
"""
project_id = project_id or get_context().project_id
object_client = object_client or ObjectClient(get_session())
object_client.delete(project_id, project_path, recursive=recursive)
def _default_session_object_client():
session = get_session()
url = session.service_url(ObjectClient.SERVICE_NAME)
return ObjectClient(url, session)