def test_object_client_create_directory_default(mocker): mocker.patch.object(ObjectClient, "_put_raw") client = ObjectClient(mocker.Mock()) client.create_directory(PROJECT_ID, "test-path") ObjectClient._put_raw.assert_called_once_with( "/project/{}/directory/{}".format(PROJECT_ID, "test-path"), params={"parents": 0}, )
def test_object_client_create_directory_url_encoding(mocker): mocker.patch.object(ObjectClient, "_put_raw") client = ObjectClient(mocker.Mock()) client.create_directory(PROJECT_ID, "[1]") ObjectClient._put_raw.assert_called_once_with( "/project/{}/directory/%5B1%5D".format(PROJECT_ID), params={"parents": 0}, )
def test_object_client_delete_url_encoding(mocker): path = "[1]" mocker.patch.object(ObjectClient, "_delete_raw") client = ObjectClient(mocker.Mock(), mocker.Mock()) client.delete(PROJECT_ID, path) ObjectClient._delete_raw.assert_called_once_with( "/project/{}/object/%5B1%5D".format(PROJECT_ID), params={"recursive": 0}, )
def test_object_client_delete(mocker, recursive, expected_recursive): path = "test-path" mocker.patch.object(ObjectClient, "_delete_raw") client = ObjectClient(mocker.Mock(), mocker.Mock()) client.delete(PROJECT_ID, path, recursive=recursive) ObjectClient._delete_raw.assert_called_once_with( "/project/{}/object/{}".format(PROJECT_ID, path), params={"recursive": expected_recursive}, )
def test_object_client_get(mocker, path): mocker.patch.object(ObjectClient, "_get", return_value=OBJECT) schema_mock = mocker.patch("faculty.clients.object._ObjectSchema") client = ObjectClient(mocker.Mock(), mocker.Mock()) assert client.get(PROJECT_ID, path) == OBJECT schema_mock.assert_called_once_with() ObjectClient._get.assert_called_once_with( "/project/{}/object/test/path".format(PROJECT_ID), schema_mock.return_value, )
def test_object_client_copy(mocker, recursive, expected_recursive): mocker.patch.object(ObjectClient, "_put_raw") client = ObjectClient(mocker.Mock(), mocker.Mock()) client.copy(PROJECT_ID, "source", "destination", recursive=recursive) ObjectClient._put_raw.assert_called_once_with( "/project/{}/object/{}".format(PROJECT_ID, "destination"), params={ "sourcePath": "source", "recursive": expected_recursive }, )
def test_object_client_copy_url_encoding(mocker): mocker.patch.object(ObjectClient, "_put_raw") client = ObjectClient(mocker.Mock(), mocker.Mock()) client.copy(PROJECT_ID, "source", "/[1]/") ObjectClient._put_raw.assert_called_once_with( "/project/{}/object/%5B1%5D/".format(PROJECT_ID), params={ "sourcePath": "source", "recursive": 0 }, )
def test_object_client_create_directory_already_exists(mocker): error_code = "object_already_exists" exception = Conflict(mocker.Mock(), mocker.Mock(), error_code) mocker.patch.object(ObjectClient, "_put_raw", side_effect=exception) client = ObjectClient(mocker.Mock(), mocker.Mock()) with pytest.raises(PathAlreadyExists, match="'test-path' already exists"): client.create_directory(PROJECT_ID, "test-path") ObjectClient._put_raw.assert_called_once_with( "/project/{}/directory/{}".format(PROJECT_ID, "test-path"), params={"parents": 0}, )
def test_object_client_get_url_encoding(mocker): path = "/test/[1].txt" mocker.patch.object(ObjectClient, "_get", return_value=OBJECT) schema_mock = mocker.patch("faculty.clients.object.ObjectSchema") client = ObjectClient(mocker.Mock()) assert client.get(PROJECT_ID, path) == OBJECT schema_mock.assert_called_once_with() ObjectClient._get.assert_called_once_with( "/project/{}/object/test/%5B1%5D.txt".format(PROJECT_ID), schema_mock.return_value, )
def test_object_client_list_defaults(mocker): mocker.patch.object(ObjectClient, "_get", return_value=LIST_OBJECTS_RESPONSE) schema_mock = mocker.patch( "faculty.clients.object._ListObjectsResponseSchema") client = ObjectClient(mocker.Mock(), mocker.Mock()) assert client.list(PROJECT_ID) == LIST_OBJECTS_RESPONSE schema_mock.assert_called_once_with() ObjectClient._get.assert_called_once_with( "/project/{}/object-list/".format(PROJECT_ID), schema_mock.return_value, params={}, )
def etag(project_path, project_id=None, object_client=None): """Get a unique identifier for the current version of a file. Parameters ---------- project_path : str The path in the project datasets. project_id : str, optional The project to get files from. You need to have access to this project for it to work. Defaults to the project set by FACULTY_PROJECT_ID in your environment. object_client : faculty.clients.object.ObjectClient, optional Advanced - can be used to benefit from caching in chain interactions with datasets. Returns ------- str """ project_id = project_id or get_context().project_id object_client = object_client or ObjectClient(get_session()) object = object_client.get(project_id, project_path) return object.etag
def put(local_path, project_path, project_id=None, object_client=None): """Copy from the local filesystem to a project's datasets. Parameters ---------- local_path : str or os.PathLike The source path in the local filesystem to copy. project_path : str The destination path in the project directory. project_id : str, optional The project to put files in. You need to have access to this project for it to work. Defaults to the project set by FACULTY_PROJECT_ID in your environment. object_client : faculty.clients.object.ObjectClient, optional Advanced - can be used to benefit from caching in chain interactions with datasets. """ project_id = project_id or get_context().project_id object_client = object_client or ObjectClient(get_session()) if hasattr(os, "fspath"): local_path = os.fspath(local_path) _create_parent_directories(project_path, project_id, object_client) _put_recursive(local_path, project_path, project_id, object_client)
def cp( source_path, destination_path, project_id=None, recursive=False, object_client=None, ): """Copy a file or directory within a project's datasets. Parameters ---------- source_path : str The source path in the project datasets to copy. destination_path : str The destination path in the project datasets. project_id : str, optional The project to get files from. You need to have access to this project for it to work. Defaults to the project set by FACULTY_PROJECT_ID in your environment. recursive : bool, optional If True, allows copying directories like a recursive copy in a filesystem. By default the action is not recursive. object_client : faculty.clients.object.ObjectClient, optional Advanced - can be used to benefit from caching in chain interactions with datasets. """ project_id = project_id or get_context().project_id object_client = object_client or ObjectClient(get_session()) _create_parent_directories(destination_path, project_id, object_client) object_client.copy( project_id, source_path, destination_path, recursive=recursive )
def get(project_path, local_path, project_id=None, object_client=None): """Copy from a project's datasets to the local filesystem. Parameters ---------- project_path : str The source path in the project datasets to copy. local_path : str or os.PathLike The destination path in the local filesystem. project_id : str, optional The project to get files from. You need to have access to this project for it to work. Defaults to the project set by FACULTY_PROJECT_ID in your environment. object_client : faculty.clients.object.ObjectClient, optional Advanced - can be used to benefit from caching in chain interactions with datasets. """ project_id = project_id or get_context().project_id object_client = object_client or ObjectClient(get_session()) if hasattr(os, "fspath"): local_path = os.fspath(local_path) if _isdir(project_path, project_id, object_client): _get_directory(project_path, local_path, project_id, object_client) else: _get_file(project_path, local_path, project_id, object_client)
def mv(source_path, destination_path, project_id=None, object_client=None): """Move a file or directory within a project's datasets. Parameters ---------- source_path : str The source path in the project datasets to move. destination_path : str The destination path in the project datasets. project_id : str, optional The project to get files from. You need to have access to this project for it to work. Defaults to the project set by FACULTY_PROJECT_ID in your environment. object_client : faculty.clients.object.ObjectClient, optional Advanced - can be used to benefit from caching in chain interactions with datasets. """ project_id = project_id or get_context().project_id object_client = object_client or ObjectClient(get_session()) cp( source_path, destination_path, project_id=project_id, recursive=True, object_client=object_client, ) rm( source_path, project_id=project_id, recursive=True, object_client=object_client, )
def test_object_client_presign_upload(mocker): mocker.patch.object(ObjectClient, "_post", return_value=PRESIGN_UPLOAD_RESPONSE_S3) schema_mock = mocker.patch( "faculty.clients.object._PresignUploadResponseSchema") client = ObjectClient(mocker.Mock(), mocker.Mock()) returned = client.presign_upload(PROJECT_ID, "/path") assert returned == PRESIGN_UPLOAD_RESPONSE_S3 schema_mock.assert_called_once_with() ObjectClient._post.assert_called_once_with( "/project/{}/presign/upload".format(PROJECT_ID), schema_mock.return_value, json={"path": "/path"}, )
def test_object_client_presign_download_defaults(mocker): mocker.patch.object(ObjectClient, "_post", return_value=SIMPLE_PRESIGN_RESPONSE) schema_mock = mocker.patch( "faculty.clients.object._SimplePresignResponseSchema") client = ObjectClient(mocker.Mock(), mocker.Mock()) returned = client.presign_download(PROJECT_ID, "/path") assert returned == SIMPLE_PRESIGN_RESPONSE.url schema_mock.assert_called_once_with() ObjectClient._post.assert_called_once_with( "/project/{}/presign/download".format(PROJECT_ID), schema_mock.return_value, json={"path": "/path"}, )
def test_object_client_list(mocker, path): mocker.patch.object(ObjectClient, "_get", return_value=LIST_OBJECTS_RESPONSE) schema_mock = mocker.patch( "faculty.clients.object._ListObjectsResponseSchema") client = ObjectClient(mocker.Mock(), mocker.Mock()) response = client.list(PROJECT_ID, path, page_token="token") assert response == LIST_OBJECTS_RESPONSE schema_mock.assert_called_once_with() ObjectClient._get.assert_called_once_with( "/project/{}/object-list/test/path".format(PROJECT_ID), schema_mock.return_value, params={"pageToken": "token"}, )
def test_object_client_complete_multipart_upload(mocker): mocker.patch.object(ObjectClient, "_put_raw") payload_schema_mock = mocker.patch( "faculty.clients.object._CompleteMultipartUploadSchema") client = ObjectClient(mocker.Mock(), mocker.Mock()) client.complete_multipart_upload(PROJECT_ID, "/path", "upload-id", [COMPLETED_UPLOAD_PART]) payload_schema_mock.assert_called_once_with() dump_mock = payload_schema_mock.return_value.dump dump_mock.assert_called_once_with({ "path": "/path", "upload_id": "upload-id", "parts": [COMPLETED_UPLOAD_PART], }) ObjectClient._put_raw.assert_called_once_with( "/project/{}/presign/upload/complete".format(PROJECT_ID), json=dump_mock.return_value, )
def ls(prefix="/", project_id=None, show_hidden=False, object_client=None): """List contents of project datasets. Parameters ---------- prefix : str, optional List only files in the datasets matching this prefix. Default behaviour is to list all files. project_id : str, optional The project to list files from. You need to have access to this project for it to work. Defaults to the project set by FACULTY_PROJECT_ID in your environment. show_hidden : bool, optional Include hidden files in the output. Defaults to False. object_client : faculty.clients.object.ObjectClient, optional Advanced - can be used to benefit from caching in chain interactions with datasets. Returns ------- list The list of files from the project datasets. """ project_id = project_id or get_context().project_id object_client = object_client or ObjectClient(get_session()) list_response = object_client.list(project_id, prefix) paths = [obj.path for obj in list_response.objects] while list_response.next_page_token is not None: list_response = object_client.list( project_id, prefix, list_response.next_page_token ) paths += [obj.path for obj in list_response.objects] if show_hidden: return paths else: non_hidden_paths = [ path for path in paths if not any(element.startswith(".") for element in path.split("/")) ] return non_hidden_paths
def rm(project_path, project_id=None, recursive=False, object_client=None): """Remove a file or directory from the project directory. Parameters ---------- project_path : str The path in the project datasets to remove. project_id : str, optional The project to get files from. You need to have access to this project for it to work. Defaults to the project set by FACULTY_PROJECT_ID in your environment. recursive : bool, optional If True, allows deleting directories like a recursive delete in a filesystem. By default the action is not recursive. object_client : faculty.clients.object.ObjectClient, optional Advanced - can be used to benefit from caching in chain interactions with datasets. """ project_id = project_id or get_context().project_id object_client = object_client or ObjectClient(get_session()) object_client.delete(project_id, project_path, recursive=recursive)
def _default_session_object_client(): session = get_session() url = session.service_url(ObjectClient.SERVICE_NAME) return ObjectClient(url, session)