def test_get_session_cache(mocker, isolated_session_cache):
mocker.patch("faculty.config.resolve_profile", return_value=PROFILE)
access_token_cache = mocker.Mock()
mocker.spy(Session, "__init__")
session1 = get_session(kwarg="foo", access_token_cache=access_token_cache)
session2 = get_session(kwarg="foo", access_token_cache=access_token_cache)
assert session1 is session2
faculty.config.resolve_profile.call_count == 1
Session.__init__.call_count == 1
def etag(project_path, project_id=None, object_client=None):
"""Get a unique identifier for the current version of a file.
Parameters
----------
project_path : str
The path in the project datasets.
project_id : str, optional
The project to get files from. You need to have access to this project
for it to work. Defaults to the project set by FACULTY_PROJECT_ID in
your environment.
object_client : faculty.clients.object.ObjectClient, optional
Advanced - can be used to benefit from caching in chain interactions
with datasets.
Returns
-------
str
"""
project_id = project_id or get_context().project_id
object_client = object_client or ObjectClient(get_session())
object = object_client.get(project_id, project_path)
return object.etag
def cp(
source_path,
destination_path,
project_id=None,
recursive=False,
object_client=None,
):
"""Copy a file or directory within a project's datasets.
Parameters
----------
source_path : str
The source path in the project datasets to copy.
destination_path : str
The destination path in the project datasets.
project_id : str, optional
The project to get files from. You need to have access to this project
for it to work. Defaults to the project set by FACULTY_PROJECT_ID in
your environment.
recursive : bool, optional
If True, allows copying directories
like a recursive copy in a filesystem. By default the action
is not recursive.
object_client : faculty.clients.object.ObjectClient, optional
Advanced - can be used to benefit from caching in chain interactions
with datasets.
"""
project_id = project_id or get_context().project_id
object_client = object_client or ObjectClient(get_session())
_create_parent_directories(destination_path, project_id, object_client)
object_client.copy(
project_id, source_path, destination_path, recursive=recursive
)
def get(project_path, local_path, project_id=None, object_client=None):
"""Copy from a project's datasets to the local filesystem.
Parameters
----------
project_path : str
The source path in the project datasets to copy.
local_path : str or os.PathLike
The destination path in the local filesystem.
project_id : str, optional
The project to get files from. You need to have access to this project
for it to work. Defaults to the project set by FACULTY_PROJECT_ID in
your environment.
object_client : faculty.clients.object.ObjectClient, optional
Advanced - can be used to benefit from caching in chain interactions
with datasets.
"""
project_id = project_id or get_context().project_id
object_client = object_client or ObjectClient(get_session())
if hasattr(os, "fspath"):
local_path = os.fspath(local_path)
if _isdir(project_path, project_id, object_client):
_get_directory(project_path, local_path, project_id, object_client)
else:
_get_file(project_path, local_path, project_id, object_client)
def put(local_path, project_path, project_id=None, object_client=None):
"""Copy from the local filesystem to a project's datasets.
Parameters
----------
local_path : str or os.PathLike
The source path in the local filesystem to copy.
project_path : str
The destination path in the project directory.
project_id : str, optional
The project to put files in. You need to have access to this project
for it to work. Defaults to the project set by FACULTY_PROJECT_ID in
your environment.
object_client : faculty.clients.object.ObjectClient, optional
Advanced - can be used to benefit from caching in chain interactions
with datasets.
"""
project_id = project_id or get_context().project_id
object_client = object_client or ObjectClient(get_session())
if hasattr(os, "fspath"):
local_path = os.fspath(local_path)
_create_parent_directories(project_path, project_id, object_client)
_put_recursive(local_path, project_path, project_id, object_client)
def mv(source_path, destination_path, project_id=None, object_client=None):
"""Move a file or directory within a project's datasets.
Parameters
----------
source_path : str
The source path in the project datasets to move.
destination_path : str
The destination path in the project datasets.
project_id : str, optional
The project to get files from. You need to have access to this project
for it to work. Defaults to the project set by FACULTY_PROJECT_ID in
your environment.
object_client : faculty.clients.object.ObjectClient, optional
Advanced - can be used to benefit from caching in chain interactions
with datasets.
"""
project_id = project_id or get_context().project_id
object_client = object_client or ObjectClient(get_session())
cp(
source_path,
destination_path,
project_id=project_id,
recursive=True,
object_client=object_client,
)
rm(
source_path,
project_id=project_id,
recursive=True,
object_client=object_client,
)
def test_get_session_defaults(mocker, isolated_session_cache):
mocker.patch("faculty.config.resolve_profile", return_value=PROFILE)
access_token_cache = mocker.Mock()
mocker.patch(
"faculty.session.AccessTokenMemoryCache",
return_value=access_token_cache,
)
mocker.spy(Session, "__init__")
session = get_session()
faculty.config.resolve_profile.assert_called_once_with()
Session.__init__.assert_called_once_with(
session, PROFILE, access_token_cache
)
def test_get_session(mocker, isolated_session_cache):
mocker.patch("faculty.config.resolve_profile", return_value=PROFILE)
access_token_cache = mocker.Mock()
mocker.spy(Session, "__init__")
session = get_session(
kwarg1="foo", kwarg2="bar", access_token_cache=access_token_cache
)
faculty.config.resolve_profile.assert_called_once_with(
kwarg1="foo", kwarg2="bar"
)
Session.__init__.assert_called_once_with(
session, PROFILE, access_token_cache
)
def ls(prefix="/", project_id=None, show_hidden=False, object_client=None):
"""List contents of project datasets.
Parameters
----------
prefix : str, optional
List only files in the datasets matching this prefix. Default behaviour
is to list all files.
project_id : str, optional
The project to list files from. You need to have access to this project
for it to work. Defaults to the project set by FACULTY_PROJECT_ID in
your environment.
show_hidden : bool, optional
Include hidden files in the output. Defaults to False.
object_client : faculty.clients.object.ObjectClient, optional
Advanced - can be used to benefit from caching in chain interactions
with datasets.
Returns
-------
list
The list of files from the project datasets.
"""
project_id = project_id or get_context().project_id
object_client = object_client or ObjectClient(get_session())
list_response = object_client.list(project_id, prefix)
paths = [obj.path for obj in list_response.objects]
while list_response.next_page_token is not None:
list_response = object_client.list(
project_id, prefix, list_response.next_page_token
)
paths += [obj.path for obj in list_response.objects]
if show_hidden:
return paths
else:
non_hidden_paths = [
path
for path in paths
if not any(element.startswith(".") for element in path.split("/"))
]
return non_hidden_paths
def test_get_session(mocker, isolated_session_cache):
mocker.patch("faculty.config.resolve_profile", return_value=PROFILE)
access_token_cache = mocker.Mock()
mocker.spy(Session, "__init__")
config_kwargs = {
"credentials_path": "/path/to/credentials",
"profile_name": "my-profile",
"domain": "domain.com",
"protocol": "http",
"client_id": "client-id",
"client_secret": "client-secret",
}
session = get_session(access_token_cache=access_token_cache,
**config_kwargs)
faculty.config.resolve_profile.assert_called_once_with(**config_kwargs)
Session.__init__.assert_called_once_with(session, PROFILE,
access_token_cache)
def test_get_session_defaults(mocker, isolated_session_cache):
mocker.patch("faculty.config.resolve_profile", return_value=PROFILE)
access_token_cache = mocker.Mock()
mocker.patch(
"faculty.session.AccessTokenMemoryCache",
return_value=access_token_cache,
)
mocker.spy(Session, "__init__")
session = get_session()
faculty.config.resolve_profile.assert_called_once_with(
credentials_path=None,
profile_name=None,
domain=None,
protocol=None,
client_id=None,
client_secret=None,
)
Session.__init__.assert_called_once_with(session, PROFILE,
access_token_cache)
def rm(project_path, project_id=None, recursive=False, object_client=None):
"""Remove a file or directory from the project directory.
Parameters
----------
project_path : str
The path in the project datasets to remove.
project_id : str, optional
The project to get files from. You need to have access to this project
for it to work. Defaults to the project set by FACULTY_PROJECT_ID in
your environment.
recursive : bool, optional
If True, allows deleting directories
like a recursive delete in a filesystem. By default the action
is not recursive.
object_client : faculty.clients.object.ObjectClient, optional
Advanced - can be used to benefit from caching in chain interactions
with datasets.
"""
project_id = project_id or get_context().project_id
object_client = object_client or ObjectClient(get_session())
object_client.delete(project_id, project_path, recursive=recursive)
def _default_session_object_client():
session = get_session()
url = session.service_url(ObjectClient.SERVICE_NAME)
return ObjectClient(url, session)
