def test_sync_from_local(mock_get_project, mock_run):
rubicon = Rubicon(persistence="filesystem", root_dir="./local/path")
project_name = "Sync Test Project"
mock_get_project.return_value = client.Project(
domain.Project(project_name))
rubicon.sync(project_name, "s3://test/path")
assert "aws s3 sync ./local/path/sync-test-project s3://test/path" in str(
mock_run._mock_call_args_list)
def test_repository_storage_options():
storage_options = {"key": "secret"}
rubicon_memory = Rubicon(persistence="memory",
root_dir="./",
**storage_options)
rubicon_s3 = Rubicon(persistence="filesystem",
root_dir="s3://nothing",
**storage_options)
assert rubicon_memory.config.repository.filesystem.storage_options[
"key"] == "secret"
assert rubicon_s3.config.repository.filesystem.storage_options[
"key"] == "secret"
def test_sync_from_local_error(mock_get_project, mock_run):
rubicon = Rubicon(persistence="filesystem", root_dir="./local/path")
project_name = "Sync Test Project"
mock_get_project.return_value = client.Project(
domain.Project(project_name))
mock_run.side_effect = subprocess.CalledProcessError(
cmd="aws cli sync",
stderr="Some error. I bet it was proxy tho.",
returncode=1)
with pytest.raises(RubiconException) as e:
rubicon.sync(project_name, "s3://test/path")
assert "Some error. I bet it was proxy tho." in str(e)
def test_create_project_with_auto_git(mock_completed_process_git):
with mock.patch("subprocess.run") as mock_run:
mock_run.return_value = mock_completed_process_git
rubicon = Rubicon("memory", "test-root", auto_git_enabled=True)
rubicon.create_project("Test Project A")
expected = [
mock.call(["git", "rev-parse", "--git-dir"], capture_output=True),
mock.call(["git", "remote", "-v"], capture_output=True),
]
assert mock_run.mock_calls == expected
rubicon.repository.filesystem.store = {}
class ProjectSource(DataSourceMixin):
"""An Intake data source for reading `rubicon` projects.
Parameters
----------
urlpath : str
The root directory the `rubicon` data is logged to.
project_name : str
The name of the `rubicon` project to load.
"""
version = __version__
container = "python"
name = "rubicon_project"
def _get_schema(self):
"""Load the project named `self._project_name`."""
self._rubicon = Rubicon(persistence="filesystem",
root_dir=self._urlpath)
self._rubicon_object = self._rubicon.get_project(self._project_name)
self._metadata.update({
"project": {
"name": self._rubicon_object.name,
"id": self._rubicon_object.id,
"description": self._rubicon_object.description,
"created_id": self._rubicon_object.created_at,
}
})
return super()._get_schema()
def _get_schema(self):
"""Load the project named `self._project_name`."""
self._rubicon = Rubicon(persistence="filesystem",
root_dir=self._urlpath)
self._rubicon_object = self._rubicon.get_project(self._project_name)
self._metadata.update({
"project": {
"name": self._rubicon_object.name,
"id": self._rubicon_object.id,
"description": self._rubicon_object.description,
"created_id": self._rubicon_object.created_at,
}
})
return super()._get_schema()
class ExperimentSource(DataSourceMixin):
"""An Intake data source for reading `rubicon` experiments.
Parameters
----------
urlpath : str
The root directory the `rubicon` data is logged to.
project_name : str
The name of the `rubicon` project to load.
experiment_id : str
The ID of the `rubicon` experiment to load.
"""
version = __version__
container = "python"
name = "rubicon_experiment"
def __init__(self,
urlpath,
project_name,
experiment_id,
metadata=None,
storage_options=None,
**kwargs):
self._experiment_id = experiment_id
super().__init__(urlpath,
project_name,
metadata=metadata,
storage_options=storage_options,
**kwargs)
def _get_schema(self):
"""Load the experiment with ID `self._experiment_id`
from the project named `self._project_name`.
"""
self._rubicon = Rubicon(persistence="filesystem",
root_dir=self._urlpath)
self._rubicon_object = self._rubicon.get_project(
self._project_name).experiment(self._experiment_id)
self._metadata.update({
"experiment": {
"name": self._rubicon_object.name,
"id": self._rubicon_object.id,
"created_id": self._rubicon_object.created_at,
"commit_hash": self._rubicon_object.commit_hash,
"tags": self._rubicon_object.tags,
"project": self._rubicon_object.project,
}
})
return super()._get_schema()
def test_create_experiment_with_auto_git():
with mock.patch("subprocess.run") as mock_run:
mock_run.return_value = MockCompletedProcess(stdout=b"test",
returncode=0)
rubicon = Rubicon("memory", "test-root", auto_git_enabled=True)
project = rubicon.create_project("Test Project A")
project.log_experiment()
expected = [
mock.call(["git", "rev-parse", "--git-dir"], capture_output=True),
mock.call(["git", "remote", "-v"], capture_output=True),
mock.call(["git", "rev-parse", "--abbrev-ref", "HEAD"],
capture_output=True),
mock.call(["git", "rev-parse", "HEAD"], capture_output=True),
]
assert mock_run.mock_calls == expected
rubicon.repository.filesystem.store = {}
def _get_schema(self):
"""Load the experiment with ID `self._experiment_id`
from the project named `self._project_name`.
"""
self._rubicon = Rubicon(persistence="filesystem",
root_dir=self._urlpath)
self._rubicon_object = self._rubicon.get_project(
self._project_name).experiment(self._experiment_id)
self._metadata.update({
"experiment": {
"name": self._rubicon_object.name,
"id": self._rubicon_object.id,
"created_id": self._rubicon_object.created_at,
"commit_hash": self._rubicon_object.commit_hash,
"tags": self._rubicon_object.tags,
"project": self._rubicon_object.project,
}
})
return super()._get_schema()