def test_create_dataset_invalid_storage_type(self, mock_config_file):
inv_manager = InventoryManager(mock_config_file[0])
auth = GitAuthor(name="username", email="*****@*****.**")
with pytest.raises(ValueError):
inv_manager.create_dataset("test", "test", "dataset1", "asdfdfgh",
description="my first dataset",
author=auth)
def test_create_dataset_that_exists(self, mock_config_file):
"""Test trying to create a labbook with a name that already exists locally"""
inv_manager = InventoryManager(mock_config_file[0])
auth = GitAuthor(name="username", email="*****@*****.**")
ds = inv_manager.create_dataset("test", "test", "dataset1", "gigantum_object_v1",
description="my first dataset",
author=auth)
with pytest.raises(ValueError):
inv_manager.create_dataset("test", "test", "dataset1", "gigantum_object_v1",
description="my first dataset",
author=auth)
def test_list_datasets_modified_on(self, mock_config_file):
"""Test list az datasets"""
inv_manager = InventoryManager(mock_config_file[0])
inv_manager.create_dataset("user1", "user1", "dataset2", "gigantum_object_v1", description="my dataset")
time.sleep(1)
inv_manager.create_dataset("user1", "user2", "a-dataset3", "gigantum_object_v1", description="my dataset")
time.sleep(1)
inv_manager.create_dataset("user1", "user1", "dataset12", "gigantum_object_v1", description="my dataset")
time.sleep(1)
inv_manager.create_dataset("user2", "user1", "dataset1", "gigantum_object_v1", description="my dataset")
datasets = inv_manager.list_datasets(username="******", sort_mode="modified_on")
assert len(datasets) == 3
assert datasets[0].name == 'dataset2'
assert datasets[1].name == 'a-dataset3'
assert datasets[2].name == 'dataset12'
# modify a repo
time.sleep(1.2)
ds = inv_manager.load_dataset('user1', 'user1', 'dataset2')
with open(os.path.join(ds.root_dir, "manifest", "test.txt"), 'wt') as tf:
tf.write("asdfasdf")
ds.git.add_all()
ds.git.commit("Changing the repo")
datasets = inv_manager.list_datasets(username="******", sort_mode="modified_on")
assert len(datasets) == 3
assert datasets[0].name == 'a-dataset3'
assert datasets[1].name == 'dataset12'
assert datasets[2].name == 'dataset2'
def create_datasets_to_list(mock_config_file):
inv_manager = InventoryManager(mock_config_file[0])
inv_manager.create_dataset("user1",
"user1",
"dataset2",
"gigantum_object_v1",
description="my dataset")
time.sleep(1.5)
inv_manager.create_dataset("user1",
"user2",
"a-dataset3",
"gigantum_object_v1",
description="my dataset")
time.sleep(1.5)
inv_manager.create_dataset("user1",
"user1",
"dataset12",
"gigantum_object_v1",
description="my dataset")
time.sleep(1.5)
inv_manager.create_dataset("user2",
"user1",
"dataset1",
"gigantum_object_v1",
description="my dataset")
yield mock_config_file
def test_success_export_then_import_different_users(
self, mock_config_file):
inv_manager = InventoryManager(mock_config_file[0])
ds = inv_manager.create_dataset('unittester', 'unittester',
'unittest-zip', 'gigantum_object_v1')
with tempfile.TemporaryDirectory() as tempd:
path = ZipExporter.export_dataset(ds.root_dir, tempd)
newds = ZipExporter.import_dataset(path, "unittester2",
"unittester2",
mock_config_file[0])
assert not os.path.exists(path)
assert 'unittester2' == InventoryManager(
mock_config_file[0]).query_owner(newds)
assert newds.is_repo_clean
assert newds.active_branch == 'master'
# Now try with same user as exporter
path2 = ZipExporter.export_dataset(ds.root_dir, tempd)
shutil.rmtree(ds.root_dir)
lb2 = ZipExporter.import_dataset(path2, "unittester", "unittester",
mock_config_file[0])
assert 'unittester' == InventoryManager(
mock_config_file[0]).query_owner(lb2)
assert lb2.is_repo_clean
assert lb2.active_branch == 'master'
def test_get_commits_behind(self, fixture_working_dir):
"""Test temporar field commitsBehind on dataset objects"""
im = InventoryManager(fixture_working_dir[0])
ds = im.create_dataset("default", "default", "test-ds", "gigantum_object_v1",
description="my first dataset",
author=GitAuthor(name="default", email="*****@*****.**"))
lb = im.create_labbook("default", "default", "test-lb")
im.link_dataset_to_labbook(f"{ds.root_dir}/.git", 'default', 'test-ds', lb)
query = """
{
labbook(owner: "default", name:"test-lb")
{
linkedDatasets{
name
commitsBehind
}
}
}
"""
r = fixture_working_dir[2].execute(query)
assert 'errors' not in r
assert r['data']['labbook']['linkedDatasets'][0]['name'] == 'test-ds'
assert r['data']['labbook']['linkedDatasets'][0]['commitsBehind'] == 0
ds.write_readme("test contents to make a commit")
r = fixture_working_dir[2].execute(query)
assert 'errors' not in r
assert r['data']['labbook']['linkedDatasets'][0]['name'] == 'test-ds'
assert r['data']['labbook']['linkedDatasets'][0]['commitsBehind'] == 1
def test_is_dataset_modified_date(self, mock_config_file):
"""Test getting the modified date"""
im = InventoryManager(mock_config_file[0])
ds = im.create_dataset("test",
"test",
"dataset1",
"gigantum_object_v1",
description="my first dataset",
author=GitAuthor(name="test",
email="*****@*****.**"))
modified_1 = ds.modified_on
time.sleep(3)
helper_modify_dataset(ds)
modified_2 = ds.modified_on
assert modified_2 > modified_1
assert modified_1.microsecond == 0
assert modified_1.tzname() == "UTC"
assert modified_2.microsecond == 0
assert modified_2.tzname() == "UTC"
assert (datetime.datetime.now(datetime.timezone.utc) -
modified_1).total_seconds() < 10
assert (datetime.datetime.now(datetime.timezone.utc) -
modified_2).total_seconds() < 10
def test_delete_labbook_linked_dataset(self, mock_config_file):
"""Test trying to create a labbook with a name that already exists locally"""
inv_manager = InventoryManager(mock_config_file[0])
inv_manager.create_labbook("test",
"test",
"labbook1",
description="my first labbook")
lb = inv_manager.load_labbook("test", "test", "labbook1")
auth = GitAuthor(name="test", email="*****@*****.**")
ds = inv_manager.create_dataset("test",
"test",
"dataset1",
"gigantum_object_v1",
description="my first dataset",
author=auth)
inv_manager.link_dataset_to_labbook(f"{ds.root_dir}/.git", "test",
"dataset1", lb)
dataset_delete_jobs = inv_manager.delete_labbook(
"test", "test", "labbook1")
assert len(dataset_delete_jobs) == 1
assert dataset_delete_jobs[0].namespace == "test"
assert dataset_delete_jobs[0].name == "dataset1"
with pytest.raises(InventoryException):
inv_manager.load_labbook("test", "test", "labbook1")
def test_check_and_import_dataset(self, mock_config_file):
im = InventoryManager(mock_config_file[0])
ds = im.create_dataset('default',
'default',
"dataset100",
storage_type="gigantum_object_v1",
description="100")
# Fake publish to a local bare repo
_MOCK_create_remote_repo2(ds, 'test', None, None)
remote_url = ds.remote
im.delete_dataset('default', 'default', "dataset100")
with pytest.raises(InventoryException):
im.load_dataset('default', 'default', "dataset100")
kwargs = {
'logged_in_username': "******",
'dataset_owner': "default",
'dataset_name': "dataset100",
'remote_url': remote_url,
'config_file': mock_config_file[0]
}
gtmcore.dispatcher.dataset_jobs.check_and_import_dataset(**kwargs)
ds = im.load_dataset('default', 'default', "dataset100")
assert ds.name == 'dataset100'
assert ds.namespace == 'default'
def test_delete_local_dataset(self, fixture_working_dir):
im = InventoryManager(fixture_working_dir[0])
ds = im.create_dataset('default',
'default',
"dataset100",
storage_type="gigantum_object_v1",
description="100")
dataset_dir = ds.root_dir
assert os.path.exists(dataset_dir) is True
query = """
mutation myMutation($owner: String!, $datasetName: String!) {
deleteDataset(input: {owner: $owner, datasetName: $datasetName, local: true, remote: false}) {
localDeleted
remoteDeleted
}
}
"""
variables = {"datasetName": "dataset100", "owner": "default"}
result = fixture_working_dir[2].execute(query,
variable_values=variables)
assert "errors" not in result
assert result['data']['deleteDataset']['localDeleted'] is True
assert result['data']['deleteDataset']['remoteDeleted'] is False
assert os.path.exists(dataset_dir) is False
def test_get_linked_datasets(self, mock_labbook):
inv_manager = InventoryManager(mock_labbook[0])
lb = mock_labbook[2]
datasets = inv_manager.get_linked_datasets(lb)
assert len(datasets) == 0
ds = inv_manager.create_dataset("test",
"test",
"dataset100",
"gigantum_object_v1",
description="my dataset")
# Fake publish to a local bare repo
_MOCK_create_remote_repo2(ds, 'test', None, None)
assert os.path.exists(os.path.join(lb.root_dir,
'.gitmodules')) is False
inv_manager.link_dataset_to_labbook(ds.remote, 'test', 'dataset100',
lb)
assert os.path.exists(os.path.join(lb.root_dir, '.gitmodules')) is True
dataset_submodule_dir = os.path.join(lb.root_dir, '.gigantum',
'datasets', 'test', 'dataset100')
assert os.path.exists(dataset_submodule_dir) is True
assert os.path.exists(os.path.join(dataset_submodule_dir,
'.gigantum')) is True
datasets = inv_manager.get_linked_datasets(lb)
assert len(datasets) == 1
assert datasets[0].name == ds.name
assert datasets[0].namespace == ds.namespace
def test_delete_dataset_files_errors(self, fixture_working_dir, snapshot):
im = InventoryManager(fixture_working_dir[0])
ds = im.create_dataset('default',
'default',
"dataset-delete-2",
storage_type="gigantum_object_v1",
description="testing delete")
m = Manifest(ds, 'default')
helper_append_file(m.cache_mgr.cache_root, m.dataset_revision,
"test1.txt", "asdfadfsdf")
helper_append_file(m.cache_mgr.cache_root, m.dataset_revision,
"test2.txt", "fdsfgfd")
m.sweep_all_changes()
revision = m.dataset_revision
assert os.path.exists(
os.path.join(m.cache_mgr.cache_root, revision,
"test1.txt")) is True
assert os.path.exists(
os.path.join(m.cache_mgr.cache_root, revision,
"test2.txt")) is True
query = """
mutation myMutation {
deleteDatasetFiles(input: {datasetOwner: "default", datasetName: "dataset-delete-2",
keys: ["testdfdfdfdf.txt"]}) {
success
}
}
"""
result = fixture_working_dir[2].execute(query)
assert 'errors' in result
def test_link_unlink_dataset(self, mock_labbook):
inv_manager = InventoryManager(mock_labbook[0])
lb = mock_labbook[2]
ds = inv_manager.create_dataset("test", "test", "dataset100", "gigantum_object_v1", description="my dataset")
# Fake publish to a local bare repo
_MOCK_create_remote_repo2(ds, 'test', None, None)
assert os.path.exists(os.path.join(lb.root_dir, '.gitmodules')) is False
inv_manager.link_dataset_to_labbook(ds.remote, 'test', 'dataset100', lb)
assert os.path.exists(os.path.join(lb.root_dir, '.gitmodules')) is True
dataset_submodule_dir = os.path.join(lb.root_dir, '.gigantum', 'datasets', 'test', 'dataset100')
assert os.path.exists(dataset_submodule_dir) is True
assert os.path.exists(os.path.join(dataset_submodule_dir, '.gigantum')) is True
inv_manager.unlink_dataset_from_labbook('test', 'dataset100', lb)
dataset_submodule_dir = os.path.join(lb.root_dir, '.gigantum', 'datasets', 'test', 'dataset100')
assert os.path.exists(dataset_submodule_dir) is False
assert os.path.exists(os.path.join(dataset_submodule_dir, '.gigantum')) is False
with open(os.path.join(lb.root_dir, '.gitmodules'), 'rt') as mf:
data = mf.read()
assert len(data) == 0
def test_update_unmanaged_dataset_local_errors(
self, fixture_working_dir_dataset_tests):
im = InventoryManager(fixture_working_dir_dataset_tests[0])
ds = im.create_dataset('default',
'default',
"adataset",
storage_type="local_filesystem",
description="100")
dataset_dir = ds.root_dir
assert os.path.exists(dataset_dir) is True
flask.g.access_token = "asdf"
flask.g.id_token = "1234"
query = """
mutation myMutation{
updateUnmanagedDataset(input: {datasetOwner: "default", datasetName: "adataset"}) {
backgroundJobKey
}
}
"""
result = fixture_working_dir_dataset_tests[2].execute(query)
assert "errors" in result
# not configured
query = """
mutation myMutation{
updateUnmanagedDataset(input: {datasetOwner: "default", datasetName: "adataset",
fromLocal: true}) {
backgroundJobKey
}
}
"""
result = fixture_working_dir_dataset_tests[2].execute(query)
assert "errors" in result
def test_change_dataset_name(self, mock_config_file):
inv_manager = InventoryManager(mock_config_file[0])
auth = GitAuthor(name="username", email="*****@*****.**")
ds = inv_manager.create_dataset("test",
"test",
"dataset1",
"gigantum_object_v1",
description="my first dataset",
author=auth)
root_dir1 = ds.root_dir
assert ds.name == 'dataset1'
ds.name = 'dataset-updated'
root_dir2 = ds.root_dir
parts1 = root_dir1.split(os.path.sep)
parts2 = root_dir2.split(os.path.sep)
assert parts1[0] == parts2[0]
assert parts1[1] == parts2[1]
assert parts1[2] == parts2[2]
assert parts1[3] == parts2[3]
assert parts1[4] == parts2[4]
assert parts1[5] == parts2[5]
assert parts1[6] == 'dataset1'
assert parts2[6] == 'dataset-updated'
def test_verify_unmanaged_dataset(self, fixture_working_dir_dataset_tests):
im = InventoryManager(fixture_working_dir_dataset_tests[0])
ds = im.create_dataset('default',
'default',
"adataset",
storage_type="local_filesystem",
description="100")
dataset_dir = ds.root_dir
assert os.path.exists(dataset_dir) is True
flask.g.access_token = "asdf"
flask.g.id_token = "1234"
# configure backend and local dir
working_dir = fixture_working_dir_dataset_tests[1]
ds.backend.set_default_configuration('default', 'asdf', '1234')
current_config = ds.backend_config
current_config['Data Directory'] = "test_dir"
ds.backend_config = current_config
test_dir = os.path.join(working_dir, "local_data", "test_dir")
os.makedirs(test_dir)
with open(os.path.join(test_dir, "test.txt"), 'wt') as temp:
temp.write(f'dummy data: asdfasdf')
query = """
mutation myMutation{
verifyDataset(input: {datasetOwner: "default", datasetName: "adataset"}) {
backgroundJobKey
}
}
"""
result = fixture_working_dir_dataset_tests[2].execute(query)
assert "errors" not in result
assert "rq:job" in result['data']['verifyDataset']['backgroundJobKey']
def test_is_dataset_is_managed(self, mock_config_file):
"""Test getting the create date, both when stored in the buildinfo file and when using git fallback"""
im = InventoryManager(mock_config_file[0])
ds = im.create_dataset("test", "test", "dataset1", "gigantum_object_v1",
description="my first dataset",
author=GitAuthor(name="test", email="*****@*****.**"))
assert ds.is_managed() is True
def test_load_one_dataset(self, fixture_working_dir):
im = InventoryManager(fixture_working_dir[0])
im.create_dataset("default",
"default",
"dataset1",
storage_type="gigantum_object_v1",
description="a dataset")
im.create_dataset("default",
"default",
"dataset2",
storage_type="gigantum_object_v1",
description="another dataset")
loader = FetchLoader()
key = f"dataset&default&default&dataset1"
promise1 = loader.load(key)
assert isinstance(promise1, Promise)
assert promise1.get() is None
def fixture_single_dataset():
"""A pytest fixture that creates a temporary working directory, a config file to match, creates the schema,
and populates the environment component repository.
Class scope modifier attached
"""
# Create temp dir
config_file, temp_dir = _create_temp_work_dir()
# Create user identity
insert_cached_identity(temp_dir)
# Create test client
schema = graphene.Schema(query=LabbookQuery, mutation=LabbookMutations)
# Create a bunch of lab books
im = InventoryManager(config_file)
ds = im.create_dataset('default',
'default',
"test-dataset",
storage_type="gigantum_object_v1",
description="Cats 2")
m = Manifest(ds, 'default')
cm_class = get_cache_manager_class(ds.client_config)
cache_mgr = cm_class(ds, 'default')
revision = ds.git.repo.head.commit.hexsha
os.makedirs(os.path.join(cache_mgr.cache_root, revision, "other_dir"))
helper_append_file(cache_mgr.cache_root, revision, "test1.txt", "asdfasdf")
helper_append_file(cache_mgr.cache_root, revision, "test2.txt", "rtg")
helper_append_file(cache_mgr.cache_root, revision, "test3.txt", "wer")
helper_append_file(cache_mgr.cache_root, revision, "other_dir/test4.txt",
"dfasdfhfgjhg")
helper_append_file(cache_mgr.cache_root, revision, "other_dir/test5.txt",
"fdghdfgsa")
m.update()
with patch.object(Configuration, 'find_default_config',
lambda self: config_file):
# Load User identity into app context
app = Flask("lmsrvlabbook")
app.config["LABMGR_CONFIG"] = Configuration()
app.config["LABMGR_ID_MGR"] = get_identity_manager(Configuration())
with app.app_context():
# within this block, current_app points to app. Set current user explicitly (this is done in the middleware)
flask.g.user_obj = app.config["LABMGR_ID_MGR"].get_user_profile()
# Create a test client
client = Client(schema,
middleware=[DataloaderMiddleware()],
context_value=ContextMock())
yield config_file, temp_dir, client, ds, cache_mgr
# Remove the temp_dir
shutil.rmtree(temp_dir)
def mutate_and_get_payload(cls,
root,
info,
name,
description,
storage_type,
client_mutation_id=None):
username = get_logged_in_username()
inv_manager = InventoryManager()
inv_manager.create_dataset(username=username,
owner=username,
dataset_name=name,
description=description,
storage_type=storage_type,
author=get_logged_in_author())
return CreateDataset(dataset=Dataset(
id="{}&{}".format(username, name), name=name, owner=username))
def test_create_rollback_branch_remove_linked_dataset(
self, mock_create_labbooks):
""" test creating a rollback branch that removes a linked dataset"""
lb, client = mock_create_labbooks[0], mock_create_labbooks[1]
im = InventoryManager(config_file=lb.client_config.config_file)
ds = im.create_dataset(UT_USERNAME,
UT_USERNAME,
'test-ds',
storage_type='gigantum_object_v1')
rollback_to = lb.git.commit_hash
# Link dataset to project
im.link_dataset_to_labbook(f"{ds.root_dir}/.git", UT_USERNAME, ds.name,
lb)
dataset_dir = os.path.join(lb.root_dir, '.gigantum', 'datasets',
UT_USERNAME, 'test-ds')
assert os.path.exists(dataset_dir) is True
q = f"""
mutation makeFeatureBranch {{
createExperimentalBranch(input: {{
owner: "{UT_USERNAME}",
labbookName: "{UT_LBNAME}",
branchName: "rollback-branch",
revision: "{rollback_to}",
description: "testing rollback",
}}) {{
labbook{{
name
activeBranchName
description
branches {{
branchName
}}
linkedDatasets{{
name
}}
}}
}}
}}
"""
r = client.execute(q)
assert 'errors' not in r
assert r['data']['createExperimentalBranch']['labbook'][
'activeBranchName'] == 'rollback-branch'
assert r['data']['createExperimentalBranch']['labbook'][
'description'] == "testing rollback"
assert r['data']['createExperimentalBranch']['labbook'][
'linkedDatasets'] == []
assert lb.is_repo_clean
assert os.path.exists(dataset_dir) is False
def test_delete_remote_dataset_gitlab_error(self, fixture_working_dir):
responses.add(responses.GET,
'https://usersrv.gigantum.io/key',
json={'key': 'afaketoken'},
status=200)
responses.add(
responses.GET,
'https://repo.gigantum.io/api/v4/projects/default%2Fdataset100',
json=[{
"id": 27,
"description": "",
}],
status=200)
responses.add(responses.DELETE,
'https://api.gigantum.com/object-v1/default/dataset100',
json=[{
'status': 'queueing for delete'
}],
status=200)
responses.add(
responses.DELETE,
'https://repo.gigantum.io/api/v4/projects/default%2Fdataset100',
json={"message": "fail"},
status=400)
flask.g.access_token = "asdfasdfasdfasdf"
flask.g.id_token = "ghjfghjfghjfghj"
im = InventoryManager(fixture_working_dir[0])
ds = im.create_dataset('default',
'default',
"dataset100",
storage_type="gigantum_object_v1",
description="100")
dataset_dir = ds.root_dir
assert os.path.exists(dataset_dir) is True
query = """
mutation myMutation($owner: String!, $datasetName: String!) {
deleteDataset(input: {owner: $owner, datasetName: $datasetName, local: false, remote: true}) {
localDeleted
remoteDeleted
}
}
"""
variables = {"datasetName": "dataset100", "owner": "default"}
result = fixture_working_dir[2].execute(query,
variable_values=variables)
assert "errors" in result
assert result['errors'][0][
'message'] == "Failed to remove remote repository"
assert os.path.exists(dataset_dir) is True
def test_delete_dataset_while_linked(self, mock_config_file):
inv_manager = InventoryManager(mock_config_file[0])
auth = GitAuthor(name="test", email="*****@*****.**")
lb = inv_manager.create_labbook("test",
"test",
"labbook1",
description="my first labbook")
ds = inv_manager.create_dataset("test",
"test",
"dataset1",
"gigantum_object_v1",
description="my first dataset",
author=auth)
ds_root_dir = ds.root_dir
lb_root_dir = lb.root_dir
assert os.path.exists(ds_root_dir) is True
assert os.path.exists(lb_root_dir) is True
# Link dataset
inv_manager.link_dataset_to_labbook(f"{ds_root_dir}/.git", "test",
"dataset1", lb)
m = Manifest(ds, 'test')
helper_append_file(m.cache_mgr.cache_root, m.dataset_revision,
"test1.txt", "asdfasdf")
helper_append_file(m.cache_mgr.cache_root, m.dataset_revision,
"test2.txt", "dfg")
assert os.path.exists(
os.path.join(m.cache_mgr.cache_root, m.dataset_revision,
"test1.txt")) is True
assert os.path.exists(
os.path.join(m.cache_mgr.cache_root, m.dataset_revision,
"test2.txt")) is True
dataset_delete_job = inv_manager.delete_dataset(
"test", "test", "dataset1")
assert os.path.exists(ds_root_dir) is False
assert os.path.exists(lb_root_dir) is True
assert os.path.exists(m.cache_mgr.cache_root) is True
assert dataset_delete_job.namespace == "test"
assert dataset_delete_job.name == "dataset1"
assert dataset_delete_job.cache_root == m.cache_mgr.cache_root
jobs.clean_dataset_file_cache("test",
dataset_delete_job.namespace,
dataset_delete_job.name,
dataset_delete_job.cache_root,
config_file=mock_config_file[0])
assert os.path.exists(m.cache_mgr.cache_root) is True
cache_base, _ = m.cache_mgr.cache_root.rsplit(os.path.sep, 1)
assert os.path.exists(cache_base) is True
def mock_dataset_with_cache_dir():
"""A pytest fixture that creates a dataset in a temp working dir. Deletes directory after test"""
conf_file, working_dir = _create_temp_work_dir()
with patch.object(Configuration, 'find_default_config', lambda self: conf_file):
im = InventoryManager(conf_file)
ds = im.create_dataset(USERNAME, USERNAME, 'dataset-1', description="my dataset 1",
storage_type="gigantum_object_v1")
yield ds, working_dir, ds.git.repo.head.commit.hexsha
shutil.rmtree(working_dir)
def mock_create_dataset(fixture_working_dir):
# Create a dataset in the temporary directory
im = InventoryManager(fixture_working_dir[0])
ds = im.create_dataset("default",
"default",
"dataset1",
storage_type="gigantum_object_v1",
description="my dataset")
# name of the config file, temporary working directory, the schema
yield fixture_working_dir
def test_download_dataset_files_linked(self, fixture_working_dir,
snapshot):
def dispatcher_mock(self, function_ref, kwargs, metadata, persist):
assert kwargs['logged_in_username'] == 'default'
assert kwargs['access_token'] == 'asdf'
assert kwargs['id_token'] == '1234'
assert kwargs['dataset_owner'] == 'default'
assert kwargs['dataset_name'] == 'dataset100'
assert kwargs['labbook_owner'] == 'default'
assert kwargs['labbook_name'] == 'test-lb'
assert kwargs['all_keys'] is None
assert kwargs['keys'] == ["test1.txt"]
assert persist is True
assert metadata[
'dataset'] == 'default|default|test-lb|LINKED|default|default|dataset100'
assert metadata['labbook'] == 'default|default|test-lb'
assert metadata['method'] == 'download_dataset_files'
return JobResponseMock(
"rq:job:00923477-d46b-479c-ad0c-2b66f90b6b10")
im = InventoryManager(fixture_working_dir[0])
ds = im.create_dataset('default',
'default',
"dataset100",
storage_type="gigantum_object_v1",
description="100")
lb = im.create_labbook('default',
'default',
"test-lb",
description="tester")
im.link_dataset_to_labbook(f"{ds.root_dir}/.git", 'default',
'dataset100', lb)
flask.g.access_token = "asdf"
flask.g.id_token = "1234"
with patch.object(Dispatcher, 'dispatch_task', dispatcher_mock):
query = """
mutation myMutation {
downloadDatasetFiles(input: {datasetOwner: "default", datasetName: "dataset100", keys: ["test1.txt"], labbookOwner: "default", labbookName: "test-lb"}){
backgroundJobKey
}
}
"""
r = fixture_working_dir[2].execute(query)
assert 'errors' not in r
assert isinstance(
r['data']['downloadDatasetFiles']['backgroundJobKey'], str)
assert "rq:" in r['data']['downloadDatasetFiles'][
'backgroundJobKey']
def test_load_many_dataset(self, fixture_working_dir):
im = InventoryManager(fixture_working_dir[0])
im.create_dataset("default",
"default",
"dataset1",
storage_type="gigantum_object_v1",
description="a dataset")
im.create_dataset("default",
"default",
"dataset2",
storage_type="gigantum_object_v1",
description="another dataset")
im.create_dataset("default",
"tester_user",
"dataset3",
storage_type="gigantum_object_v1",
description="yet another dataset")
loader = FetchLoader()
keys = [
"dataset&default&default&dataset1",
"dataset&default&default&dataset2",
"dataset&default&tester_user&dataset3"
]
promise1 = loader.load_many(keys)
assert isinstance(promise1, Promise)
promise1.get()
def test_load_many(self, fixture_working_dir):
"""Test loading many labbooks"""
im = InventoryManager(fixture_working_dir[0])
im.create_dataset("default",
"default",
"dataset1",
storage_type="gigantum_object_v1",
description="a dataset")
im.create_dataset("default",
"default",
"dataset2",
storage_type="gigantum_object_v1",
description="another dataset")
im.create_dataset("default",
"tester_user",
"dataset3",
storage_type="gigantum_object_v1",
description="yet another dataset")
loader = DatasetLoader()
keys = [
"default&default&dataset1", "default&default&dataset2",
"default&tester_user&dataset3"
]
promise1 = loader.load_many(keys)
assert isinstance(promise1, Promise)
ds_list = promise1.get()
assert ds_list[0].name == "dataset1"
assert ds_list[0].description == "a dataset"
assert ds_list[1].name == "dataset2"
assert ds_list[1].description == "another dataset"
assert ds_list[2].name == "dataset3"
assert ds_list[2].description == "yet another dataset"
def test_load_one(self, fixture_working_dir):
"""Test loading 1 dataset"""
im = InventoryManager(fixture_working_dir[0])
im.create_dataset("default",
"default",
"dataset1",
storage_type="gigantum_object_v1",
description="a dataset")
im.create_dataset("default",
"default",
"dataset2",
storage_type="gigantum_object_v1",
description="another dataset")
loader = DatasetLoader()
key = f"default&default&dataset1"
promise1 = loader.load(key)
assert isinstance(promise1, Promise)
lb = promise1.get()
assert lb.name == "dataset1"
assert lb.description == "a dataset"
def test_change_dataset_name_errors(self, mock_config_file):
inv_manager = InventoryManager(mock_config_file[0])
auth = GitAuthor(name="username", email="*****@*****.**")
ds = inv_manager.create_dataset("test", "test", "dataset1", "gigantum_object_v1",
description="my first dataset",
author=auth)
with pytest.raises(ValueError):
ds.name = 'dataset_updated'
ds.name = 'd' * 100
with pytest.raises(ValueError):
ds.name = 'd' * 101
