def test_delete_dataset_files_errors(self, fixture_working_dir, snapshot):
im = InventoryManager(fixture_working_dir[0])
ds = im.create_dataset('default',
'default',
"dataset-delete-2",
storage_type="gigantum_object_v1",
description="testing delete")
m = Manifest(ds, 'default')
helper_append_file(m.cache_mgr.cache_root, m.dataset_revision,
"test1.txt", "asdfadfsdf")
helper_append_file(m.cache_mgr.cache_root, m.dataset_revision,
"test2.txt", "fdsfgfd")
m.sweep_all_changes()
revision = m.dataset_revision
assert os.path.exists(
os.path.join(m.cache_mgr.cache_root, revision,
"test1.txt")) is True
assert os.path.exists(
os.path.join(m.cache_mgr.cache_root, revision,
"test2.txt")) is True
query = """
mutation myMutation {
deleteDatasetFiles(input: {datasetOwner: "default", datasetName: "dataset-delete-2",
keys: ["testdfdfdfdf.txt"]}) {
success
}
}
"""
result = fixture_working_dir[2].execute(query)
assert 'errors' in result
def test_empty_package_counts(self, fixture_working_dir_env_repo_scoped,
snapshot):
"""Test getting the a LabBook's package manager dependencies"""
# Create labbook
im = InventoryManager(fixture_working_dir_env_repo_scoped[0])
lb = im.create_labbook("default",
"default",
"labbook4",
description="my first labbook10000")
query = """
{
labbook(owner: "default", name: "labbook4") {
overview {
numAptPackages
numConda2Packages
numConda3Packages
numPipPackages
numCustomDependencies
}
}
}
"""
snapshot.assert_match(
fixture_working_dir_env_repo_scoped[2].execute(query))
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
migrated = False
with lb.lock():
t0 = time.time()
workflow = LabbookWorkflow(lb)
migrated = workflow.migrate()
tf = time.time()
if migrated:
logger.info(f"Migrated {str(lb)} in {tf-t0:.2}sec")
else:
logger.info(f"No migration needed for {str(lb)}")
return MigrateLabbookSchema(
Labbook(id=f"{owner}&{labbook_name}",
name=labbook_name,
owner=owner))
def test_update_unmanaged_dataset_local_errors(
self, fixture_working_dir_dataset_tests):
im = InventoryManager(fixture_working_dir_dataset_tests[0])
ds = im.create_dataset('default',
'default',
"adataset",
storage_type="local_filesystem",
description="100")
dataset_dir = ds.root_dir
assert os.path.exists(dataset_dir) is True
flask.g.access_token = "asdf"
flask.g.id_token = "1234"
query = """
mutation myMutation{
updateUnmanagedDataset(input: {datasetOwner: "default", datasetName: "adataset"}) {
backgroundJobKey
}
}
"""
result = fixture_working_dir_dataset_tests[2].execute(query)
assert "errors" in result
# not configured
query = """
mutation myMutation{
updateUnmanagedDataset(input: {datasetOwner: "default", datasetName: "adataset",
fromLocal: true}) {
backgroundJobKey
}
}
"""
result = fixture_working_dir_dataset_tests[2].execute(query)
assert "errors" in result
def migrate(self) -> bool:
""" Migrate the given LabBook to the most recent schema AND branch version.
Returns:
Boolean indicating whether a migration was performed (False if already up-to-date)
"""
if self.repository.schema == CURRENT_LABBOOK_SCHEMA:
logger.info(f"{str(self.labbook)} already migrated.")
return False
if 'gm.workspace' not in BranchManager(self.labbook).active_branch:
raise GitWorkflowException(
'Must be on a gm.workspace branch to migrate')
im = InventoryManager(self.labbook.client_config.config_file)
gitworkflows_utils.migrate_labbook_branches(self.labbook)
self.repository = im.load_labbook_from_directory(self.labbook.root_dir)
gitworkflows_utils.migrate_labbook_schema(self.labbook)
self.repository = im.load_labbook_from_directory(self.labbook.root_dir)
gitworkflows_utils.migrate_labbook_untracked_space(self.labbook)
self.repository = im.load_labbook_from_directory(self.labbook.root_dir)
# Pushes up the new master branch
if self.repository.has_remote:
self.sync(username='')
return True
def build_lb_image_for_env_conda(mock_config_with_repo):
"""A fixture that installs an old version of matplotlib and latest version of requests to increase code coverage"""
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook('unittester', 'unittester', "containerunittestbookenvconda",
description="Testing environment functions.")
cm = ComponentManager(lb)
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
cm.add_packages('conda3', [{'package': 'python-coveralls', 'version': '2.7.0'}])
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
client = get_docker_client()
client.containers.prune()
try:
lb, docker_image_id = ContainerOperations.build_image(labbook=lb, username="******")
yield lb, 'unittester'
finally:
shutil.rmtree(lb.root_dir)
try:
client.images.remove(docker_image_id, force=True, noprune=False)
except:
pass
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
confirm,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
if confirm:
logger.info(f"Deleting {str(lb)}...")
try:
lb, stopped = ContainerOperations.stop_container(
labbook=lb, username=username)
except OSError as e:
logger.warning(e)
lb, docker_removed = ContainerOperations.delete_image(
labbook=lb, username=username)
if not docker_removed:
raise ValueError(
f'Cannot delete docker image for {str(lb)} - unable to delete Project from disk'
)
datasets_to_schedule = InventoryManager().delete_labbook(
username, owner, labbook_name)
# Schedule jobs to clean the file cache for any linked datasets (if no other references exist)
for cleanup_job in datasets_to_schedule:
# Schedule Job to clear file cache if dataset is no longer in use
job_metadata = {'method': 'clean_dataset_file_cache'}
job_kwargs = {
'logged_in_username': username,
'dataset_owner': cleanup_job.namespace,
'dataset_name': cleanup_job.name,
'cache_location': cleanup_job.cache_root
}
dispatcher = Dispatcher()
job_key = dispatcher.dispatch_task(
jobs.clean_dataset_file_cache,
metadata=job_metadata,
kwargs=job_kwargs)
logger.info(
f"Dispatched clean_dataset_file_cache({ cleanup_job.namespace}/{cleanup_job.name})"
f" to Job {job_key}")
# Verify Delete worked
if os.path.exists(lb.root_dir):
logger.error(
f'Deleted {str(lb)} but root directory {lb.root_dir} still exists!'
)
return DeleteLabbook(success=False)
else:
return DeleteLabbook(success=True)
else:
logger.info(f"Dry run in deleting {str(lb)} -- not deleted.")
return DeleteLabbook(success=False)
def mutate_and_get_payload(cls,
root,
info,
owner,
dataset_name,
description,
client_mutation_id=None):
username = get_logged_in_username()
ds = InventoryManager().load_dataset(username,
owner,
dataset_name,
author=get_logged_in_author())
ds.description = description
with ds.lock():
ds.git.add(os.path.join(ds.root_dir, '.gigantum/gigantum.yaml'))
commit = ds.git.commit('Updating description')
adr = ActivityDetailRecord(ActivityDetailType.LABBOOK, show=False)
adr.add_value('text/plain',
f"Updated Dataset description: {description}")
ar = ActivityRecord(ActivityType.LABBOOK,
message="Updated Dataset description",
linked_commit=commit.hexsha,
tags=["dataset"],
show=False)
ar.add_detail_object(adr)
ars = ActivityStore(ds)
ars.create_activity_record(ar)
return SetDatasetDescription(
updated_dataset=Dataset(owner=owner, name=dataset_name))
def savehook(username, owner, labbook_name):
try:
changed_file = request.args.get('file')
jupyter_token = request.args.get('jupyter_token')
logger.debug(f"Received save hook for {changed_file} in {username}/{owner}/{labbook_name}")
redis_conn = redis.Redis(db=1)
lb_jupyter_token_key = '-'.join(['gmlb', username, owner, labbook_name, 'jupyter-token'])
lb_active_key = f"{'|'.join([username, owner, labbook_name])}&is-busy*"
r = redis_conn.get(lb_jupyter_token_key.encode())
if r is None:
logger.error(f"Could not find jupyter token for {username}/{owner}/{labbook_name}")
abort(400)
if r.decode() != jupyter_token:
raise ValueError("Incoming jupyter token must be valid")
if len(redis_conn.keys(lb_active_key.encode())) > 0:
# A kernel in this project is still active. Don't save auto-commit because it can blow up the
# repository size depending on what the user is doing
logger.info(f"Skipping jupyter savehook for {username}/{owner}/{labbook_name} due to active kernel")
return 'success'
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with lb.lock():
lb.sweep_uncommitted_changes()
logger.info(f"Jupyter save hook saved {changed_file} from {str(lb)}")
return 'success'
except Exception as err:
logger.error(err)
return abort(400)
def test_package_query_with_errors_conda(
self, snapshot, fixture_working_dir_env_repo_scoped):
"""Test querying for package info"""
# Create labbook
im = InventoryManager(fixture_working_dir_env_repo_scoped[0])
lb = im.create_labbook("default",
"default",
"labbook5conda",
description="my first labbook10000")
query = """
{
labbook(owner: "default", name: "labbook5conda"){
id
checkPackages(packageInput: [
{manager: "conda3", package: "cdutil", version:"8.1"},
{manager: "conda3", package: "nltk", version:"100.00"},
{manager: "conda3", package: "python-coveralls", version:""},
{manager: "conda3", package: "thisshouldtotallyfail", version:"1.0"},
{manager: "conda3", package: "notarealpackage", version:""}]){
id
manager
package
version
latestVersion
description
isValid
}
}
}
"""
snapshot.assert_match(
fixture_working_dir_env_repo_scoped[2].execute(query))
def test_package_query(self, snapshot,
fixture_working_dir_env_repo_scoped):
"""Test querying for package info"""
im = InventoryManager(fixture_working_dir_env_repo_scoped[0])
lb = im.create_labbook("default",
"default",
"labbook6",
description="my first labbook10000")
query = """
{
labbook(owner: "default", name: "labbook6"){
id
checkPackages(packageInput: [
{manager: "pip", package: "gtmunit1", version:"0.2.4"},
{manager: "pip", package: "gtmunit2", version:""}]){
id
manager
package
version
isValid
}
}
}
"""
snapshot.assert_match(
fixture_working_dir_env_repo_scoped[2].execute(query))
def mutate_and_process_upload(cls, info, upload_file_path, upload_filename,
**kwargs):
if not upload_file_path:
logger.error('No file uploaded')
raise ValueError('No file uploaded')
username = get_logged_in_username()
owner = kwargs.get('owner')
dataset_name = kwargs.get('dataset_name')
file_path = kwargs.get('file_path')
try:
ds = InventoryManager().load_dataset(username,
owner,
dataset_name,
author=get_logged_in_author())
with ds.lock():
if not os.path.abspath(upload_file_path):
raise ValueError(
f"Source file `{upload_file_path}` not an absolute path"
)
if not os.path.isfile(upload_file_path):
raise ValueError(
f"Source file does not exist at `{upload_file_path}`")
manifest = Manifest(ds, username)
full_dst = manifest.get_abs_path(file_path)
# If file (hard link) already exists, remove it first so you don't write to all files with same content
if os.path.isfile(full_dst):
os.remove(full_dst)
full_dst_base = os.path.dirname(full_dst)
if not os.path.isdir(full_dst_base):
pathlib.Path(full_dst_base).mkdir(parents=True,
exist_ok=True)
shutil.move(upload_file_path, full_dst)
file_info = manifest.gen_file_info(file_path)
finally:
try:
logger.debug(f"Removing temp file {upload_file_path}")
os.remove(upload_file_path)
except FileNotFoundError:
pass
# Create data to populate edge
create_data = {
'owner': owner,
'name': dataset_name,
'key': file_info['key'],
'_file_info': file_info
}
# TODO: Fix cursor implementation. this currently doesn't make sense when adding edges
cursor = base64.b64encode(f"{0}".encode('utf-8'))
return AddDatasetFile(new_dataset_file_edge=DatasetFileConnection.Edge(
node=DatasetFile(**create_data), cursor=cursor))
def mutate_and_get_payload(cls,
root,
info,
owner,
dataset_name,
transaction_id,
cancel=False,
rollback=False,
client_mutation_id=None):
username = get_logged_in_username()
ds = InventoryManager().load_dataset(username,
owner,
dataset_name,
author=get_logged_in_author())
with ds.lock():
if cancel and rollback:
logger.warning(
f"Cancelled tx {transaction_id}, doing git reset")
# TODO: Add ability to reset
else:
logger.info(
f"Done batch upload {transaction_id}, cancelled={cancel}")
if cancel:
logger.warning("Sweeping aborted batch upload.")
m = "Cancelled upload `{transaction_id}`. " if cancel else ''
# Sweep up and process all files added during upload
manifest = Manifest(ds, username)
manifest.sweep_all_changes(upload=True, extra_msg=m)
return CompleteDatasetUploadTransaction(success=True)
def test_remove_secrets_entry(self, fixture_working_dir_env_repo_scoped):
client = fixture_working_dir_env_repo_scoped[2]
im = InventoryManager(fixture_working_dir_env_repo_scoped[0])
lb = im.create_labbook("default", "default",
"unittest-mutation-remove-secret")
SecretStore(lb, "default")['remove.key'] = '/mnt/nowhere'
SecretStore(lb, "default")['absent.key'] = '/mnt/nowhere2'
query = """
mutation remove {
removeSecretsEntry(input: {
owner: "default",
labbookName: "unittest-mutation-remove-secret",
filename: "remove.key",
}) {
environment {
secretsFileMapping {
edges {
node {
filename
mountPath
isPresent
}
}
}
}
}
}"""
r = client.execute(query)
assert 'errors' not in r
n = r['data']['removeSecretsEntry']['environment'][
'secretsFileMapping']['edges']
assert len(n) == 1
assert n[0]['node']['filename'] == 'absent.key'
assert n[0]['node']['isPresent'] == False
def remote_labbook_repo():
# TODO: Remove after integration tests with LFS support are available
conf_file, working_dir = _create_temp_work_dir(lfs_enabled=False)
im = InventoryManager(conf_file)
lb = im.create_labbook('test',
'test',
'sample-repo-lb',
description="my first labbook")
bm = BranchManager(lb, username='******')
bm.create_branch('testing-branch')
#with tempfile.TemporaryDirectory() as tmpdirname:
with open(os.path.join('/tmp', 'codefile.c'), 'wb') as codef:
codef.write(b'// Cody McCodeface ...')
FileOperations.insert_file(lb, "code", "/tmp/codefile.c")
assert lb.is_repo_clean
bm.workon_branch('master')
# Location of the repo to push/pull from
yield lb.root_dir
shutil.rmtree(working_dir)
try:
os.remove('/tmp/codefile.c')
except:
pass
def mutate_and_get_payload(cls,
root,
info,
name,
description,
repository,
base_id,
revision,
is_untracked=False,
client_mutation_id=None):
username = get_logged_in_username()
inv_manager = InventoryManager()
if is_untracked:
lb = inv_manager.create_labbook_disabled_lfs(
username=username,
owner=username,
labbook_name=name,
description=description,
author=get_logged_in_author())
else:
lb = inv_manager.create_labbook(username=username,
owner=username,
labbook_name=name,
description=description,
author=get_logged_in_author())
if is_untracked:
FileOperations.set_untracked(lb, 'input')
FileOperations.set_untracked(lb, 'output')
input_set = FileOperations.is_set_untracked(lb, 'input')
output_set = FileOperations.is_set_untracked(lb, 'output')
if not (input_set and output_set):
raise ValueError(
f'{str(lb)} untracking for input/output in malformed state'
)
if not lb.is_repo_clean:
raise ValueError(
f'{str(lb)} should have clean Git state after setting for untracked'
)
adr = ActivityDetailRecord(ActivityDetailType.LABBOOK,
show=False,
importance=0)
adr.add_value('text/plain', f"Created new LabBook: {username}/{name}")
# Create activity record
ar = ActivityRecord(ActivityType.LABBOOK,
message=f"Created new LabBook: {username}/{name}",
show=True,
importance=255,
linked_commit=lb.git.commit_hash)
ar.add_detail_object(adr)
store = ActivityStore(lb)
store.create_activity_record(ar)
cm = ComponentManager(lb)
cm.add_base(repository, base_id, revision)
return CreateLabbook(labbook=Labbook(owner=username, name=lb.name))
def resolve_repository_name_is_available(self, info, name: str):
"""Resolver to check if a repository name is in use locally or remotely
Args:
info:
name: desired name for the repository
Returns:
"""
# Check if repository exists locally
logged_in_username = get_logged_in_username()
im = InventoryManager()
if im.repository_exists(logged_in_username, logged_in_username, name):
return False
# Check if repository exists remotely
remote_config = Configuration().get_remote_configuration()
auth_service = None
remote = None
if remote_config:
auth_service = remote_config.get('admin_service')
remote = remote_config.get('git_remote')
# Get collaborators from remote service
mgr = GitLabManager(remote, auth_service, flask.g.access_token)
if mgr.repository_exists(logged_in_username, name):
return False
# If you get here the name is available
return True
def test_add_package_no_version(self, fixture_working_dir_env_repo_scoped,
snapshot):
"""Test adding a package but omitting the version"""
im = InventoryManager(fixture_working_dir_env_repo_scoped[0])
lb = im.create_labbook('default',
'default',
'catbook-package-no-version',
description="LB to test package mutation")
# Add a base image
pkg_query = """
mutation myPkgMutation {
addPackageComponents (input: {
owner: "default",
labbookName: "catbook-package-no-version",
packages: [{manager: "pip3", package: "gtmunit1"}]
}) {
clientMutationId
newPackageComponentEdges {
node{
id
manager
package
version
fromBase
}
cursor
}
}
}
"""
result = fixture_working_dir_env_repo_scoped[2].execute(pkg_query)
assert "errors" in result
assert result['errors'][0]['message'] == "'version'"
def build_lb_image_for_env(mock_config_with_repo):
# Create a labook
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook('unittester', 'unittester', "containerunittestbookenv",
description="Testing environment functions.")
# Create Component Manager
cm = ComponentManager(lb)
# Add a component
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
client = get_docker_client()
client.containers.prune()
try:
lb, docker_image_id = ContainerOperations.build_image(labbook=lb, username="******")
yield lb, 'unittester'
finally:
shutil.rmtree(lb.root_dir)
# Remove image if it's still there
try:
client.images.remove(docker_image_id, force=True, noprune=False)
except:
pass
def test_add_package(self, fixture_working_dir_env_repo_scoped, snapshot):
"""Test listing labbooks"""
im = InventoryManager(fixture_working_dir_env_repo_scoped[0])
lb = im.create_labbook('default',
'default',
'catbook-package-tester',
description="LB to test package mutation")
# Add a base image
pkg_query = """
mutation myPkgMutation {
addPackageComponents (input: {
owner: "default",
labbookName: "catbook-package-tester",
packages: [{manager: "conda3", package: "python-coveralls", version: "2.9.1"}]
}) {
clientMutationId
newPackageComponentEdges {
node{
id
manager
package
version
fromBase
}
cursor
}
}
}
"""
snapshot.assert_match(
fixture_working_dir_env_repo_scoped[2].execute(pkg_query))
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
description_content,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
lb.description = description_content
with lb.lock():
lb.git.add(os.path.join(lb.config_path))
commit = lb.git.commit('Updating description')
adr = ActivityDetailRecord(ActivityDetailType.LABBOOK, show=False)
adr.add_value('text/plain', "Updated description of Project")
ar = ActivityRecord(ActivityType.LABBOOK,
message="Updated description of Project",
linked_commit=commit.hexsha,
tags=["labbook"],
show=False)
ar.add_detail_object(adr)
ars = ActivityStore(lb)
ars.create_activity_record(ar)
return SetLabbookDescription(success=True)
def test_file_node(self, fixture_working_dir):
"""Test listing labbook favorites"""
im = InventoryManager(fixture_working_dir[0])
lb = im.create_labbook("default",
"default",
"labbook1",
description="my first labbook1")
# Setup some favorites in code
with open(os.path.join(lb.root_dir, 'code', 'test1.txt'),
'wt') as test_file:
test_file.write("blah1")
query = """
{
node(id: "TGFiYm9va0ZpbGU6ZGVmYXVsdCZsYWJib29rMSZjb2RlJnRlc3QxLnR4dA==") {
... on LabbookFile {
id
key
isDir
size
}
}
}
"""
r = fixture_working_dir[2].execute(query)
assert 'errors' not in r
assert r['data']['node']['isDir'] is False
assert r['data']['node']['key'] == 'test1.txt'
assert r['data']['node']['size'] == '5'
def test_verify_unmanaged_dataset(self, fixture_working_dir_dataset_tests):
im = InventoryManager(fixture_working_dir_dataset_tests[0])
ds = im.create_dataset('default',
'default',
"adataset",
storage_type="local_filesystem",
description="100")
dataset_dir = ds.root_dir
assert os.path.exists(dataset_dir) is True
flask.g.access_token = "asdf"
flask.g.id_token = "1234"
# configure backend and local dir
working_dir = fixture_working_dir_dataset_tests[1]
ds.backend.set_default_configuration('default', 'asdf', '1234')
current_config = ds.backend_config
current_config['Data Directory'] = "test_dir"
ds.backend_config = current_config
test_dir = os.path.join(working_dir, "local_data", "test_dir")
os.makedirs(test_dir)
with open(os.path.join(test_dir, "test.txt"), 'wt') as temp:
temp.write(f'dummy data: asdfasdf')
query = """
mutation myMutation{
verifyDataset(input: {datasetOwner: "default", datasetName: "adataset"}) {
backgroundJobKey
}
}
"""
result = fixture_working_dir_dataset_tests[2].execute(query)
assert "errors" not in result
assert "rq:job" in result['data']['verifyDataset']['backgroundJobKey']
def test_node_labbook_from_object(self, fixture_working_dir):
im = InventoryManager(fixture_working_dir[0])
lb = im.create_labbook("default",
"default",
"cat-lab-book1",
description="Test cat labbook from obj")
query = """
{
node(id: "TGFiYm9vazpkZWZhdWx0JmNhdC1sYWItYm9vazE=") {
... on Labbook {
name
description
activeBranchName
}
id
}
}
"""
r = fixture_working_dir[2].execute(query)
assert r['data']['node']['description'] == 'Test cat labbook from obj'
assert r['data']['node'][
'id'] == 'TGFiYm9vazpkZWZhdWx0JmNhdC1sYWItYm9vazE='
assert r['data']['node']['name'] == 'cat-lab-book1'
assert r['data']['node']['activeBranchName'] == 'master'
def test_readme(self, fixture_working_dir_env_repo_scoped, snapshot):
"""Test getting a labbook's readme document"""
# Create labbook
im = InventoryManager(fixture_working_dir_env_repo_scoped[0])
lb = im.create_labbook("default",
"default",
"labbook77",
description="my first labbook10000")
query = """
{
labbook(owner: "default", name: "labbook77") {
overview {
readme
}
}
}
"""
snapshot.assert_match(
fixture_working_dir_env_repo_scoped[2].execute(query))
lb.write_readme("##Summary\nThis is my readme!!")
snapshot.assert_match(
fixture_working_dir_env_repo_scoped[2].execute(query))
def test_cuda_version_property(self, mock_config_with_repo):
"""Test getting the cuda version"""
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook('test',
'test',
'labbook1',
description="my first labbook")
assert lb.cuda_version is None
# Add base without GPU support
cm = ComponentManager(lb)
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
base_yaml_file = glob.glob(
os.path.join(lb.root_dir, '.gigantum', 'env', 'base', '*.yaml'))[0]
assert lb.cuda_version is None
# Fake a version
with open(base_yaml_file, 'rt') as bf:
base_data = yaml.safe_load(bf)
base_data['cuda_version'] = '10.0'
with open(base_yaml_file, 'wt') as bf:
yaml.safe_dump(base_data, bf)
assert lb.cuda_version == '10.0'
def test_query_owner(self, mock_config_file):
inv_manager = InventoryManager(mock_config_file[0])
lb = inv_manager.create_labbook("test",
"test",
"labbook1",
description="my first labbook")
assert "test" == inv_manager.query_owner(lb)
def mock_create_labbooks(fixture_working_dir):
# Create a labbook in the temporary directory
config_file = fixture_working_dir[0]
im = InventoryManager(fixture_working_dir[0])
lb = im.create_labbook(UT_USERNAME,
UT_USERNAME,
UT_LBNAME,
description="Cats labbook 1")
# Create a file in the dir
with open(os.path.join(fixture_working_dir[1], 'unittest-examplefile'),
'w') as sf:
sf.write("test data")
sf.seek(0)
FileOperations.insert_file(lb, 'code', sf.name)
assert os.path.isfile(
os.path.join(lb.root_dir, 'code', 'unittest-examplefile'))
# Create test client
schema = graphene.Schema(query=LabbookQuery, mutation=LabbookMutations)
with patch.object(Configuration, 'find_default_config',
lambda self: config_file):
app = Flask("lmsrvlabbook")
app.config["LABMGR_CONFIG"] = Configuration()
app.config["LABMGR_ID_MGR"] = get_identity_manager(Configuration())
with app.app_context():
flask.g.user_obj = app.config["LABMGR_ID_MGR"].get_user_profile()
client = Client(
schema,
middleware=[DataloaderMiddleware(), error_middleware],
context_value=ContextMock())
yield lb, client, schema
shutil.rmtree(fixture_working_dir, ignore_errors=True)
def test_create_dataset_invalid_storage_type(self, mock_config_file):
inv_manager = InventoryManager(mock_config_file[0])
auth = GitAuthor(name="username", email="*****@*****.**")
with pytest.raises(ValueError):
inv_manager.create_dataset("test", "test", "dataset1", "asdfdfgh",
description="my first dataset",
author=auth)
def test_link_unlink_dataset_with_repair(self, mock_labbook):
inv_manager = InventoryManager(mock_labbook[0])
lb = mock_labbook[2]
ds = inv_manager.create_dataset("test",
"test",
"dataset100",
"gigantum_object_v1",
description="my dataset")
# Fake publish to a local bare repo
_MOCK_create_remote_repo2(ds, 'test', None, None)
assert os.path.exists(os.path.join(lb.root_dir,
'.gitmodules')) is False
dataset_submodule_dir = os.path.join(lb.root_dir, '.gigantum',
'datasets', 'test', 'dataset100')
git_module_dir = os.path.join(lb.root_dir, '.git', 'modules',
f"test&dataset100")
# Add dirs as if lingering submodule config
os.makedirs(dataset_submodule_dir)
os.makedirs(git_module_dir)
inv_manager.link_dataset_to_labbook(ds.remote, 'test', 'dataset100',
lb)
assert os.path.exists(os.path.join(lb.root_dir, '.gitmodules')) is True
assert os.path.exists(dataset_submodule_dir) is True
assert os.path.exists(os.path.join(dataset_submodule_dir,
'.gigantum')) is True
with open(os.path.join(lb.root_dir, '.gitmodules'), 'rt') as mf:
data = mf.read()
assert len(data) > 0
