def test_delete_labbook_linked_dataset(self, mock_config_file): """Test trying to create a labbook with a name that already exists locally""" inv_manager = InventoryManager(mock_config_file[0]) inv_manager.create_labbook("test", "test", "labbook1", description="my first labbook") lb = inv_manager.load_labbook("test", "test", "labbook1") auth = GitAuthor(name="test", email="*****@*****.**") ds = inv_manager.create_dataset("test", "test", "dataset1", "gigantum_object_v1", description="my first dataset", author=auth) inv_manager.link_dataset_to_labbook(f"{ds.root_dir}/.git", "test", "dataset1", lb) dataset_delete_jobs = inv_manager.delete_labbook( "test", "test", "labbook1") assert len(dataset_delete_jobs) == 1 assert dataset_delete_jobs[0].namespace == "test" assert dataset_delete_jobs[0].name == "dataset1" with pytest.raises(InventoryException): inv_manager.load_labbook("test", "test", "labbook1")
def test_list_labbooks_az(self, mock_config_file): """Test list az labbooks""" inv_manager = InventoryManager(mock_config_file[0]) lb1 = inv_manager.create_labbook("user1", "user1", "labbook0", description="my first labbook") lb2 = inv_manager.create_labbook("user1", "user1", "labbook12", description="my second labbook") lb3 = inv_manager.create_labbook("user1", "user2", "labbook3", description="my other labbook") lb4 = inv_manager.create_labbook("user2", "user1", "labbook4", description="my other labbook") labbooks = inv_manager.list_labbooks(username="******") assert len(labbooks) == 3 assert labbooks[0].name == 'labbook0' assert labbooks[1].name == 'labbook3' assert labbooks[2].name == 'labbook12'
def test_load_err(self, fixture_working_dir): im = InventoryManager(fixture_working_dir[0]) im.create_labbook("default", "default", "lb1", description="a project") loader = FetchLoader() key = "invalidkey&default&default&lb1" with pytest.raises(ValueError): promise1 = loader.load(key) promise1.get()
def test_load_one_labbook(self, fixture_working_dir): im = InventoryManager(fixture_working_dir[0]) im.create_labbook("default", "default", "lb1", description="a project") im.create_labbook("default", "default", "lb2", description="a project") loader = FetchLoader() key = "labbook&default&default&lb1" promise1 = loader.load(key) assert isinstance(promise1, Promise) assert promise1.get() is None
def test_create_labbook_that_exists(self, mock_config_file): """Test trying to create a labbook with a name that already exists locally""" inv_manager = InventoryManager(mock_config_file[0]) inv_manager.create_labbook("test", "test", "labbook1", description="my first labbook") with pytest.raises(ValueError): inv_manager.create_labbook("test", "test", "labbook1", description="my first labbook")
def test_list_labbooks_modified_date(self, mock_config_file): """Test list modified dated sorted labbooks""" inv_manager = InventoryManager(mock_config_file[0]) lb1 = inv_manager.create_labbook("user1", "user1", "labbook3", description="my first labbook") time.sleep(1.2) lb2 = inv_manager.create_labbook("user1", "user1", "asdf", description="my second labbook") time.sleep(1.2) lb3 = inv_manager.create_labbook("user1", "user2", "labbook1", description="my other labbook") time.sleep(1.2) lb4 = inv_manager.create_labbook("user1", "user1", "hhghg", description="my other labbook") inv_manager = InventoryManager(mock_config_file[0]) labbooks = inv_manager.list_labbooks(username="******", sort_mode="modified_on") assert len(labbooks) == 4 assert labbooks[0].name == 'labbook3' assert labbooks[1].name == 'asdf' assert labbooks[2].name == 'labbook1' assert labbooks[3].name == 'hhghg' # modify a repo time.sleep(1.2) with open(os.path.join(lb2.root_dir, "code", "test.txt"), 'wt') as tf: tf.write("asdfasdf") lb2.git.add_all() lb2.git.commit("Changing the repo") labbooks = inv_manager.list_labbooks(username="******", sort_mode="modified_on") assert len(labbooks) == 4 assert labbooks[0].name == 'labbook3' assert labbooks[1].name == 'labbook1' assert labbooks[2].name == 'hhghg' assert labbooks[3].name == 'asdf'
def test_delete_labbook_no_linked_datasets(self, mock_config_file): """Test trying to create a labbook with a name that already exists locally""" inv_manager = InventoryManager(mock_config_file[0]) inv_manager.create_labbook("test", "test", "labbook1", description="my first labbook") inv_manager.load_labbook("test", "test", "labbook1") dataset_delete_jobs = inv_manager.delete_labbook( "test", "test", "labbook1") assert dataset_delete_jobs == [] with pytest.raises(InventoryException): inv_manager.load_labbook("test", "test", "labbook1")
def test_checkout_id_property(self, mock_config_file): """Test trying to create a labbook with a name that already exists locally""" im = InventoryManager(mock_config_file[0]) lb = im.create_labbook('test', 'test', 'labbook1', description="my first labbook") checkout_file = os.path.join(lb.root_dir, '.gigantum', '.checkout') assert os.path.exists(checkout_file) is False checkout_id = lb.checkout_id assert os.path.exists(checkout_file) is True parts = checkout_id.split("-") assert len(parts) == 5 assert parts[0] == "test" assert parts[1] == "test" assert parts[2] == "labbook1" assert len(parts[-1]) == 10 # Check repo is clean status = lb.git.status() for key in status: assert len(status[key]) == 0 # Remove checkout file os.remove(checkout_file) # Repo should STILL be clean as it is not tracked status = lb.git.status() for key in status: assert len(status[key]) == 0
def build_lb_image_for_env(mock_config_with_repo): # Create a labook im = InventoryManager(mock_config_with_repo[0]) lb = im.create_labbook('unittester', 'unittester', "containerunittestbookenv", description="Testing environment functions.") # Create Component Manager cm = ComponentManager(lb) # Add a component cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV) ib = ImageBuilder(lb) ib.assemble_dockerfile(write=True) client = get_docker_client() client.containers.prune() try: lb, docker_image_id = ContainerOperations.build_image(labbook=lb, username="******") yield lb, 'unittester' finally: shutil.rmtree(lb.root_dir) # Remove image if it's still there try: client.images.remove(docker_image_id, force=True, noprune=False) except: pass
def build_lb_image_for_env_conda(mock_config_with_repo): """A fixture that installs an old version of matplotlib and latest version of requests to increase code coverage""" im = InventoryManager(mock_config_with_repo[0]) lb = im.create_labbook('unittester', 'unittester', "containerunittestbookenvconda", description="Testing environment functions.") cm = ComponentManager(lb) cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV) cm.add_packages('conda3', [{'package': 'python-coveralls', 'version': '2.7.0'}]) ib = ImageBuilder(lb) ib.assemble_dockerfile(write=True) client = get_docker_client() client.containers.prune() try: lb, docker_image_id = ContainerOperations.build_image(labbook=lb, username="******") yield lb, 'unittester' finally: shutil.rmtree(lb.root_dir) try: client.images.remove(docker_image_id, force=True, noprune=False) except: pass
def remote_labbook_repo(): # TODO: Remove after integration tests with LFS support are available conf_file, working_dir = _create_temp_work_dir(lfs_enabled=False) im = InventoryManager(conf_file) lb = im.create_labbook('test', 'test', 'sample-repo-lb', description="my first labbook") bm = BranchManager(lb, username='******') bm.create_branch('testing-branch') #with tempfile.TemporaryDirectory() as tmpdirname: with open(os.path.join('/tmp', 'codefile.c'), 'wb') as codef: codef.write(b'// Cody McCodeface ...') FileOperations.insert_file(lb, "code", "/tmp/codefile.c") assert lb.is_repo_clean bm.workon_branch('master') # Location of the repo to push/pull from yield lb.root_dir shutil.rmtree(working_dir) try: os.remove('/tmp/codefile.c') except: pass
def mock_create_labbooks(fixture_working_dir): # Create a labbook in the temporary directory config_file = fixture_working_dir[0] im = InventoryManager(fixture_working_dir[0]) lb = im.create_labbook(UT_USERNAME, UT_USERNAME, UT_LBNAME, description="Cats labbook 1") # Create a file in the dir with open(os.path.join(fixture_working_dir[1], 'unittest-examplefile'), 'w') as sf: sf.write("test data") sf.seek(0) FileOperations.insert_file(lb, 'code', sf.name) assert os.path.isfile( os.path.join(lb.root_dir, 'code', 'unittest-examplefile')) # Create test client schema = graphene.Schema(query=LabbookQuery, mutation=LabbookMutations) with patch.object(Configuration, 'find_default_config', lambda self: config_file): app = Flask("lmsrvlabbook") app.config["LABMGR_CONFIG"] = Configuration() app.config["LABMGR_ID_MGR"] = get_identity_manager(Configuration()) with app.app_context(): flask.g.user_obj = app.config["LABMGR_ID_MGR"].get_user_profile() client = Client( schema, middleware=[DataloaderMiddleware(), error_middleware], context_value=ContextMock()) yield lb, client, schema shutil.rmtree(fixture_working_dir, ignore_errors=True)
def test_get_commits_behind(self, fixture_working_dir): """Test temporar field commitsBehind on dataset objects""" im = InventoryManager(fixture_working_dir[0]) ds = im.create_dataset("default", "default", "test-ds", "gigantum_object_v1", description="my first dataset", author=GitAuthor(name="default", email="*****@*****.**")) lb = im.create_labbook("default", "default", "test-lb") im.link_dataset_to_labbook(f"{ds.root_dir}/.git", 'default', 'test-ds', lb) query = """ { labbook(owner: "default", name:"test-lb") { linkedDatasets{ name commitsBehind } } } """ r = fixture_working_dir[2].execute(query) assert 'errors' not in r assert r['data']['labbook']['linkedDatasets'][0]['name'] == 'test-ds' assert r['data']['labbook']['linkedDatasets'][0]['commitsBehind'] == 0 ds.write_readme("test contents to make a commit") r = fixture_working_dir[2].execute(query) assert 'errors' not in r assert r['data']['labbook']['linkedDatasets'][0]['name'] == 'test-ds' assert r['data']['labbook']['linkedDatasets'][0]['commitsBehind'] == 1
def test_empty_package_counts(self, fixture_working_dir_env_repo_scoped, snapshot): """Test getting the a LabBook's package manager dependencies""" # Create labbook im = InventoryManager(fixture_working_dir_env_repo_scoped[0]) lb = im.create_labbook("default", "default", "labbook4", description="my first labbook10000") query = """ { labbook(owner: "default", name: "labbook4") { overview { numAptPackages numConda2Packages numConda3Packages numPipPackages numCustomDependencies } } } """ snapshot.assert_match( fixture_working_dir_env_repo_scoped[2].execute(query))
def test_query_owner(self, mock_config_file): inv_manager = InventoryManager(mock_config_file[0]) lb = inv_manager.create_labbook("test", "test", "labbook1", description="my first labbook") assert "test" == inv_manager.query_owner(lb)
def mutate_and_get_payload(cls, root, info, name, description, repository, base_id, revision, is_untracked=False, client_mutation_id=None): username = get_logged_in_username() inv_manager = InventoryManager() if is_untracked: lb = inv_manager.create_labbook_disabled_lfs( username=username, owner=username, labbook_name=name, description=description, author=get_logged_in_author()) else: lb = inv_manager.create_labbook(username=username, owner=username, labbook_name=name, description=description, author=get_logged_in_author()) if is_untracked: FileOperations.set_untracked(lb, 'input') FileOperations.set_untracked(lb, 'output') input_set = FileOperations.is_set_untracked(lb, 'input') output_set = FileOperations.is_set_untracked(lb, 'output') if not (input_set and output_set): raise ValueError( f'{str(lb)} untracking for input/output in malformed state' ) if not lb.is_repo_clean: raise ValueError( f'{str(lb)} should have clean Git state after setting for untracked' ) adr = ActivityDetailRecord(ActivityDetailType.LABBOOK, show=False, importance=0) adr.add_value('text/plain', f"Created new LabBook: {username}/{name}") # Create activity record ar = ActivityRecord(ActivityType.LABBOOK, message=f"Created new LabBook: {username}/{name}", show=True, importance=255, linked_commit=lb.git.commit_hash) ar.add_detail_object(adr) store = ActivityStore(lb) store.create_activity_record(ar) cm = ComponentManager(lb) cm.add_base(repository, base_id, revision) return CreateLabbook(labbook=Labbook(owner=username, name=lb.name))
def test_add_package_no_version(self, fixture_working_dir_env_repo_scoped, snapshot): """Test adding a package but omitting the version""" im = InventoryManager(fixture_working_dir_env_repo_scoped[0]) lb = im.create_labbook('default', 'default', 'catbook-package-no-version', description="LB to test package mutation") # Add a base image pkg_query = """ mutation myPkgMutation { addPackageComponents (input: { owner: "default", labbookName: "catbook-package-no-version", packages: [{manager: "pip3", package: "gtmunit1"}] }) { clientMutationId newPackageComponentEdges { node{ id manager package version fromBase } cursor } } } """ result = fixture_working_dir_env_repo_scoped[2].execute(pkg_query) assert "errors" in result assert result['errors'][0]['message'] == "'version'"
def test_package_query(self, snapshot, fixture_working_dir_env_repo_scoped): """Test querying for package info""" im = InventoryManager(fixture_working_dir_env_repo_scoped[0]) lb = im.create_labbook("default", "default", "labbook6", description="my first labbook10000") query = """ { labbook(owner: "default", name: "labbook6"){ id checkPackages(packageInput: [ {manager: "pip", package: "gtmunit1", version:"0.2.4"}, {manager: "pip", package: "gtmunit2", version:""}]){ id manager package version isValid } } } """ snapshot.assert_match( fixture_working_dir_env_repo_scoped[2].execute(query))
def test_package_query_with_errors_conda( self, snapshot, fixture_working_dir_env_repo_scoped): """Test querying for package info""" # Create labbook im = InventoryManager(fixture_working_dir_env_repo_scoped[0]) lb = im.create_labbook("default", "default", "labbook5conda", description="my first labbook10000") query = """ { labbook(owner: "default", name: "labbook5conda"){ id checkPackages(packageInput: [ {manager: "conda3", package: "cdutil", version:"8.1"}, {manager: "conda3", package: "nltk", version:"100.00"}, {manager: "conda3", package: "python-coveralls", version:""}, {manager: "conda3", package: "thisshouldtotallyfail", version:"1.0"}, {manager: "conda3", package: "notarealpackage", version:""}]){ id manager package version latestVersion description isValid } } } """ snapshot.assert_match( fixture_working_dir_env_repo_scoped[2].execute(query))
def test_remove_secrets_entry(self, fixture_working_dir_env_repo_scoped): client = fixture_working_dir_env_repo_scoped[2] im = InventoryManager(fixture_working_dir_env_repo_scoped[0]) lb = im.create_labbook("default", "default", "unittest-mutation-remove-secret") SecretStore(lb, "default")['remove.key'] = '/mnt/nowhere' SecretStore(lb, "default")['absent.key'] = '/mnt/nowhere2' query = """ mutation remove { removeSecretsEntry(input: { owner: "default", labbookName: "unittest-mutation-remove-secret", filename: "remove.key", }) { environment { secretsFileMapping { edges { node { filename mountPath isPresent } } } } } }""" r = client.execute(query) assert 'errors' not in r n = r['data']['removeSecretsEntry']['environment'][ 'secretsFileMapping']['edges'] assert len(n) == 1 assert n[0]['node']['filename'] == 'absent.key' assert n[0]['node']['isPresent'] == False
def test_untracked_output_dir(self, mock_config_file): """Test that contents of the untracked directory (in output) truly is untracked. """ im = InventoryManager(mock_config_file[0]) lb = im.create_labbook('test', 'test', 'labbook1', description="my first labbook", author=GitAuthor(name="test", email="*****@*****.**")) assert os.path.isdir(os.path.join(lb.root_dir, 'output', 'untracked')) altered_file = os.path.join(lb.root_dir, 'output', 'untracked', 'samplefiles') with open(altered_file, 'w') as f: f.write('Initial Content') lb.sweep_uncommitted_changes() c1 = lb.git.commit_hash with open(altered_file, 'w') as f: f.write('Changed Content') lb.sweep_uncommitted_changes() c2 = lb.git.commit_hash # Assert that Git detects no changes assert c1 == c2
def test_is_labbook_modified_date(self, mock_config_file): """Test getting the modified date""" im = InventoryManager(mock_config_file[0]) lb = im.create_labbook('test', 'test', 'labbook1', description="my first labbook", author=GitAuthor(name="test", email="*****@*****.**")) modified_1 = lb.modified_on time.sleep(2) lb.write_readme("doing something to change the modified time") modified_2 = lb.modified_on assert modified_2 > modified_1 assert modified_1.microsecond == 0 assert modified_1.tzname() == "UTC" assert modified_2.microsecond == 0 assert modified_2.tzname() == "UTC" assert (datetime.datetime.now(datetime.timezone.utc) - modified_1).total_seconds() < 10 assert (datetime.datetime.now(datetime.timezone.utc) - modified_2).total_seconds() < 10
def test_readme(self, fixture_working_dir_env_repo_scoped, snapshot): """Test getting a labbook's readme document""" # Create labbook im = InventoryManager(fixture_working_dir_env_repo_scoped[0]) lb = im.create_labbook("default", "default", "labbook77", description="my first labbook10000") query = """ { labbook(owner: "default", name: "labbook77") { overview { readme } } } """ snapshot.assert_match( fixture_working_dir_env_repo_scoped[2].execute(query)) lb.write_readme("##Summary\nThis is my readme!!") snapshot.assert_match( fixture_working_dir_env_repo_scoped[2].execute(query))
def test_cuda_version_property(self, mock_config_with_repo): """Test getting the cuda version""" im = InventoryManager(mock_config_with_repo[0]) lb = im.create_labbook('test', 'test', 'labbook1', description="my first labbook") assert lb.cuda_version is None # Add base without GPU support cm = ComponentManager(lb) cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV) base_yaml_file = glob.glob( os.path.join(lb.root_dir, '.gigantum', 'env', 'base', '*.yaml'))[0] assert lb.cuda_version is None # Fake a version with open(base_yaml_file, 'rt') as bf: base_data = yaml.safe_load(bf) base_data['cuda_version'] = '10.0' with open(base_yaml_file, 'wt') as bf: yaml.safe_dump(base_data, bf) assert lb.cuda_version == '10.0'
def test_add_package(self, fixture_working_dir_env_repo_scoped, snapshot): """Test listing labbooks""" im = InventoryManager(fixture_working_dir_env_repo_scoped[0]) lb = im.create_labbook('default', 'default', 'catbook-package-tester', description="LB to test package mutation") # Add a base image pkg_query = """ mutation myPkgMutation { addPackageComponents (input: { owner: "default", labbookName: "catbook-package-tester", packages: [{manager: "conda3", package: "python-coveralls", version: "2.9.1"}] }) { clientMutationId newPackageComponentEdges { node{ id manager package version fromBase } cursor } } } """ snapshot.assert_match( fixture_working_dir_env_repo_scoped[2].execute(pkg_query))
def test_favorites_node(self, fixture_working_dir): """Test listing labbook favorites""" im = InventoryManager(fixture_working_dir[0]) lb = im.create_labbook("default", "default", "labbook1", description="my first labbook1") # Setup some favorites in code with open(os.path.join(lb.root_dir, 'code', 'test1.txt'), 'wt') as test_file: test_file.write("blah1") # Create favorites lb.create_favorite("code", "test1.txt", description="My file with stuff 1") # Test bad node that isn't a file query = """ { node(id: "TGFiYm9va0Zhdm9yaXRlOmRlZmF1bHQmbGFiYm9vazEmY29kZSZ0ZXN0MzMzLnR4dA==") { ... on LabbookFavorite { id key description isDir index } } } """ r = fixture_working_dir[2].execute(query) # Assert that there ARE INDEED errors assert 'errors' in r # These are the fields with error and are therefore None assert r['data']['node']['description'] is None assert r['data']['node']['index'] is None assert r['data']['node']['isDir'] is None # The one field that is NOT in error assert r['data']['node']['key'] == 'test333.txt' # Get the actual item query = """ { node(id: "TGFiYm9va0Zhdm9yaXRlOmRlZmF1bHQmbGFiYm9vazEmY29kZSZ0ZXN0MS50eHQ=") { ... on LabbookFavorite { id key description isDir index } } } """ r2 = fixture_working_dir[2].execute(query) assert 'errors' not in r2 assert r2['data']['node']['description'] == 'My file with stuff 1' assert r2['data']['node']['index'] == 0 assert r2['data']['node']['isDir'] == False assert r2['data']['node']['key'] == 'test1.txt'
def test_list_labbooks_full_set(self, mock_config_file): ut_username = "******" owners = [f"ut-owner-{i}" for i in range(4)] lbnames = [f'unittest-labbook-{i}' for i in range(6)] created_lbs = [] inv_manager = InventoryManager(mock_config_file[0]) for ow in owners: for lbname in lbnames: l = inv_manager.create_labbook(ut_username, ow, lbname) created_lbs.append(l) get_owner = lambda x: InventoryManager(mock_config_file[0] ).query_owner(x) condensed_lbs = [(ut_username, get_owner(l), l.name) for l in created_lbs] inv_manager = InventoryManager(mock_config_file[0]) t0 = time.time() result_under_test = inv_manager.list_repository_ids( username=ut_username, repository_type="labbook") assert time.time( ) - t0 < 1.0, "list_repository_ids should return in under 1 second" assert len(list(set(condensed_lbs))) == 6 * 4 assert len(list(set(result_under_test))) == 6 * 4 for r in result_under_test: assert r in condensed_lbs
def test_file_node(self, fixture_working_dir): """Test listing labbook favorites""" im = InventoryManager(fixture_working_dir[0]) lb = im.create_labbook("default", "default", "labbook1", description="my first labbook1") # Setup some favorites in code with open(os.path.join(lb.root_dir, 'code', 'test1.txt'), 'wt') as test_file: test_file.write("blah1") query = """ { node(id: "TGFiYm9va0ZpbGU6ZGVmYXVsdCZsYWJib29rMSZjb2RlJnRlc3QxLnR4dA==") { ... on LabbookFile { id key isDir size } } } """ r = fixture_working_dir[2].execute(query) assert 'errors' not in r assert r['data']['node']['isDir'] is False assert r['data']['node']['key'] == 'test1.txt' assert r['data']['node']['size'] == '5'
def test_node_labbook_from_object(self, fixture_working_dir): im = InventoryManager(fixture_working_dir[0]) lb = im.create_labbook("default", "default", "cat-lab-book1", description="Test cat labbook from obj") query = """ { node(id: "TGFiYm9vazpkZWZhdWx0JmNhdC1sYWItYm9vazE=") { ... on Labbook { name description activeBranchName } id } } """ r = fixture_working_dir[2].execute(query) assert r['data']['node']['description'] == 'Test cat labbook from obj' assert r['data']['node'][ 'id'] == 'TGFiYm9vazpkZWZhdWx0JmNhdC1sYWItYm9vazE=' assert r['data']['node']['name'] == 'cat-lab-book1' assert r['data']['node']['activeBranchName'] == 'master'
def test_readme_wierd_strings(self, mock_config_file): """Test creating a reading and writing a readme file to the labbook with complex strings""" im = InventoryManager(mock_config_file[0]) lb = im.create_labbook('test', 'test', 'labbook1', description="my first labbook", author=GitAuthor(name="test", email="*****@*****.**")) assert lb.get_readme() is None assert os.path.exists(os.path.join(lb.root_dir, 'README.md')) is False rand_str = os.urandom(1000000) with pytest.raises(TypeError): lb.write_readme(rand_str) assert lb.get_readme() is None assert os.path.exists(os.path.join(lb.root_dir, 'README.md')) is False with pytest.raises(TypeError): lb.write_readme(None) assert lb.get_readme() is None assert os.path.exists(os.path.join(lb.root_dir, 'README.md')) is False lb.write_readme("") assert lb.get_readme() == "" assert os.path.exists(os.path.join(lb.root_dir, 'README.md')) is True