def test_reset_branch_to_remote(self, fixture_working_dir, mock_create_labbooks_no_lfs): # Mock the request context so a fake authorization header is present builder = EnvironBuilder(path='/labbook', method='POST', headers={'Authorization': 'Bearer AJDFHASD'}) env = builder.get_environ() req = Request(environ=env) im = InventoryManager(mock_create_labbooks_no_lfs[0]) test_user_lb = im.load_labbook('default', 'default', 'labbook1') wf = LabbookWorkflow(test_user_lb) wf.publish(username='******') hash_original = wf.labbook.git.commit_hash new_file_path = os.path.join(wf.labbook.root_dir, 'input', 'new-file') with open(new_file_path, 'w') as f: f.write('File data') wf.labbook.sweep_uncommitted_changes() hash_before_reset = wf.labbook.git.commit_hash publish_query = f""" mutation c {{ resetBranchToRemote(input: {{ labbookName: "labbook1", owner: "default" }}) {{ labbook {{ activeBranchName }} }} }} """ r = mock_create_labbooks_no_lfs[2].execute(publish_query, context_value=req) assert 'errors' not in r assert wf.labbook.git.commit_hash == hash_original
def test_sync_1(self, mock_create_labbooks_no_lfs, mock_config_file): # Setup responses mock for this test responses.add(responses.GET, 'https://usersrv.gigantum.io/key', json={'key': 'afaketoken'}, status=200) im = InventoryManager(mock_create_labbooks_no_lfs[0]) test_user_lb = im.load_labbook('default', 'default', 'labbook1') test_user_wf = LabbookWorkflow(test_user_lb) test_user_wf.publish('default') # Mock the request context so a fake authorization header is present builder = EnvironBuilder(path='/labbook', method='POST', headers={'Authorization': 'Bearer AJDFHASD'}) env = builder.get_environ() req = Request(environ=env) sally_wf = LabbookWorkflow.import_from_remote(test_user_wf.remote, 'sally', config_file=mock_config_file[0]) sally_lb = sally_wf.labbook FileOperations.makedir(sally_lb, relative_path='code/sally-dir', create_activity_record=True) sally_wf.sync('sally') sync_query = """ mutation x { syncLabbook(input: { labbookName: "labbook1", owner: "default" }) { jobKey } } """ r = mock_create_labbooks_no_lfs[2].execute(sync_query, context_value=req) assert 'errors' not in r
def mutate_and_get_payload(cls, root, info, owner, labbook_name, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) migrated = False with lb.lock(): t0 = time.time() workflow = LabbookWorkflow(lb) migrated = workflow.migrate() tf = time.time() if migrated: logger.info(f"Migrated {str(lb)} in {tf-t0:.2}sec") else: logger.info(f"No migration needed for {str(lb)}") return MigrateLabbookSchema( Labbook(id=f"{owner}&{labbook_name}", name=labbook_name, owner=owner))
def publish_repository(repository: Repository, username: str, access_token: str, remote: Optional[str] = None, public: bool = False, id_token: str = None) -> None: p = os.getpid() logger = LMLogger.get_logger() logger.info(f"(Job {p}) Starting publish_repository({str(repository)})") def update_meta(msg): job = get_current_job() if not job: return if 'feedback' not in job.meta: job.meta['feedback'] = msg else: job.meta['feedback'] = job.meta['feedback'] + f'\n{msg}' job.save_meta() try: with repository.lock(): if isinstance(repository, LabBook): wf = LabbookWorkflow(repository) else: wf = DatasetWorkflow(repository) # type: ignore wf.publish(username=username, access_token=access_token, remote=remote or "origin", public=public, feedback_callback=update_meta, id_token=id_token) except Exception as e: logger.exception(f"(Job {p}) Error on publish_repository: {e}") raise
def sync_repository(repository: Repository, username: str, override: MergeOverride, remote: str = "origin", access_token: str = None, pull_only: bool = False, id_token: str = None) -> int: p = os.getpid() logger = LMLogger.get_logger() logger.info(f"(Job {p}) Starting sync_repository({str(repository)})") def update_meta(msg): job = get_current_job() if not job: return if 'feedback' not in job.meta: job.meta['feedback'] = msg else: job.meta['feedback'] = job.meta['feedback'] + f'\n{msg}' job.save_meta() try: with repository.lock(): if isinstance(repository, LabBook): wf = LabbookWorkflow(repository) else: wf = DatasetWorkflow(repository) # type: ignore cnt = wf.sync(username=username, remote=remote, override=override, feedback_callback=update_meta, access_token=access_token, id_token=id_token, pull_only=pull_only) logger.info(f"(Job {p} Completed sync_repository with cnt={cnt}") return cnt except Exception as e: logger.exception(f"(Job {p}) Error on sync_repository: {e}") raise
def mutate_and_get_payload(cls, root, info, owner, labbook_name, branch_name, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): wf = LabbookWorkflow(lb) wf.checkout(username, branch_name) return WorkonBranch(Labbook(id="{}&{}".format(owner, labbook_name), name=labbook_name, owner=owner))
def test_migrate_no_op_on_new_labbook(self, mock_labbook_lfs_disabled): username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) h1 = wf.labbook.git.commit_hash wf.migrate() # No change of git status after no-op migration assert h1 == wf.labbook.git.commit_hash
def _stop_container(cls, lb, username): """Stop container and also do necessary cleanup of confhttpproxy, monitors, etc. Currently, this supports two cases, applications monitored by MITMProxy, and Jupyter. So, for now, if we can't find an mitmproxy endpoint, we assume we're dealing with a jupyter container. """ pr = confhttpproxy.ProxyRouter.get_proxy( lb.client_config.config['proxy']) # Remove route from proxy lb_name = ContainerOperations.labbook_image_name(lb, username) if MITMProxyOperations.get_mitmendpoint(lb_name): # there is an MITMProxy (currently only used for RStudio) proxy_endpoint = MITMProxyOperations.stop_mitm_proxy(lb_name) tool = 'rserver' else: lb_ip = ContainerOperations.get_labbook_ip(lb, username) # The only alternative to mitmproxy (currently) is jupyter # TODO in #453: Construction of this URL should be encapsulated in Jupyter Dev Tool logic proxy_endpoint = f'http://{lb_ip}:8888' tool = 'jupyter' est_target = pr.get_matching_routes(proxy_endpoint, tool) for i, target in enumerate(est_target): if i == 1: # We have > 1 entry in the router, which shouldn't happen logger.warning( f'Removing multiple routes for {tool} on {proxy_endpoint} during Project container stop.' ) pr.remove(target[1:]) wf = LabbookWorkflow(lb) wf.garbagecollect() # Clean up empty bind mount dirs from datasets if needed submodules = lb.git.list_submodules() for submodule in submodules: namespace, dataset_name = submodule['name'].split("&") bind_location = os.path.join(lb.root_dir, 'input', dataset_name) if os.path.isdir(bind_location): os.rmdir(bind_location) # stop labbook monitor stop_labbook_monitor(lb, username) lb, stopped = ContainerOperations.stop_container(labbook=lb, username=username) if not stopped: # TODO DK: Why would stopped=False? Should this move up?? raise ValueError(f"Failed to stop labbook {lb.name}")
def sync_repository(repository: Repository, username: str, override: MergeOverride, remote: str = "origin", access_token: str = None, pull_only: bool = False, id_token: str = None) -> int: p = os.getpid() logger = LMLogger.get_logger() logger.info(f"(Job {p}) Starting sync_repository({str(repository)})") def update_feedback(msg: str, has_failures: Optional[bool] = None, failure_detail: Optional[str] = None, percent_complete: Optional[float] = None): """Method to update the job's metadata and provide feedback to the UI""" current_job = get_current_job() if not current_job: return if has_failures: current_job.meta['has_failures'] = has_failures if failure_detail: current_job.meta['failure_detail'] = failure_detail if percent_complete: current_job.meta['percent_complete'] = percent_complete current_job.meta['feedback'] = msg current_job.save_meta() try: update_feedback("Sync task in queue") with repository.lock(): if isinstance(repository, LabBook): wf = LabbookWorkflow(repository) else: wf = DatasetWorkflow(repository) # type: ignore cnt = wf.sync(username=username, remote=remote, override=override, feedback_callback=update_feedback, access_token=access_token, id_token=id_token, pull_only=pull_only) logger.info(f"(Job {p} Completed sync_repository with cnt={cnt}") return cnt except MergeConflict as me: logger.exception(f"(Job {p}) Merge conflict: {me}") raise except IOError: raise except Exception as e: logger.exception(e) raise Exception("Could not sync - try to log out and log in again.")
def test_sync___override_merge_conflict_ours(self, mock_labbook_lfs_disabled, mock_config_file): """ test sync, with override in case of merge conflict. """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) bm = BranchManager(lb, username='******') bm.create_branch('test-conflict-branch') fpath = os.path.join(lb.root_dir, 'input', 'testfile') with open(fpath, 'w') as f: f.write('filedata') lb.sweep_uncommitted_changes() wf.sync('test') other_user = '******' wf_other = LabbookWorkflow.import_from_remote( wf.remote, username=other_user, config_file=mock_config_file[0]) bm_other = BranchManager(wf_other.labbook, username=other_user) bm_other.workon_branch('test-conflict-branch') with open(os.path.join(wf_other.labbook.root_dir, 'input', 'testfile'), 'w') as f: f.write('conflicting-change-other-user') wf_other.labbook.sweep_uncommitted_changes() wf_other.sync(username=username) fpath = os.path.join(wf.labbook.root_dir, 'input', 'testfile') with open(fpath, 'w') as f: f.write('conflicting-change-original-user') wf.labbook.sweep_uncommitted_changes() n = wf.sync(username=username, override=MergeOverride.OURS) flines = open(os.path.join(wf.labbook.root_dir, 'input', 'testfile')).read() assert 'conflicting-change-original-user' == flines
def test_sync___detect_merge_conflict(self, mock_labbook_lfs_disabled, mock_config_file): """ test import_from_remote method """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) bm = BranchManager(lb, username='******') bm.create_branch('test-conflict-branch') fpath = os.path.join(lb.root_dir, 'input', 'testfile') with open(fpath, 'w') as f: f.write('filedata') lb.sweep_uncommitted_changes() wf.sync('test') other_user = '******' wf_other = LabbookWorkflow.import_from_remote( wf.remote, username=other_user, config_file=mock_config_file[0]) bm_other = BranchManager(wf_other.labbook, username=other_user) bm_other.workon_branch('test-conflict-branch') with open(os.path.join(wf_other.labbook.root_dir, 'input', 'testfile'), 'w') as f: f.write('conflicting-change-other-user') wf_other.labbook.sweep_uncommitted_changes() wf_other.sync(username=username) with open(fpath, 'w') as f: f.write('conflicting-change-original-user') wf.labbook.sweep_uncommitted_changes() h = wf.labbook.git.commit_hash with pytest.raises(MergeConflict): n = wf.sync(username=username) assert h == wf.labbook.git.commit_hash
def test_publish__publish_then_import_with_another_user( self, mock_labbook_lfs_disabled, mock_config_file): """ Test cannot publish a project already published. """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) other_user = '******' wf_other = LabbookWorkflow.import_from_remote( remote_url=wf.remote, username=other_user, config_file=mock_config_file[0]) lb_other = wf_other.repository assert lb_other.root_dir != lb.root_dir
def publish_repository(repository: Repository, username: str, access_token: str, remote: Optional[str] = None, public: bool = False, id_token: str = None) -> None: p = os.getpid() logger = LMLogger.get_logger() logger.info(f"(Job {p}) Starting publish_repository({str(repository)})") def update_feedback(msg: str, has_failures: Optional[bool] = None, failure_detail: Optional[str] = None, percent_complete: Optional[float] = None): """Method to update the job's metadata and provide feedback to the UI""" current_job = get_current_job() if not current_job: return if has_failures: current_job.meta['has_failures'] = has_failures if failure_detail: current_job.meta['failure_detail'] = failure_detail if percent_complete: current_job.meta['percent_complete'] = percent_complete current_job.meta['feedback'] = msg current_job.save_meta() logger = LMLogger.get_logger() try: update_feedback("Publish task in queue") with repository.lock(): if isinstance(repository, LabBook): wf = LabbookWorkflow(repository) else: wf = DatasetWorkflow(repository) # type: ignore wf.publish(username=username, access_token=access_token, remote=remote or "origin", public=public, feedback_callback=update_feedback, id_token=id_token) except IOError: raise except Exception as e: logger.exception(e) raise Exception("Could not publish - try to log out and log in again.")
def _mock_import_labbook_from_remote(remote_url, username, config_file): print('X' * 200) lb = InventoryManager(config_file).create_labbook( username, username, remote_url.split('/')[-1]) return LabbookWorkflow(lb)
def import_labbook_from_remote(remote_url: str, username: str, config_file: str = None) -> str: """Return the root directory of the newly imported Project""" p = os.getpid() logger = LMLogger.get_logger() logger.info( f"(Job {p}) Starting import_labbook_from_remote({remote_url}, {username})" ) def update_meta(msg): job = get_current_job() if not job: return if 'feedback' not in job.meta: job.meta['feedback'] = msg else: job.meta['feedback'] = job.meta['feedback'] + f'\n{msg}' job.save_meta() try: toks = remote_url.split("/") if len(toks) > 1: proj_path = f'{toks[-2]}/{toks[-1].replace(".git", "")}' else: proj_path = remote_url update_meta(f"Importing Project from {proj_path!r}...") wf = LabbookWorkflow.import_from_remote(remote_url, username, config_file) update_meta(f"Imported Project {wf.labbook.name}!") return wf.labbook.root_dir except Exception as e: update_meta(f"Could not import Project from {remote_url}.") logger.exception(f"(Job {p}) Error on import_labbook_from_remote: {e}") raise
def test_import_from_remote__nominal(self, mock_labbook_lfs_disabled, mock_config_file): """ test import_from_remote method """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) other_user = '******' wf_other = LabbookWorkflow.import_from_remote( wf.remote, username=other_user, config_file=mock_config_file[0]) # The remotes must be the same, cause it's the same remote repo assert wf_other.remote == wf.remote # The actual path on disk will be different, though assert wf_other.repository != wf.repository # Check imported into namespace of original owner (testuser) assert f'{other_user}/{username}/labbooks/labbook1' in wf_other.repository.root_dir
def test_publish__publish_then_import_then_sync(self, mock_labbook_lfs_disabled, mock_config_file): """ Test cannot publish a project already published. """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) other_user = '******' wf_other = LabbookWorkflow.import_from_remote( remote_url=wf.remote, username=other_user, config_file=mock_config_file[0]) with open(os.path.join(wf_other.repository.root_dir, 'testfile'), 'w') as f: f.write('filedata') wf_other.repository.sweep_uncommitted_changes() wf_other.sync(username=other_user) commit_hash = wf_other.repository.git.commit_hash assert wf.repository.git.commit_hash != commit_hash wf.sync(username=username) assert len(wf.repository.git.commit_hash) == len(commit_hash) assert wf.repository.git.commit_hash == commit_hash
def test_reset__reset_local_change_same_owner(self, mock_labbook_lfs_disabled, mock_config_file): """ test reset performs no operation when there's nothing to do """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) commit_to_check = lb.git.commit_hash # Make some change locally and commit fpath = os.path.join(lb.root_dir, 'input', 'testfile') with open(fpath, 'w') as f: f.write('filedata') lb.sweep_uncommitted_changes() assert lb.git.commit_hash != commit_to_check # Make an UNTRACKED change locally, make sure it gets clared up untracked_file = os.path.join(lb.root_dir, 'output', 'untracked-file') with open(untracked_file, 'w') as f: f.write('untracked data') # Do a reset and make sure state resets appropriately wf.reset(username=username) assert lb.git.commit_hash == commit_to_check assert not os.path.exists(fpath) assert not os.path.exists(untracked_file) remote_hash = call_subprocess('git log -n 1 --oneline'.split(), cwd=wf.remote).split()[0] assert remote_hash in lb.git.commit_hash
def test_reset__no_op(self, mock_labbook_lfs_disabled, mock_config_file): """ test reset performs no operation when there's nothing to do """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.reset(username=username) wf.publish(username=username)
def test_publish__cannot_overwrite(self, mock_labbook_lfs_disabled): """ Test cannot publish a project already published. """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) with pytest.raises(GitWorkflowException): wf.publish(username=username)
def test_sync___push_up_then_sync(self, mock_labbook_lfs_disabled, mock_config_file): """ test import_from_remote method """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) bm = BranchManager(lb, username='******') bm.create_branch('new-branch-to-push') wf.sync('test') # Make some change locally and commit, then sync. fpath = os.path.join(lb.root_dir, 'input', 'testfile') with open(fpath, 'w') as f: f.write('filedata') lb.sweep_uncommitted_changes() wf.sync(username=username)
def test_migrate_old_schema_1_project(self, mock_config_file): """ Test migrating a very old schema 1/gm.workspace LabBook """ p = resource_filename('gtmcore', 'workflows') p2 = os.path.join(p, 'tests', 'snappy.zip') with tempfile.TemporaryDirectory() as td: call_subprocess(f"unzip {p2} -d {td}".split(), cwd=td) temp_lb_path = os.path.join(td, 'snappy') # Tests backwards compatibility (test.zip is a very old schema 1 LabBook) lb = InventoryManager( mock_config_file[0]).load_labbook_from_directory(temp_lb_path) wf = LabbookWorkflow(lb) wf.labbook.remove_remote() wf.migrate() # Test that current branch is as appropriate assert wf.labbook.active_branch == 'master' # Test that there is an activity record indicate migration assert any([ 'Migrate schema to 2' in c['message'] for c in wf.labbook.git.log()[:5] ]) # Test schema has successfully rolled to 2 assert wf.labbook.schema == 2 # Test that untracked space exists (if we add something to untracked space) assert wf.labbook.is_repo_clean with open( os.path.join(lb.root_dir, 'output/untracked', 'untracked-file'), 'wb') as fb: fb.write(b'cat' * 100) assert wf.labbook.is_repo_clean
def test_sync___push_up_new_branch(self, mock_labbook_lfs_disabled, mock_config_file): """ test import_from_remote method """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) bm = BranchManager(lb, username='******') bm.create_branch('new-branch-to-push') assert 'new-branch-to-push' not in bm.branches_remote wf.sync('test') assert 'new-branch-to-push' in bm.branches_remote
def mutate_and_get_payload(cls, root, info, owner, labbook_name, remote_url, client_mutation_id=None): username = get_logged_in_username() logger.info(f"Importing remote labbook from {remote_url}") lb = LabBook(author=get_logged_in_author()) default_remote = lb.client_config.config['git']['default_remote'] admin_service = None for remote in lb.client_config.config['git']['remotes']: if default_remote == remote: admin_service = lb.client_config.config['git']['remotes'][ remote]['admin_service'] break # Extract valid Bearer token if hasattr(info.context, 'headers' ) and "HTTP_AUTHORIZATION" in info.context.headers.environ: token = parse_token( info.context.headers.environ["HTTP_AUTHORIZATION"]) else: raise ValueError( "Authorization header not provided. Must have a valid session to query for collaborators" ) gl_mgr = GitLabManager(default_remote, admin_service=admin_service, access_token=token) gl_mgr.configure_git_credentials(default_remote, username) wf = LabbookWorkflow.import_from_remote(remote_url, username=username) import_owner = InventoryManager().query_owner(wf.labbook) # TODO: Fix cursor implementation, this currently doesn't make sense cursor = base64.b64encode(f"{0}".encode('utf-8')) lbedge = LabbookConnection.Edge(node=Labbook(owner=import_owner, name=wf.labbook.name), cursor=cursor) return ImportRemoteLabbook(new_labbook_edge=lbedge)
def test_sync___simple_push_to_master(self, mock_labbook_lfs_disabled, mock_config_file): """ test import_from_remote method """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) fpath = os.path.join(lb.root_dir, 'input', 'testfile') with open(fpath, 'w') as f: f.write('filedata') lb.sweep_uncommitted_changes() wf.sync(username=username) # Check hash on remote - make sure it matches local. remote_hash = call_subprocess('git log -n 1 --oneline'.split(), cwd=wf.remote).split()[0] assert remote_hash in lb.git.commit_hash
def test_should_migrate_on_old_project(self, mock_config_file): p = resource_filename('gtmcore', 'workflows') p2 = os.path.join(p, 'tests', 'lb-to-migrate-197b6a.zip') with tempfile.TemporaryDirectory() as tempdir: lbp = shutil.copyfile(p2, os.path.join(tempdir, 'lb-to-migrate.zip')) subprocess.run(f'unzip lb-to-migrate.zip'.split(), check=True, cwd=tempdir) im = InventoryManager(mock_config_file[0]) lb = im.load_labbook_from_directory( os.path.join(tempdir, 'lb-to-migrate')) wf = LabbookWorkflow(lb) assert wf.should_migrate() is True wf.migrate() assert wf.labbook.active_branch == 'master' assert wf.should_migrate() is False wf.labbook.git.checkout('gm.workspace') assert wf.should_migrate() is False
def test_publish__simple(self, mock_labbook_lfs_disabled): """Test a simple publish and ensuring master is active branch. """ username = '******' lb = mock_labbook_lfs_disabled[2] bm = BranchManager(lb, username) bm.create_branch('test-local-only') assert bm.branches_remote == [] assert bm.branches_local == ['master', 'test-local-only'] wf = LabbookWorkflow(lb) # Test you can only publish on master. with pytest.raises(GitWorkflowException): wf.publish(username=username) assert wf.remote is None # Once we return to master branch, then we can publish. bm.workon_branch(bm.workspace_branch) wf.publish(username=username) assert os.path.exists(wf.remote) # Assert that publish only pushes up the master branch. assert bm.branches_local == ['master', 'test-local-only'] assert bm.branches_remote == ['master']
def test_import_from_remote__linked_dataset(self, mock_labbook_lfs_disabled, mock_config_file): """ test importing a project with a linked dataset""" def dispatcher_mock(self, function_ref, kwargs, metadata): assert kwargs['logged_in_username'] == 'other-test-user2' assert kwargs['dataset_owner'] == 'testuser' assert kwargs['dataset_name'] == 'test-ds' # Inject mocked config file kwargs['config_file'] = mock_config_file[0] # Stop patching so job gets scheduled for real dispatcher_patch.stop() # Call same method as in mutation d = Dispatcher() res = d.dispatch_task( gtmcore.dispatcher.dataset_jobs.check_and_import_dataset, kwargs=kwargs, metadata=metadata) return res username = '******' lb = mock_labbook_lfs_disabled[2] im = InventoryManager(config_file=mock_labbook_lfs_disabled[0]) ds = im.create_dataset(username, username, 'test-ds', storage_type='gigantum_object_v1') # Publish dataset dataset_wf = DatasetWorkflow(ds) dataset_wf.publish(username=username) # Link to project im.link_dataset_to_labbook(dataset_wf.remote, username, username, lb) # Publish project labbook_wf = LabbookWorkflow(lb) labbook_wf.publish(username=username) # Patch dispatch_task so you can inject the mocked config file dispatcher_patch = patch.object(Dispatcher, 'dispatch_task', dispatcher_mock) dispatcher_patch.start() # Import project, triggering an auto-import of the dataset other_user = '******' wf_other = LabbookWorkflow.import_from_remote( labbook_wf.remote, username=other_user, config_file=mock_config_file[0]) # The remotes must be the same, cause it's the same remote repo assert wf_other.remote == labbook_wf.remote # The actual path on disk will be different, though assert wf_other.repository != labbook_wf.repository # Check imported into namespace of original owner (testuser) assert f'{other_user}/{username}/labbooks/labbook1' in wf_other.repository.root_dir cnt = 0 while cnt < 20: try: im_other_user = InventoryManager( config_file=mock_config_file[0]) ds = im_other_user.load_dataset(other_user, username, 'test-ds') break except InventoryException: cnt += 1 time.sleep(1) assert cnt < 20 assert ds.name == 'test-ds' assert ds.namespace == username assert mock_config_file[1] in ds.root_dir
def _should_migrate(lb): wf = LabbookWorkflow(lb) return wf.should_migrate()
def test_checkout__linked_dataset(self, mock_labbook_lfs_disabled, mock_config_file): """ test checking out a branch in a project that pulls in a linked dataset""" def dispatcher_mock(self, function_ref, kwargs, metadata): assert kwargs['logged_in_username'] == 'other-test-user2' assert kwargs['dataset_owner'] == 'testuser' assert kwargs['dataset_name'] == 'test-ds' # Inject mocked config file kwargs['config_file'] = mock_config_file[0] # Stop patching so job gets scheduled for real dispatcher_patch.stop() # Call same method as in mutation d = Dispatcher() res = d.dispatch_task( gtmcore.dispatcher.dataset_jobs.check_and_import_dataset, kwargs=kwargs, metadata=metadata) return res username = '******' lb = mock_labbook_lfs_disabled[2] im = InventoryManager(config_file=mock_labbook_lfs_disabled[0]) ds = im.create_dataset(username, username, 'test-ds', storage_type='gigantum_object_v1') # Publish dataset dataset_wf = DatasetWorkflow(ds) dataset_wf.publish(username=username) # Publish project labbook_wf = LabbookWorkflow(lb) labbook_wf.publish(username=username) # Switch branches labbook_wf.labbook.checkout_branch(branch_name="dataset-branch", new=True) # Link to project im.link_dataset_to_labbook(dataset_wf.remote, username, username, labbook_wf.labbook) # Publish branch labbook_wf.sync(username=username) # Import project other_user = '******' wf_other = LabbookWorkflow.import_from_remote( labbook_wf.remote, username=other_user, config_file=mock_config_file[0]) # The remotes must be the same, cause it's the same remote repo assert wf_other.remote == labbook_wf.remote assert wf_other.repository != labbook_wf.repository assert f'{other_user}/{username}/labbooks/labbook1' in wf_other.repository.root_dir with pytest.raises(InventoryException): im_other_user = InventoryManager(config_file=mock_config_file[0]) ds = im_other_user.load_dataset(other_user, username, 'test-ds') # Patch dispatch_task so you can inject the mocked config file dispatcher_patch = patch.object(Dispatcher, 'dispatch_task', dispatcher_mock) dispatcher_patch.start() # Checkout the branch assert wf_other.labbook.active_branch == "master" wf_other.checkout(username=other_user, branch_name="dataset-branch") cnt = 0 while cnt < 20: try: im_other_user = InventoryManager( config_file=mock_config_file[0]) ds = im_other_user.load_dataset(other_user, username, 'test-ds') break except InventoryException: cnt += 1 time.sleep(1) assert cnt < 20 assert ds.name == 'test-ds' assert ds.namespace == username assert mock_config_file[1] in ds.root_dir assert wf_other.labbook.active_branch == "dataset-branch"