def test_reset__reset_local_change_same_owner(self, mock_labbook_lfs_disabled, mock_config_file): """ test reset performs no operation when there's nothing to do """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) commit_to_check = lb.git.commit_hash # Make some change locally and commit fpath = os.path.join(lb.root_dir, 'input', 'testfile') with open(fpath, 'w') as f: f.write('filedata') lb.sweep_uncommitted_changes() assert lb.git.commit_hash != commit_to_check # Make an UNTRACKED change locally, make sure it gets clared up untracked_file = os.path.join(lb.root_dir, 'output', 'untracked-file') with open(untracked_file, 'w') as f: f.write('untracked data') # Do a reset and make sure state resets appropriately wf.reset(username=username) assert lb.git.commit_hash == commit_to_check assert not os.path.exists(fpath) assert not os.path.exists(untracked_file) remote_hash = call_subprocess('git log -n 1 --oneline'.split(), cwd=wf.remote).split()[0] assert remote_hash in lb.git.commit_hash
def test_sync___override_merge_conflict_ours(self, mock_labbook_lfs_disabled, mock_config_file): """ test sync, with override in case of merge conflict. """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) bm = BranchManager(lb, username='******') bm.create_branch('test-conflict-branch') fpath = os.path.join(lb.root_dir, 'input', 'testfile') with open(fpath, 'w') as f: f.write('filedata') lb.sweep_uncommitted_changes() wf.sync('test') other_user = '******' wf_other = LabbookWorkflow.import_from_remote( wf.remote, username=other_user, config_file=mock_config_file[0]) bm_other = BranchManager(wf_other.labbook, username=other_user) bm_other.workon_branch('test-conflict-branch') with open(os.path.join(wf_other.labbook.root_dir, 'input', 'testfile'), 'w') as f: f.write('conflicting-change-other-user') wf_other.labbook.sweep_uncommitted_changes() wf_other.sync(username=username) fpath = os.path.join(wf.labbook.root_dir, 'input', 'testfile') with open(fpath, 'w') as f: f.write('conflicting-change-original-user') wf.labbook.sweep_uncommitted_changes() n = wf.sync(username=username, override=MergeOverride.OURS) flines = open(os.path.join(wf.labbook.root_dir, 'input', 'testfile')).read() assert 'conflicting-change-original-user' == flines
def test_reset__no_op(self, mock_labbook_lfs_disabled, mock_config_file): """ test reset performs no operation when there's nothing to do """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.reset(username=username) wf.publish(username=username)
def publish_repository(repository: Repository, username: str, access_token: str, remote: Optional[str] = None, public: bool = False, id_token: str = None) -> None: p = os.getpid() logger = LMLogger.get_logger() logger.info(f"(Job {p}) Starting publish_repository({str(repository)})") def update_meta(msg): job = get_current_job() if not job: return if 'feedback' not in job.meta: job.meta['feedback'] = msg else: job.meta['feedback'] = job.meta['feedback'] + f'\n{msg}' job.save_meta() try: with repository.lock(): if isinstance(repository, LabBook): wf = LabbookWorkflow(repository) else: wf = DatasetWorkflow(repository) # type: ignore wf.publish(username=username, access_token=access_token, remote=remote or "origin", public=public, feedback_callback=update_meta, id_token=id_token) except Exception as e: logger.exception(f"(Job {p}) Error on publish_repository: {e}") raise
def test_sync___detect_merge_conflict(self, mock_labbook_lfs_disabled, mock_config_file): """ test import_from_remote method """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) bm = BranchManager(lb, username='******') bm.create_branch('test-conflict-branch') fpath = os.path.join(lb.root_dir, 'input', 'testfile') with open(fpath, 'w') as f: f.write('filedata') lb.sweep_uncommitted_changes() wf.sync('test') other_user = '******' wf_other = LabbookWorkflow.import_from_remote( wf.remote, username=other_user, config_file=mock_config_file[0]) bm_other = BranchManager(wf_other.labbook, username=other_user) bm_other.workon_branch('test-conflict-branch') with open(os.path.join(wf_other.labbook.root_dir, 'input', 'testfile'), 'w') as f: f.write('conflicting-change-other-user') wf_other.labbook.sweep_uncommitted_changes() wf_other.sync(username=username) with open(fpath, 'w') as f: f.write('conflicting-change-original-user') wf.labbook.sweep_uncommitted_changes() h = wf.labbook.git.commit_hash with pytest.raises(MergeConflict): n = wf.sync(username=username) assert h == wf.labbook.git.commit_hash
def test_publish__publish_then_import_then_sync(self, mock_labbook_lfs_disabled, mock_config_file): """ Test cannot publish a project already published. """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) other_user = '******' wf_other = LabbookWorkflow.import_from_remote( remote_url=wf.remote, username=other_user, config_file=mock_config_file[0]) with open(os.path.join(wf_other.repository.root_dir, 'testfile'), 'w') as f: f.write('filedata') wf_other.repository.sweep_uncommitted_changes() wf_other.sync(username=other_user) commit_hash = wf_other.repository.git.commit_hash assert wf.repository.git.commit_hash != commit_hash wf.sync(username=username) assert len(wf.repository.git.commit_hash) == len(commit_hash) assert wf.repository.git.commit_hash == commit_hash
def test_reset_branch_to_remote(self, fixture_working_dir, mock_create_labbooks_no_lfs): # Mock the request context so a fake authorization header is present builder = EnvironBuilder(path='/labbook', method='POST', headers={'Authorization': 'Bearer AJDFHASD'}) env = builder.get_environ() req = Request(environ=env) im = InventoryManager(mock_create_labbooks_no_lfs[0]) test_user_lb = im.load_labbook('default', 'default', 'labbook1') wf = LabbookWorkflow(test_user_lb) wf.publish(username='******') hash_original = wf.labbook.git.commit_hash new_file_path = os.path.join(wf.labbook.root_dir, 'input', 'new-file') with open(new_file_path, 'w') as f: f.write('File data') wf.labbook.sweep_uncommitted_changes() hash_before_reset = wf.labbook.git.commit_hash publish_query = f""" mutation c {{ resetBranchToRemote(input: {{ labbookName: "labbook1", owner: "default" }}) {{ labbook {{ activeBranchName }} }} }} """ r = mock_create_labbooks_no_lfs[2].execute(publish_query, context_value=req) assert 'errors' not in r assert wf.labbook.git.commit_hash == hash_original
def test_sync_1(self, mock_create_labbooks_no_lfs, mock_config_file): # Setup responses mock for this test responses.add(responses.GET, 'https://usersrv.gigantum.io/key', json={'key': 'afaketoken'}, status=200) im = InventoryManager(mock_create_labbooks_no_lfs[0]) test_user_lb = im.load_labbook('default', 'default', 'labbook1') test_user_wf = LabbookWorkflow(test_user_lb) test_user_wf.publish('default') # Mock the request context so a fake authorization header is present builder = EnvironBuilder(path='/labbook', method='POST', headers={'Authorization': 'Bearer AJDFHASD'}) env = builder.get_environ() req = Request(environ=env) sally_wf = LabbookWorkflow.import_from_remote(test_user_wf.remote, 'sally', config_file=mock_config_file[0]) sally_lb = sally_wf.labbook FileOperations.makedir(sally_lb, relative_path='code/sally-dir', create_activity_record=True) sally_wf.sync('sally') sync_query = """ mutation x { syncLabbook(input: { labbookName: "labbook1", owner: "default" }) { jobKey } } """ r = mock_create_labbooks_no_lfs[2].execute(sync_query, context_value=req) assert 'errors' not in r
def test_publish__cannot_overwrite(self, mock_labbook_lfs_disabled): """ Test cannot publish a project already published. """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) with pytest.raises(GitWorkflowException): wf.publish(username=username)
def test_sync___push_up_new_branch(self, mock_labbook_lfs_disabled, mock_config_file): """ test import_from_remote method """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) bm = BranchManager(lb, username='******') bm.create_branch('new-branch-to-push') assert 'new-branch-to-push' not in bm.branches_remote wf.sync('test') assert 'new-branch-to-push' in bm.branches_remote
def test_publish__publish_then_import_with_another_user( self, mock_labbook_lfs_disabled, mock_config_file): """ Test cannot publish a project already published. """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) other_user = '******' wf_other = LabbookWorkflow.import_from_remote( remote_url=wf.remote, username=other_user, config_file=mock_config_file[0]) lb_other = wf_other.repository assert lb_other.root_dir != lb.root_dir
def publish_repository(repository: Repository, username: str, access_token: str, remote: Optional[str] = None, public: bool = False, id_token: str = None) -> None: p = os.getpid() logger = LMLogger.get_logger() logger.info(f"(Job {p}) Starting publish_repository({str(repository)})") def update_feedback(msg: str, has_failures: Optional[bool] = None, failure_detail: Optional[str] = None, percent_complete: Optional[float] = None): """Method to update the job's metadata and provide feedback to the UI""" current_job = get_current_job() if not current_job: return if has_failures: current_job.meta['has_failures'] = has_failures if failure_detail: current_job.meta['failure_detail'] = failure_detail if percent_complete: current_job.meta['percent_complete'] = percent_complete current_job.meta['feedback'] = msg current_job.save_meta() logger = LMLogger.get_logger() try: update_feedback("Publish task in queue") with repository.lock(): if isinstance(repository, LabBook): wf = LabbookWorkflow(repository) else: wf = DatasetWorkflow(repository) # type: ignore wf.publish(username=username, access_token=access_token, remote=remote or "origin", public=public, feedback_callback=update_feedback, id_token=id_token) except IOError: raise except Exception as e: logger.exception(e) raise Exception("Could not publish - try to log out and log in again.")
def test_sync___simple_push_to_master(self, mock_labbook_lfs_disabled, mock_config_file): """ test import_from_remote method """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) fpath = os.path.join(lb.root_dir, 'input', 'testfile') with open(fpath, 'w') as f: f.write('filedata') lb.sweep_uncommitted_changes() wf.sync(username=username) # Check hash on remote - make sure it matches local. remote_hash = call_subprocess('git log -n 1 --oneline'.split(), cwd=wf.remote).split()[0] assert remote_hash in lb.git.commit_hash
def test_import_from_remote__nominal(self, mock_labbook_lfs_disabled, mock_config_file): """ test import_from_remote method """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) other_user = '******' wf_other = LabbookWorkflow.import_from_remote( wf.remote, username=other_user, config_file=mock_config_file[0]) # The remotes must be the same, cause it's the same remote repo assert wf_other.remote == wf.remote # The actual path on disk will be different, though assert wf_other.repository != wf.repository # Check imported into namespace of original owner (testuser) assert f'{other_user}/{username}/labbooks/labbook1' in wf_other.repository.root_dir
def test_sync___push_up_then_sync(self, mock_labbook_lfs_disabled, mock_config_file): """ test import_from_remote method """ username = '******' lb = mock_labbook_lfs_disabled[2] wf = LabbookWorkflow(lb) wf.publish(username=username) bm = BranchManager(lb, username='******') bm.create_branch('new-branch-to-push') wf.sync('test') # Make some change locally and commit, then sync. fpath = os.path.join(lb.root_dir, 'input', 'testfile') with open(fpath, 'w') as f: f.write('filedata') lb.sweep_uncommitted_changes() wf.sync(username=username)
def test_publish__simple(self, mock_labbook_lfs_disabled): """Test a simple publish and ensuring master is active branch. """ username = '******' lb = mock_labbook_lfs_disabled[2] bm = BranchManager(lb, username) bm.create_branch('test-local-only') assert bm.branches_remote == [] assert bm.branches_local == ['master', 'test-local-only'] wf = LabbookWorkflow(lb) # Test you can only publish on master. with pytest.raises(GitWorkflowException): wf.publish(username=username) assert wf.remote is None # Once we return to master branch, then we can publish. bm.workon_branch(bm.workspace_branch) wf.publish(username=username) assert os.path.exists(wf.remote) # Assert that publish only pushes up the master branch. assert bm.branches_local == ['master', 'test-local-only'] assert bm.branches_remote == ['master']
def test_commits_ahead_behind(self, fixture_working_dir_lfs_disabled): with responses.RequestsMock() as rsps: rsps.add(responses.GET, 'https://usersrv.gigantum.io/key', json={'key': 'afaketoken'}, status=200) config_file, client = fixture_working_dir_lfs_disabled[0], \ fixture_working_dir_lfs_disabled[2] im = InventoryManager(config_file) lb = im.create_labbook(UT_USERNAME, UT_USERNAME, UT_LBNAME, description="tester") bm = BranchManager(lb, username=UT_USERNAME) bm.create_branch('new-branch-1') bm.create_branch('new-branch-2') bm.workon_branch('master') q = f""" {{ labbook(name: "{UT_LBNAME}", owner: "{UT_USERNAME}") {{ branches {{ branchName isLocal isRemote isActive commitsAhead commitsBehind }} }} }} """ r = client.execute(q) assert 'errors' not in r assert len(r['data']['labbook']['branches']) == 3 assert r['data']['labbook']['branches'][0][ 'branchName'] == 'master' assert r['data']['labbook']['branches'][0][ 'isLocal'] is True, "Should be local" assert r['data']['labbook']['branches'][0][ 'isRemote'] is False, "not published yet" assert r['data']['labbook']['branches'][0]['isActive'] is True assert r['data']['labbook']['branches'][0]['commitsAhead'] == 0 assert r['data']['labbook']['branches'][0]['commitsBehind'] == 0 # Make a remote change! username = '******' wf = LabbookWorkflow(lb) wf.publish(username=username) other_user = '******' wf_other = LabbookWorkflow.import_from_remote( remote_url=wf.remote, username=other_user, config_file=lb.client_config.config_file) with open(os.path.join(wf_other.repository.root_dir, 'testfile'), 'w') as f: f.write('filedata') wf_other.repository.sweep_uncommitted_changes() wf_other.sync(username=other_user) r = client.execute(q) assert 'errors' not in r assert len(r['data']['labbook']['branches']) == 3 assert r['data']['labbook']['branches'][0][ 'branchName'] == 'master' assert r['data']['labbook']['branches'][0][ 'isLocal'] is True, "Should be local" assert r['data']['labbook']['branches'][0][ 'isRemote'] is True, "There should be a remote" assert r['data']['labbook']['branches'][0]['isActive'] is True assert r['data']['labbook']['branches'][0]['commitsAhead'] == 0 assert r['data']['labbook']['branches'][0]['commitsBehind'] == 1 # Make a local change! lb.write_readme("blah") r = client.execute(q) assert 'errors' not in r assert len(r['data']['labbook']['branches']) == 3 assert r['data']['labbook']['branches'][0][ 'branchName'] == 'master' assert r['data']['labbook']['branches'][0][ 'isLocal'] is True, "Should be local" assert r['data']['labbook']['branches'][0][ 'isRemote'] is True, "There should be a remote" assert r['data']['labbook']['branches'][0]['isActive'] is True assert r['data']['labbook']['branches'][0]['commitsAhead'] == 1 assert r['data']['labbook']['branches'][0]['commitsBehind'] == 1 # Sync wf.sync(username=username) r = client.execute(q) assert 'errors' not in r assert len(r['data']['labbook']['branches']) == 3 assert r['data']['labbook']['branches'][0][ 'branchName'] == 'master' assert r['data']['labbook']['branches'][0][ 'isLocal'] is True, "Should be local" assert r['data']['labbook']['branches'][0][ 'isRemote'] is True, "There should be a remote" assert r['data']['labbook']['branches'][0]['isActive'] is True assert r['data']['labbook']['branches'][0]['commitsAhead'] == 0 assert r['data']['labbook']['branches'][0]['commitsBehind'] == 0
def test_import_from_remote__linked_dataset(self, mock_labbook_lfs_disabled, mock_config_file): """ test importing a project with a linked dataset""" def dispatcher_mock(self, function_ref, kwargs, metadata): assert kwargs['logged_in_username'] == 'other-test-user2' assert kwargs['dataset_owner'] == 'testuser' assert kwargs['dataset_name'] == 'test-ds' # Inject mocked config file kwargs['config_file'] = mock_config_file[0] # Stop patching so job gets scheduled for real dispatcher_patch.stop() # Call same method as in mutation d = Dispatcher() res = d.dispatch_task( gtmcore.dispatcher.dataset_jobs.check_and_import_dataset, kwargs=kwargs, metadata=metadata) return res username = '******' lb = mock_labbook_lfs_disabled[2] im = InventoryManager(config_file=mock_labbook_lfs_disabled[0]) ds = im.create_dataset(username, username, 'test-ds', storage_type='gigantum_object_v1') # Publish dataset dataset_wf = DatasetWorkflow(ds) dataset_wf.publish(username=username) # Link to project im.link_dataset_to_labbook(dataset_wf.remote, username, username, lb) # Publish project labbook_wf = LabbookWorkflow(lb) labbook_wf.publish(username=username) # Patch dispatch_task so you can inject the mocked config file dispatcher_patch = patch.object(Dispatcher, 'dispatch_task', dispatcher_mock) dispatcher_patch.start() # Import project, triggering an auto-import of the dataset other_user = '******' wf_other = LabbookWorkflow.import_from_remote( labbook_wf.remote, username=other_user, config_file=mock_config_file[0]) # The remotes must be the same, cause it's the same remote repo assert wf_other.remote == labbook_wf.remote # The actual path on disk will be different, though assert wf_other.repository != labbook_wf.repository # Check imported into namespace of original owner (testuser) assert f'{other_user}/{username}/labbooks/labbook1' in wf_other.repository.root_dir cnt = 0 while cnt < 20: try: im_other_user = InventoryManager( config_file=mock_config_file[0]) ds = im_other_user.load_dataset(other_user, username, 'test-ds') break except InventoryException: cnt += 1 time.sleep(1) assert cnt < 20 assert ds.name == 'test-ds' assert ds.namespace == username assert mock_config_file[1] in ds.root_dir
def test_checkout__linked_dataset(self, mock_labbook_lfs_disabled, mock_config_file): """ test checking out a branch in a project that pulls in a linked dataset""" def dispatcher_mock(self, function_ref, kwargs, metadata): assert kwargs['logged_in_username'] == 'other-test-user2' assert kwargs['dataset_owner'] == 'testuser' assert kwargs['dataset_name'] == 'test-ds' # Inject mocked config file kwargs['config_file'] = mock_config_file[0] # Stop patching so job gets scheduled for real dispatcher_patch.stop() # Call same method as in mutation d = Dispatcher() res = d.dispatch_task( gtmcore.dispatcher.dataset_jobs.check_and_import_dataset, kwargs=kwargs, metadata=metadata) return res username = '******' lb = mock_labbook_lfs_disabled[2] im = InventoryManager(config_file=mock_labbook_lfs_disabled[0]) ds = im.create_dataset(username, username, 'test-ds', storage_type='gigantum_object_v1') # Publish dataset dataset_wf = DatasetWorkflow(ds) dataset_wf.publish(username=username) # Publish project labbook_wf = LabbookWorkflow(lb) labbook_wf.publish(username=username) # Switch branches labbook_wf.labbook.checkout_branch(branch_name="dataset-branch", new=True) # Link to project im.link_dataset_to_labbook(dataset_wf.remote, username, username, labbook_wf.labbook) # Publish branch labbook_wf.sync(username=username) # Import project other_user = '******' wf_other = LabbookWorkflow.import_from_remote( labbook_wf.remote, username=other_user, config_file=mock_config_file[0]) # The remotes must be the same, cause it's the same remote repo assert wf_other.remote == labbook_wf.remote assert wf_other.repository != labbook_wf.repository assert f'{other_user}/{username}/labbooks/labbook1' in wf_other.repository.root_dir with pytest.raises(InventoryException): im_other_user = InventoryManager(config_file=mock_config_file[0]) ds = im_other_user.load_dataset(other_user, username, 'test-ds') # Patch dispatch_task so you can inject the mocked config file dispatcher_patch = patch.object(Dispatcher, 'dispatch_task', dispatcher_mock) dispatcher_patch.start() # Checkout the branch assert wf_other.labbook.active_branch == "master" wf_other.checkout(username=other_user, branch_name="dataset-branch") cnt = 0 while cnt < 20: try: im_other_user = InventoryManager( config_file=mock_config_file[0]) ds = im_other_user.load_dataset(other_user, username, 'test-ds') break except InventoryException: cnt += 1 time.sleep(1) assert cnt < 20 assert ds.name == 'test-ds' assert ds.namespace == username assert mock_config_file[1] in ds.root_dir assert wf_other.labbook.active_branch == "dataset-branch"