def test_reset__reset_local_change_same_owner(self,
mock_labbook_lfs_disabled,
mock_config_file):
""" test reset performs no operation when there's nothing to do """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
commit_to_check = lb.git.commit_hash
# Make some change locally and commit
fpath = os.path.join(lb.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('filedata')
lb.sweep_uncommitted_changes()
assert lb.git.commit_hash != commit_to_check
# Make an UNTRACKED change locally, make sure it gets clared up
untracked_file = os.path.join(lb.root_dir, 'output', 'untracked-file')
with open(untracked_file, 'w') as f:
f.write('untracked data')
# Do a reset and make sure state resets appropriately
wf.reset(username=username)
assert lb.git.commit_hash == commit_to_check
assert not os.path.exists(fpath)
assert not os.path.exists(untracked_file)
remote_hash = call_subprocess('git log -n 1 --oneline'.split(),
cwd=wf.remote).split()[0]
assert remote_hash in lb.git.commit_hash
def test_sync___override_merge_conflict_ours(self,
mock_labbook_lfs_disabled,
mock_config_file):
""" test sync, with override in case of merge conflict. """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
bm = BranchManager(lb, username='******')
bm.create_branch('test-conflict-branch')
fpath = os.path.join(lb.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('filedata')
lb.sweep_uncommitted_changes()
wf.sync('test')
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
wf.remote, username=other_user, config_file=mock_config_file[0])
bm_other = BranchManager(wf_other.labbook, username=other_user)
bm_other.workon_branch('test-conflict-branch')
with open(os.path.join(wf_other.labbook.root_dir, 'input', 'testfile'),
'w') as f:
f.write('conflicting-change-other-user')
wf_other.labbook.sweep_uncommitted_changes()
wf_other.sync(username=username)
fpath = os.path.join(wf.labbook.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('conflicting-change-original-user')
wf.labbook.sweep_uncommitted_changes()
n = wf.sync(username=username, override=MergeOverride.OURS)
flines = open(os.path.join(wf.labbook.root_dir, 'input',
'testfile')).read()
assert 'conflicting-change-original-user' == flines
def test_reset__no_op(self, mock_labbook_lfs_disabled, mock_config_file):
""" test reset performs no operation when there's nothing to do """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.reset(username=username)
wf.publish(username=username)
def publish_repository(repository: Repository, username: str, access_token: str,
remote: Optional[str] = None, public: bool = False, id_token: str = None) -> None:
p = os.getpid()
logger = LMLogger.get_logger()
logger.info(f"(Job {p}) Starting publish_repository({str(repository)})")
def update_meta(msg):
job = get_current_job()
if not job:
return
if 'feedback' not in job.meta:
job.meta['feedback'] = msg
else:
job.meta['feedback'] = job.meta['feedback'] + f'\n{msg}'
job.save_meta()
try:
with repository.lock():
if isinstance(repository, LabBook):
wf = LabbookWorkflow(repository)
else:
wf = DatasetWorkflow(repository) # type: ignore
wf.publish(username=username, access_token=access_token, remote=remote or "origin",
public=public, feedback_callback=update_meta, id_token=id_token)
except Exception as e:
logger.exception(f"(Job {p}) Error on publish_repository: {e}")
raise
def test_sync___detect_merge_conflict(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test import_from_remote method """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
bm = BranchManager(lb, username='******')
bm.create_branch('test-conflict-branch')
fpath = os.path.join(lb.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('filedata')
lb.sweep_uncommitted_changes()
wf.sync('test')
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
wf.remote, username=other_user, config_file=mock_config_file[0])
bm_other = BranchManager(wf_other.labbook, username=other_user)
bm_other.workon_branch('test-conflict-branch')
with open(os.path.join(wf_other.labbook.root_dir, 'input', 'testfile'),
'w') as f:
f.write('conflicting-change-other-user')
wf_other.labbook.sweep_uncommitted_changes()
wf_other.sync(username=username)
with open(fpath, 'w') as f:
f.write('conflicting-change-original-user')
wf.labbook.sweep_uncommitted_changes()
h = wf.labbook.git.commit_hash
with pytest.raises(MergeConflict):
n = wf.sync(username=username)
assert h == wf.labbook.git.commit_hash
def test_publish__publish_then_import_then_sync(self,
mock_labbook_lfs_disabled,
mock_config_file):
""" Test cannot publish a project already published. """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
remote_url=wf.remote,
username=other_user,
config_file=mock_config_file[0])
with open(os.path.join(wf_other.repository.root_dir, 'testfile'),
'w') as f:
f.write('filedata')
wf_other.repository.sweep_uncommitted_changes()
wf_other.sync(username=other_user)
commit_hash = wf_other.repository.git.commit_hash
assert wf.repository.git.commit_hash != commit_hash
wf.sync(username=username)
assert len(wf.repository.git.commit_hash) == len(commit_hash)
assert wf.repository.git.commit_hash == commit_hash
def test_reset_branch_to_remote(self, fixture_working_dir, mock_create_labbooks_no_lfs):
# Mock the request context so a fake authorization header is present
builder = EnvironBuilder(path='/labbook', method='POST', headers={'Authorization': 'Bearer AJDFHASD'})
env = builder.get_environ()
req = Request(environ=env)
im = InventoryManager(mock_create_labbooks_no_lfs[0])
test_user_lb = im.load_labbook('default', 'default', 'labbook1')
wf = LabbookWorkflow(test_user_lb)
wf.publish(username='******')
hash_original = wf.labbook.git.commit_hash
new_file_path = os.path.join(wf.labbook.root_dir, 'input', 'new-file')
with open(new_file_path, 'w') as f: f.write('File data')
wf.labbook.sweep_uncommitted_changes()
hash_before_reset = wf.labbook.git.commit_hash
publish_query = f"""
mutation c {{
resetBranchToRemote(input: {{
labbookName: "labbook1",
owner: "default"
}}) {{
labbook {{
activeBranchName
}}
}}
}}
"""
r = mock_create_labbooks_no_lfs[2].execute(publish_query, context_value=req)
assert 'errors' not in r
assert wf.labbook.git.commit_hash == hash_original
def test_sync_1(self, mock_create_labbooks_no_lfs, mock_config_file):
# Setup responses mock for this test
responses.add(responses.GET, 'https://usersrv.gigantum.io/key',
json={'key': 'afaketoken'}, status=200)
im = InventoryManager(mock_create_labbooks_no_lfs[0])
test_user_lb = im.load_labbook('default', 'default', 'labbook1')
test_user_wf = LabbookWorkflow(test_user_lb)
test_user_wf.publish('default')
# Mock the request context so a fake authorization header is present
builder = EnvironBuilder(path='/labbook', method='POST', headers={'Authorization': 'Bearer AJDFHASD'})
env = builder.get_environ()
req = Request(environ=env)
sally_wf = LabbookWorkflow.import_from_remote(test_user_wf.remote, 'sally', config_file=mock_config_file[0])
sally_lb = sally_wf.labbook
FileOperations.makedir(sally_lb, relative_path='code/sally-dir', create_activity_record=True)
sally_wf.sync('sally')
sync_query = """
mutation x {
syncLabbook(input: {
labbookName: "labbook1",
owner: "default"
}) {
jobKey
}
}
"""
r = mock_create_labbooks_no_lfs[2].execute(sync_query, context_value=req)
assert 'errors' not in r
def test_publish__cannot_overwrite(self, mock_labbook_lfs_disabled):
""" Test cannot publish a project already published. """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
with pytest.raises(GitWorkflowException):
wf.publish(username=username)
def test_sync___push_up_new_branch(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test import_from_remote method """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
bm = BranchManager(lb, username='******')
bm.create_branch('new-branch-to-push')
assert 'new-branch-to-push' not in bm.branches_remote
wf.sync('test')
assert 'new-branch-to-push' in bm.branches_remote
def test_publish__publish_then_import_with_another_user(
self, mock_labbook_lfs_disabled, mock_config_file):
""" Test cannot publish a project already published. """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
remote_url=wf.remote,
username=other_user,
config_file=mock_config_file[0])
lb_other = wf_other.repository
assert lb_other.root_dir != lb.root_dir
def publish_repository(repository: Repository,
username: str,
access_token: str,
remote: Optional[str] = None,
public: bool = False,
id_token: str = None) -> None:
p = os.getpid()
logger = LMLogger.get_logger()
logger.info(f"(Job {p}) Starting publish_repository({str(repository)})")
def update_feedback(msg: str,
has_failures: Optional[bool] = None,
failure_detail: Optional[str] = None,
percent_complete: Optional[float] = None):
"""Method to update the job's metadata and provide feedback to the UI"""
current_job = get_current_job()
if not current_job:
return
if has_failures:
current_job.meta['has_failures'] = has_failures
if failure_detail:
current_job.meta['failure_detail'] = failure_detail
if percent_complete:
current_job.meta['percent_complete'] = percent_complete
current_job.meta['feedback'] = msg
current_job.save_meta()
logger = LMLogger.get_logger()
try:
update_feedback("Publish task in queue")
with repository.lock():
if isinstance(repository, LabBook):
wf = LabbookWorkflow(repository)
else:
wf = DatasetWorkflow(repository) # type: ignore
wf.publish(username=username,
access_token=access_token,
remote=remote or "origin",
public=public,
feedback_callback=update_feedback,
id_token=id_token)
except IOError:
raise
except Exception as e:
logger.exception(e)
raise Exception("Could not publish - try to log out and log in again.")
def test_sync___simple_push_to_master(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test import_from_remote method """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
fpath = os.path.join(lb.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('filedata')
lb.sweep_uncommitted_changes()
wf.sync(username=username)
# Check hash on remote - make sure it matches local.
remote_hash = call_subprocess('git log -n 1 --oneline'.split(),
cwd=wf.remote).split()[0]
assert remote_hash in lb.git.commit_hash
def test_import_from_remote__nominal(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test import_from_remote method """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
wf.remote, username=other_user, config_file=mock_config_file[0])
# The remotes must be the same, cause it's the same remote repo
assert wf_other.remote == wf.remote
# The actual path on disk will be different, though
assert wf_other.repository != wf.repository
# Check imported into namespace of original owner (testuser)
assert f'{other_user}/{username}/labbooks/labbook1' in wf_other.repository.root_dir
def test_sync___push_up_then_sync(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test import_from_remote method """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
bm = BranchManager(lb, username='******')
bm.create_branch('new-branch-to-push')
wf.sync('test')
# Make some change locally and commit, then sync.
fpath = os.path.join(lb.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('filedata')
lb.sweep_uncommitted_changes()
wf.sync(username=username)
def test_publish__simple(self, mock_labbook_lfs_disabled):
"""Test a simple publish and ensuring master is active branch. """
username = '******'
lb = mock_labbook_lfs_disabled[2]
bm = BranchManager(lb, username)
bm.create_branch('test-local-only')
assert bm.branches_remote == []
assert bm.branches_local == ['master', 'test-local-only']
wf = LabbookWorkflow(lb)
# Test you can only publish on master.
with pytest.raises(GitWorkflowException):
wf.publish(username=username)
assert wf.remote is None
# Once we return to master branch, then we can publish.
bm.workon_branch(bm.workspace_branch)
wf.publish(username=username)
assert os.path.exists(wf.remote)
# Assert that publish only pushes up the master branch.
assert bm.branches_local == ['master', 'test-local-only']
assert bm.branches_remote == ['master']
def test_commits_ahead_behind(self, fixture_working_dir_lfs_disabled):
with responses.RequestsMock() as rsps:
rsps.add(responses.GET,
'https://usersrv.gigantum.io/key',
json={'key': 'afaketoken'},
status=200)
config_file, client = fixture_working_dir_lfs_disabled[0], \
fixture_working_dir_lfs_disabled[2]
im = InventoryManager(config_file)
lb = im.create_labbook(UT_USERNAME,
UT_USERNAME,
UT_LBNAME,
description="tester")
bm = BranchManager(lb, username=UT_USERNAME)
bm.create_branch('new-branch-1')
bm.create_branch('new-branch-2')
bm.workon_branch('master')
q = f"""
{{
labbook(name: "{UT_LBNAME}", owner: "{UT_USERNAME}") {{
branches {{
branchName
isLocal
isRemote
isActive
commitsAhead
commitsBehind
}}
}}
}}
"""
r = client.execute(q)
assert 'errors' not in r
assert len(r['data']['labbook']['branches']) == 3
assert r['data']['labbook']['branches'][0][
'branchName'] == 'master'
assert r['data']['labbook']['branches'][0][
'isLocal'] is True, "Should be local"
assert r['data']['labbook']['branches'][0][
'isRemote'] is False, "not published yet"
assert r['data']['labbook']['branches'][0]['isActive'] is True
assert r['data']['labbook']['branches'][0]['commitsAhead'] == 0
assert r['data']['labbook']['branches'][0]['commitsBehind'] == 0
# Make a remote change!
username = '******'
wf = LabbookWorkflow(lb)
wf.publish(username=username)
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
remote_url=wf.remote,
username=other_user,
config_file=lb.client_config.config_file)
with open(os.path.join(wf_other.repository.root_dir, 'testfile'),
'w') as f:
f.write('filedata')
wf_other.repository.sweep_uncommitted_changes()
wf_other.sync(username=other_user)
r = client.execute(q)
assert 'errors' not in r
assert len(r['data']['labbook']['branches']) == 3
assert r['data']['labbook']['branches'][0][
'branchName'] == 'master'
assert r['data']['labbook']['branches'][0][
'isLocal'] is True, "Should be local"
assert r['data']['labbook']['branches'][0][
'isRemote'] is True, "There should be a remote"
assert r['data']['labbook']['branches'][0]['isActive'] is True
assert r['data']['labbook']['branches'][0]['commitsAhead'] == 0
assert r['data']['labbook']['branches'][0]['commitsBehind'] == 1
# Make a local change!
lb.write_readme("blah")
r = client.execute(q)
assert 'errors' not in r
assert len(r['data']['labbook']['branches']) == 3
assert r['data']['labbook']['branches'][0][
'branchName'] == 'master'
assert r['data']['labbook']['branches'][0][
'isLocal'] is True, "Should be local"
assert r['data']['labbook']['branches'][0][
'isRemote'] is True, "There should be a remote"
assert r['data']['labbook']['branches'][0]['isActive'] is True
assert r['data']['labbook']['branches'][0]['commitsAhead'] == 1
assert r['data']['labbook']['branches'][0]['commitsBehind'] == 1
# Sync
wf.sync(username=username)
r = client.execute(q)
assert 'errors' not in r
assert len(r['data']['labbook']['branches']) == 3
assert r['data']['labbook']['branches'][0][
'branchName'] == 'master'
assert r['data']['labbook']['branches'][0][
'isLocal'] is True, "Should be local"
assert r['data']['labbook']['branches'][0][
'isRemote'] is True, "There should be a remote"
assert r['data']['labbook']['branches'][0]['isActive'] is True
assert r['data']['labbook']['branches'][0]['commitsAhead'] == 0
assert r['data']['labbook']['branches'][0]['commitsBehind'] == 0
def test_import_from_remote__linked_dataset(self,
mock_labbook_lfs_disabled,
mock_config_file):
""" test importing a project with a linked dataset"""
def dispatcher_mock(self, function_ref, kwargs, metadata):
assert kwargs['logged_in_username'] == 'other-test-user2'
assert kwargs['dataset_owner'] == 'testuser'
assert kwargs['dataset_name'] == 'test-ds'
# Inject mocked config file
kwargs['config_file'] = mock_config_file[0]
# Stop patching so job gets scheduled for real
dispatcher_patch.stop()
# Call same method as in mutation
d = Dispatcher()
res = d.dispatch_task(
gtmcore.dispatcher.dataset_jobs.check_and_import_dataset,
kwargs=kwargs,
metadata=metadata)
return res
username = '******'
lb = mock_labbook_lfs_disabled[2]
im = InventoryManager(config_file=mock_labbook_lfs_disabled[0])
ds = im.create_dataset(username,
username,
'test-ds',
storage_type='gigantum_object_v1')
# Publish dataset
dataset_wf = DatasetWorkflow(ds)
dataset_wf.publish(username=username)
# Link to project
im.link_dataset_to_labbook(dataset_wf.remote, username, username, lb)
# Publish project
labbook_wf = LabbookWorkflow(lb)
labbook_wf.publish(username=username)
# Patch dispatch_task so you can inject the mocked config file
dispatcher_patch = patch.object(Dispatcher, 'dispatch_task',
dispatcher_mock)
dispatcher_patch.start()
# Import project, triggering an auto-import of the dataset
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
labbook_wf.remote,
username=other_user,
config_file=mock_config_file[0])
# The remotes must be the same, cause it's the same remote repo
assert wf_other.remote == labbook_wf.remote
# The actual path on disk will be different, though
assert wf_other.repository != labbook_wf.repository
# Check imported into namespace of original owner (testuser)
assert f'{other_user}/{username}/labbooks/labbook1' in wf_other.repository.root_dir
cnt = 0
while cnt < 20:
try:
im_other_user = InventoryManager(
config_file=mock_config_file[0])
ds = im_other_user.load_dataset(other_user, username,
'test-ds')
break
except InventoryException:
cnt += 1
time.sleep(1)
assert cnt < 20
assert ds.name == 'test-ds'
assert ds.namespace == username
assert mock_config_file[1] in ds.root_dir
def test_checkout__linked_dataset(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test checking out a branch in a project that pulls in a linked dataset"""
def dispatcher_mock(self, function_ref, kwargs, metadata):
assert kwargs['logged_in_username'] == 'other-test-user2'
assert kwargs['dataset_owner'] == 'testuser'
assert kwargs['dataset_name'] == 'test-ds'
# Inject mocked config file
kwargs['config_file'] = mock_config_file[0]
# Stop patching so job gets scheduled for real
dispatcher_patch.stop()
# Call same method as in mutation
d = Dispatcher()
res = d.dispatch_task(
gtmcore.dispatcher.dataset_jobs.check_and_import_dataset,
kwargs=kwargs,
metadata=metadata)
return res
username = '******'
lb = mock_labbook_lfs_disabled[2]
im = InventoryManager(config_file=mock_labbook_lfs_disabled[0])
ds = im.create_dataset(username,
username,
'test-ds',
storage_type='gigantum_object_v1')
# Publish dataset
dataset_wf = DatasetWorkflow(ds)
dataset_wf.publish(username=username)
# Publish project
labbook_wf = LabbookWorkflow(lb)
labbook_wf.publish(username=username)
# Switch branches
labbook_wf.labbook.checkout_branch(branch_name="dataset-branch",
new=True)
# Link to project
im.link_dataset_to_labbook(dataset_wf.remote, username, username,
labbook_wf.labbook)
# Publish branch
labbook_wf.sync(username=username)
# Import project
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
labbook_wf.remote,
username=other_user,
config_file=mock_config_file[0])
# The remotes must be the same, cause it's the same remote repo
assert wf_other.remote == labbook_wf.remote
assert wf_other.repository != labbook_wf.repository
assert f'{other_user}/{username}/labbooks/labbook1' in wf_other.repository.root_dir
with pytest.raises(InventoryException):
im_other_user = InventoryManager(config_file=mock_config_file[0])
ds = im_other_user.load_dataset(other_user, username, 'test-ds')
# Patch dispatch_task so you can inject the mocked config file
dispatcher_patch = patch.object(Dispatcher, 'dispatch_task',
dispatcher_mock)
dispatcher_patch.start()
# Checkout the branch
assert wf_other.labbook.active_branch == "master"
wf_other.checkout(username=other_user, branch_name="dataset-branch")
cnt = 0
while cnt < 20:
try:
im_other_user = InventoryManager(
config_file=mock_config_file[0])
ds = im_other_user.load_dataset(other_user, username,
'test-ds')
break
except InventoryException:
cnt += 1
time.sleep(1)
assert cnt < 20
assert ds.name == 'test-ds'
assert ds.namespace == username
assert mock_config_file[1] in ds.root_dir
assert wf_other.labbook.active_branch == "dataset-branch"
