def test_sync___override_merge_conflict_ours(self,
mock_labbook_lfs_disabled,
mock_config_file):
""" test sync, with override in case of merge conflict. """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
bm = BranchManager(lb, username='******')
bm.create_branch('test-conflict-branch')
fpath = os.path.join(lb.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('filedata')
lb.sweep_uncommitted_changes()
wf.sync('test')
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
wf.remote, username=other_user, config_file=mock_config_file[0])
bm_other = BranchManager(wf_other.labbook, username=other_user)
bm_other.workon_branch('test-conflict-branch')
with open(os.path.join(wf_other.labbook.root_dir, 'input', 'testfile'),
'w') as f:
f.write('conflicting-change-other-user')
wf_other.labbook.sweep_uncommitted_changes()
wf_other.sync(username=username)
fpath = os.path.join(wf.labbook.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('conflicting-change-original-user')
wf.labbook.sweep_uncommitted_changes()
n = wf.sync(username=username, override=MergeOverride.OURS)
flines = open(os.path.join(wf.labbook.root_dir, 'input',
'testfile')).read()
assert 'conflicting-change-original-user' == flines
def test_publish__publish_then_import_then_sync(self,
mock_labbook_lfs_disabled,
mock_config_file):
""" Test cannot publish a project already published. """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
remote_url=wf.remote,
username=other_user,
config_file=mock_config_file[0])
with open(os.path.join(wf_other.repository.root_dir, 'testfile'),
'w') as f:
f.write('filedata')
wf_other.repository.sweep_uncommitted_changes()
wf_other.sync(username=other_user)
commit_hash = wf_other.repository.git.commit_hash
assert wf.repository.git.commit_hash != commit_hash
wf.sync(username=username)
assert len(wf.repository.git.commit_hash) == len(commit_hash)
assert wf.repository.git.commit_hash == commit_hash
def test_sync___detect_merge_conflict(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test import_from_remote method """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
bm = BranchManager(lb, username='******')
bm.create_branch('test-conflict-branch')
fpath = os.path.join(lb.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('filedata')
lb.sweep_uncommitted_changes()
wf.sync('test')
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
wf.remote, username=other_user, config_file=mock_config_file[0])
bm_other = BranchManager(wf_other.labbook, username=other_user)
bm_other.workon_branch('test-conflict-branch')
with open(os.path.join(wf_other.labbook.root_dir, 'input', 'testfile'),
'w') as f:
f.write('conflicting-change-other-user')
wf_other.labbook.sweep_uncommitted_changes()
wf_other.sync(username=username)
with open(fpath, 'w') as f:
f.write('conflicting-change-original-user')
wf.labbook.sweep_uncommitted_changes()
h = wf.labbook.git.commit_hash
with pytest.raises(MergeConflict):
n = wf.sync(username=username)
assert h == wf.labbook.git.commit_hash
def test_sync___push_up_new_branch(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test import_from_remote method """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
bm = BranchManager(lb, username='******')
bm.create_branch('new-branch-to-push')
assert 'new-branch-to-push' not in bm.branches_remote
wf.sync('test')
assert 'new-branch-to-push' in bm.branches_remote
def sync_repository(repository: Repository, username: str, override: MergeOverride,
remote: str = "origin", access_token: str = None,
pull_only: bool = False, id_token: str = None) -> int:
p = os.getpid()
logger = LMLogger.get_logger()
logger.info(f"(Job {p}) Starting sync_repository({str(repository)})")
def update_meta(msg):
job = get_current_job()
if not job:
return
if 'feedback' not in job.meta:
job.meta['feedback'] = msg
else:
job.meta['feedback'] = job.meta['feedback'] + f'\n{msg}'
job.save_meta()
try:
with repository.lock():
if isinstance(repository, LabBook):
wf = LabbookWorkflow(repository)
else:
wf = DatasetWorkflow(repository) # type: ignore
cnt = wf.sync(username=username, remote=remote, override=override,
feedback_callback=update_meta, access_token=access_token,
id_token=id_token, pull_only=pull_only)
logger.info(f"(Job {p} Completed sync_repository with cnt={cnt}")
return cnt
except Exception as e:
logger.exception(f"(Job {p}) Error on sync_repository: {e}")
raise
def test_sync___push_up_then_sync(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test import_from_remote method """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
bm = BranchManager(lb, username='******')
bm.create_branch('new-branch-to-push')
wf.sync('test')
# Make some change locally and commit, then sync.
fpath = os.path.join(lb.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('filedata')
lb.sweep_uncommitted_changes()
wf.sync(username=username)
def test_sync___simple_push_to_master(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test import_from_remote method """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
fpath = os.path.join(lb.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('filedata')
lb.sweep_uncommitted_changes()
wf.sync(username=username)
# Check hash on remote - make sure it matches local.
remote_hash = call_subprocess('git log -n 1 --oneline'.split(),
cwd=wf.remote).split()[0]
assert remote_hash in lb.git.commit_hash
def sync_repository(repository: Repository,
username: str,
override: MergeOverride,
remote: str = "origin",
access_token: str = None,
pull_only: bool = False,
id_token: str = None) -> int:
p = os.getpid()
logger = LMLogger.get_logger()
logger.info(f"(Job {p}) Starting sync_repository({str(repository)})")
def update_feedback(msg: str,
has_failures: Optional[bool] = None,
failure_detail: Optional[str] = None,
percent_complete: Optional[float] = None):
"""Method to update the job's metadata and provide feedback to the UI"""
current_job = get_current_job()
if not current_job:
return
if has_failures:
current_job.meta['has_failures'] = has_failures
if failure_detail:
current_job.meta['failure_detail'] = failure_detail
if percent_complete:
current_job.meta['percent_complete'] = percent_complete
current_job.meta['feedback'] = msg
current_job.save_meta()
try:
update_feedback("Sync task in queue")
with repository.lock():
if isinstance(repository, LabBook):
wf = LabbookWorkflow(repository)
else:
wf = DatasetWorkflow(repository) # type: ignore
cnt = wf.sync(username=username,
remote=remote,
override=override,
feedback_callback=update_feedback,
access_token=access_token,
id_token=id_token,
pull_only=pull_only)
logger.info(f"(Job {p} Completed sync_repository with cnt={cnt}")
return cnt
except MergeConflict as me:
logger.exception(f"(Job {p}) Merge conflict: {me}")
raise
except IOError:
raise
except Exception as e:
logger.exception(e)
raise Exception("Could not sync - try to log out and log in again.")
def test_checkout__linked_dataset(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test checking out a branch in a project that pulls in a linked dataset"""
def dispatcher_mock(self, function_ref, kwargs, metadata):
assert kwargs['logged_in_username'] == 'other-test-user2'
assert kwargs['dataset_owner'] == 'testuser'
assert kwargs['dataset_name'] == 'test-ds'
# Inject mocked config file
kwargs['config_file'] = mock_config_file[0]
# Stop patching so job gets scheduled for real
dispatcher_patch.stop()
# Call same method as in mutation
d = Dispatcher()
res = d.dispatch_task(
gtmcore.dispatcher.dataset_jobs.check_and_import_dataset,
kwargs=kwargs,
metadata=metadata)
return res
username = '******'
lb = mock_labbook_lfs_disabled[2]
im = InventoryManager(config_file=mock_labbook_lfs_disabled[0])
ds = im.create_dataset(username,
username,
'test-ds',
storage_type='gigantum_object_v1')
# Publish dataset
dataset_wf = DatasetWorkflow(ds)
dataset_wf.publish(username=username)
# Publish project
labbook_wf = LabbookWorkflow(lb)
labbook_wf.publish(username=username)
# Switch branches
labbook_wf.labbook.checkout_branch(branch_name="dataset-branch",
new=True)
# Link to project
im.link_dataset_to_labbook(dataset_wf.remote, username, username,
labbook_wf.labbook)
# Publish branch
labbook_wf.sync(username=username)
# Import project
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
labbook_wf.remote,
username=other_user,
config_file=mock_config_file[0])
# The remotes must be the same, cause it's the same remote repo
assert wf_other.remote == labbook_wf.remote
assert wf_other.repository != labbook_wf.repository
assert f'{other_user}/{username}/labbooks/labbook1' in wf_other.repository.root_dir
with pytest.raises(InventoryException):
im_other_user = InventoryManager(config_file=mock_config_file[0])
ds = im_other_user.load_dataset(other_user, username, 'test-ds')
# Patch dispatch_task so you can inject the mocked config file
dispatcher_patch = patch.object(Dispatcher, 'dispatch_task',
dispatcher_mock)
dispatcher_patch.start()
# Checkout the branch
assert wf_other.labbook.active_branch == "master"
wf_other.checkout(username=other_user, branch_name="dataset-branch")
cnt = 0
while cnt < 20:
try:
im_other_user = InventoryManager(
config_file=mock_config_file[0])
ds = im_other_user.load_dataset(other_user, username,
'test-ds')
break
except InventoryException:
cnt += 1
time.sleep(1)
assert cnt < 20
assert ds.name == 'test-ds'
assert ds.namespace == username
assert mock_config_file[1] in ds.root_dir
assert wf_other.labbook.active_branch == "dataset-branch"
def test_commits_ahead_behind(self, fixture_working_dir_lfs_disabled):
with responses.RequestsMock() as rsps:
rsps.add(responses.GET,
'https://usersrv.gigantum.io/key',
json={'key': 'afaketoken'},
status=200)
config_file, client = fixture_working_dir_lfs_disabled[0], \
fixture_working_dir_lfs_disabled[2]
im = InventoryManager(config_file)
lb = im.create_labbook(UT_USERNAME,
UT_USERNAME,
UT_LBNAME,
description="tester")
bm = BranchManager(lb, username=UT_USERNAME)
bm.create_branch('new-branch-1')
bm.create_branch('new-branch-2')
bm.workon_branch('master')
q = f"""
{{
labbook(name: "{UT_LBNAME}", owner: "{UT_USERNAME}") {{
branches {{
branchName
isLocal
isRemote
isActive
commitsAhead
commitsBehind
}}
}}
}}
"""
r = client.execute(q)
assert 'errors' not in r
assert len(r['data']['labbook']['branches']) == 3
assert r['data']['labbook']['branches'][0][
'branchName'] == 'master'
assert r['data']['labbook']['branches'][0][
'isLocal'] is True, "Should be local"
assert r['data']['labbook']['branches'][0][
'isRemote'] is False, "not published yet"
assert r['data']['labbook']['branches'][0]['isActive'] is True
assert r['data']['labbook']['branches'][0]['commitsAhead'] == 0
assert r['data']['labbook']['branches'][0]['commitsBehind'] == 0
# Make a remote change!
username = '******'
wf = LabbookWorkflow(lb)
wf.publish(username=username)
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
remote_url=wf.remote,
username=other_user,
config_file=lb.client_config.config_file)
with open(os.path.join(wf_other.repository.root_dir, 'testfile'),
'w') as f:
f.write('filedata')
wf_other.repository.sweep_uncommitted_changes()
wf_other.sync(username=other_user)
r = client.execute(q)
assert 'errors' not in r
assert len(r['data']['labbook']['branches']) == 3
assert r['data']['labbook']['branches'][0][
'branchName'] == 'master'
assert r['data']['labbook']['branches'][0][
'isLocal'] is True, "Should be local"
assert r['data']['labbook']['branches'][0][
'isRemote'] is True, "There should be a remote"
assert r['data']['labbook']['branches'][0]['isActive'] is True
assert r['data']['labbook']['branches'][0]['commitsAhead'] == 0
assert r['data']['labbook']['branches'][0]['commitsBehind'] == 1
# Make a local change!
lb.write_readme("blah")
r = client.execute(q)
assert 'errors' not in r
assert len(r['data']['labbook']['branches']) == 3
assert r['data']['labbook']['branches'][0][
'branchName'] == 'master'
assert r['data']['labbook']['branches'][0][
'isLocal'] is True, "Should be local"
assert r['data']['labbook']['branches'][0][
'isRemote'] is True, "There should be a remote"
assert r['data']['labbook']['branches'][0]['isActive'] is True
assert r['data']['labbook']['branches'][0]['commitsAhead'] == 1
assert r['data']['labbook']['branches'][0]['commitsBehind'] == 1
# Sync
wf.sync(username=username)
r = client.execute(q)
assert 'errors' not in r
assert len(r['data']['labbook']['branches']) == 3
assert r['data']['labbook']['branches'][0][
'branchName'] == 'master'
assert r['data']['labbook']['branches'][0][
'isLocal'] is True, "Should be local"
assert r['data']['labbook']['branches'][0][
'isRemote'] is True, "There should be a remote"
assert r['data']['labbook']['branches'][0]['isActive'] is True
assert r['data']['labbook']['branches'][0]['commitsAhead'] == 0
assert r['data']['labbook']['branches'][0]['commitsBehind'] == 0