def test_publish__publish_then_import_then_sync(self,
mock_labbook_lfs_disabled,
mock_config_file):
""" Test cannot publish a project already published. """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
remote_url=wf.remote,
username=other_user,
config_file=mock_config_file[0])
with open(os.path.join(wf_other.repository.root_dir, 'testfile'),
'w') as f:
f.write('filedata')
wf_other.repository.sweep_uncommitted_changes()
wf_other.sync(username=other_user)
commit_hash = wf_other.repository.git.commit_hash
assert wf.repository.git.commit_hash != commit_hash
wf.sync(username=username)
assert len(wf.repository.git.commit_hash) == len(commit_hash)
assert wf.repository.git.commit_hash == commit_hash
def import_labbook_from_remote(remote_url: str,
username: str,
config_file: str = None) -> str:
"""Return the root directory of the newly imported Project"""
p = os.getpid()
logger = LMLogger.get_logger()
logger.info(
f"(Job {p}) Starting import_labbook_from_remote({remote_url}, {username})"
)
def update_meta(msg):
job = get_current_job()
if not job:
return
if 'feedback' not in job.meta:
job.meta['feedback'] = msg
else:
job.meta['feedback'] = job.meta['feedback'] + f'\n{msg}'
job.save_meta()
try:
toks = remote_url.split("/")
if len(toks) > 1:
proj_path = f'{toks[-2]}/{toks[-1].replace(".git", "")}'
else:
proj_path = remote_url
update_meta(f"Importing Project from {proj_path!r}...")
wf = LabbookWorkflow.import_from_remote(remote_url, username,
config_file)
update_meta(f"Imported Project {wf.labbook.name}!")
return wf.labbook.root_dir
except Exception as e:
update_meta(f"Could not import Project from {remote_url}.")
logger.exception(f"(Job {p}) Error on import_labbook_from_remote: {e}")
raise
def test_sync___override_merge_conflict_ours(self,
mock_labbook_lfs_disabled,
mock_config_file):
""" test sync, with override in case of merge conflict. """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
bm = BranchManager(lb, username='******')
bm.create_branch('test-conflict-branch')
fpath = os.path.join(lb.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('filedata')
lb.sweep_uncommitted_changes()
wf.sync('test')
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
wf.remote, username=other_user, config_file=mock_config_file[0])
bm_other = BranchManager(wf_other.labbook, username=other_user)
bm_other.workon_branch('test-conflict-branch')
with open(os.path.join(wf_other.labbook.root_dir, 'input', 'testfile'),
'w') as f:
f.write('conflicting-change-other-user')
wf_other.labbook.sweep_uncommitted_changes()
wf_other.sync(username=username)
fpath = os.path.join(wf.labbook.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('conflicting-change-original-user')
wf.labbook.sweep_uncommitted_changes()
n = wf.sync(username=username, override=MergeOverride.OURS)
flines = open(os.path.join(wf.labbook.root_dir, 'input',
'testfile')).read()
assert 'conflicting-change-original-user' == flines
def test_sync_1(self, mock_create_labbooks_no_lfs, mock_config_file):
# Setup responses mock for this test
responses.add(responses.GET, 'https://usersrv.gigantum.io/key',
json={'key': 'afaketoken'}, status=200)
im = InventoryManager(mock_create_labbooks_no_lfs[0])
test_user_lb = im.load_labbook('default', 'default', 'labbook1')
test_user_wf = LabbookWorkflow(test_user_lb)
test_user_wf.publish('default')
# Mock the request context so a fake authorization header is present
builder = EnvironBuilder(path='/labbook', method='POST', headers={'Authorization': 'Bearer AJDFHASD'})
env = builder.get_environ()
req = Request(environ=env)
sally_wf = LabbookWorkflow.import_from_remote(test_user_wf.remote, 'sally', config_file=mock_config_file[0])
sally_lb = sally_wf.labbook
FileOperations.makedir(sally_lb, relative_path='code/sally-dir', create_activity_record=True)
sally_wf.sync('sally')
sync_query = """
mutation x {
syncLabbook(input: {
labbookName: "labbook1",
owner: "default"
}) {
jobKey
}
}
"""
r = mock_create_labbooks_no_lfs[2].execute(sync_query, context_value=req)
assert 'errors' not in r
def test_sync___detect_merge_conflict(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test import_from_remote method """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
bm = BranchManager(lb, username='******')
bm.create_branch('test-conflict-branch')
fpath = os.path.join(lb.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('filedata')
lb.sweep_uncommitted_changes()
wf.sync('test')
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
wf.remote, username=other_user, config_file=mock_config_file[0])
bm_other = BranchManager(wf_other.labbook, username=other_user)
bm_other.workon_branch('test-conflict-branch')
with open(os.path.join(wf_other.labbook.root_dir, 'input', 'testfile'),
'w') as f:
f.write('conflicting-change-other-user')
wf_other.labbook.sweep_uncommitted_changes()
wf_other.sync(username=username)
with open(fpath, 'w') as f:
f.write('conflicting-change-original-user')
wf.labbook.sweep_uncommitted_changes()
h = wf.labbook.git.commit_hash
with pytest.raises(MergeConflict):
n = wf.sync(username=username)
assert h == wf.labbook.git.commit_hash
def test_publish__publish_then_import_with_another_user(
self, mock_labbook_lfs_disabled, mock_config_file):
""" Test cannot publish a project already published. """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
remote_url=wf.remote,
username=other_user,
config_file=mock_config_file[0])
lb_other = wf_other.repository
assert lb_other.root_dir != lb.root_dir
def test_import_from_remote__nominal(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test import_from_remote method """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
wf.remote, username=other_user, config_file=mock_config_file[0])
# The remotes must be the same, cause it's the same remote repo
assert wf_other.remote == wf.remote
# The actual path on disk will be different, though
assert wf_other.repository != wf.repository
# Check imported into namespace of original owner (testuser)
assert f'{other_user}/{username}/labbooks/labbook1' in wf_other.repository.root_dir
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
remote_url,
client_mutation_id=None):
username = get_logged_in_username()
logger.info(f"Importing remote labbook from {remote_url}")
lb = LabBook(author=get_logged_in_author())
default_remote = lb.client_config.config['git']['default_remote']
admin_service = None
for remote in lb.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = lb.client_config.config['git']['remotes'][
remote]['admin_service']
break
# Extract valid Bearer token
if hasattr(info.context, 'headers'
) and "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
else:
raise ValueError(
"Authorization header not provided. Must have a valid session to query for collaborators"
)
gl_mgr = GitLabManager(default_remote,
admin_service=admin_service,
access_token=token)
gl_mgr.configure_git_credentials(default_remote, username)
wf = LabbookWorkflow.import_from_remote(remote_url, username=username)
import_owner = InventoryManager().query_owner(wf.labbook)
# TODO: Fix cursor implementation, this currently doesn't make sense
cursor = base64.b64encode(f"{0}".encode('utf-8'))
lbedge = LabbookConnection.Edge(node=Labbook(owner=import_owner,
name=wf.labbook.name),
cursor=cursor)
return ImportRemoteLabbook(new_labbook_edge=lbedge)
def test_checkout__linked_dataset(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test checking out a branch in a project that pulls in a linked dataset"""
def dispatcher_mock(self, function_ref, kwargs, metadata):
assert kwargs['logged_in_username'] == 'other-test-user2'
assert kwargs['dataset_owner'] == 'testuser'
assert kwargs['dataset_name'] == 'test-ds'
# Inject mocked config file
kwargs['config_file'] = mock_config_file[0]
# Stop patching so job gets scheduled for real
dispatcher_patch.stop()
# Call same method as in mutation
d = Dispatcher()
res = d.dispatch_task(
gtmcore.dispatcher.dataset_jobs.check_and_import_dataset,
kwargs=kwargs,
metadata=metadata)
return res
username = '******'
lb = mock_labbook_lfs_disabled[2]
im = InventoryManager(config_file=mock_labbook_lfs_disabled[0])
ds = im.create_dataset(username,
username,
'test-ds',
storage_type='gigantum_object_v1')
# Publish dataset
dataset_wf = DatasetWorkflow(ds)
dataset_wf.publish(username=username)
# Publish project
labbook_wf = LabbookWorkflow(lb)
labbook_wf.publish(username=username)
# Switch branches
labbook_wf.labbook.checkout_branch(branch_name="dataset-branch",
new=True)
# Link to project
im.link_dataset_to_labbook(dataset_wf.remote, username, username,
labbook_wf.labbook)
# Publish branch
labbook_wf.sync(username=username)
# Import project
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
labbook_wf.remote,
username=other_user,
config_file=mock_config_file[0])
# The remotes must be the same, cause it's the same remote repo
assert wf_other.remote == labbook_wf.remote
assert wf_other.repository != labbook_wf.repository
assert f'{other_user}/{username}/labbooks/labbook1' in wf_other.repository.root_dir
with pytest.raises(InventoryException):
im_other_user = InventoryManager(config_file=mock_config_file[0])
ds = im_other_user.load_dataset(other_user, username, 'test-ds')
# Patch dispatch_task so you can inject the mocked config file
dispatcher_patch = patch.object(Dispatcher, 'dispatch_task',
dispatcher_mock)
dispatcher_patch.start()
# Checkout the branch
assert wf_other.labbook.active_branch == "master"
wf_other.checkout(username=other_user, branch_name="dataset-branch")
cnt = 0
while cnt < 20:
try:
im_other_user = InventoryManager(
config_file=mock_config_file[0])
ds = im_other_user.load_dataset(other_user, username,
'test-ds')
break
except InventoryException:
cnt += 1
time.sleep(1)
assert cnt < 20
assert ds.name == 'test-ds'
assert ds.namespace == username
assert mock_config_file[1] in ds.root_dir
assert wf_other.labbook.active_branch == "dataset-branch"
def test_import_from_remote__linked_dataset(self,
mock_labbook_lfs_disabled,
mock_config_file):
""" test importing a project with a linked dataset"""
def dispatcher_mock(self, function_ref, kwargs, metadata):
assert kwargs['logged_in_username'] == 'other-test-user2'
assert kwargs['dataset_owner'] == 'testuser'
assert kwargs['dataset_name'] == 'test-ds'
# Inject mocked config file
kwargs['config_file'] = mock_config_file[0]
# Stop patching so job gets scheduled for real
dispatcher_patch.stop()
# Call same method as in mutation
d = Dispatcher()
res = d.dispatch_task(
gtmcore.dispatcher.dataset_jobs.check_and_import_dataset,
kwargs=kwargs,
metadata=metadata)
return res
username = '******'
lb = mock_labbook_lfs_disabled[2]
im = InventoryManager(config_file=mock_labbook_lfs_disabled[0])
ds = im.create_dataset(username,
username,
'test-ds',
storage_type='gigantum_object_v1')
# Publish dataset
dataset_wf = DatasetWorkflow(ds)
dataset_wf.publish(username=username)
# Link to project
im.link_dataset_to_labbook(dataset_wf.remote, username, username, lb)
# Publish project
labbook_wf = LabbookWorkflow(lb)
labbook_wf.publish(username=username)
# Patch dispatch_task so you can inject the mocked config file
dispatcher_patch = patch.object(Dispatcher, 'dispatch_task',
dispatcher_mock)
dispatcher_patch.start()
# Import project, triggering an auto-import of the dataset
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
labbook_wf.remote,
username=other_user,
config_file=mock_config_file[0])
# The remotes must be the same, cause it's the same remote repo
assert wf_other.remote == labbook_wf.remote
# The actual path on disk will be different, though
assert wf_other.repository != labbook_wf.repository
# Check imported into namespace of original owner (testuser)
assert f'{other_user}/{username}/labbooks/labbook1' in wf_other.repository.root_dir
cnt = 0
while cnt < 20:
try:
im_other_user = InventoryManager(
config_file=mock_config_file[0])
ds = im_other_user.load_dataset(other_user, username,
'test-ds')
break
except InventoryException:
cnt += 1
time.sleep(1)
assert cnt < 20
assert ds.name == 'test-ds'
assert ds.namespace == username
assert mock_config_file[1] in ds.root_dir
def test_commits_ahead_behind(self, fixture_working_dir_lfs_disabled):
with responses.RequestsMock() as rsps:
rsps.add(responses.GET,
'https://usersrv.gigantum.io/key',
json={'key': 'afaketoken'},
status=200)
config_file, client = fixture_working_dir_lfs_disabled[0], \
fixture_working_dir_lfs_disabled[2]
im = InventoryManager(config_file)
lb = im.create_labbook(UT_USERNAME,
UT_USERNAME,
UT_LBNAME,
description="tester")
bm = BranchManager(lb, username=UT_USERNAME)
bm.create_branch('new-branch-1')
bm.create_branch('new-branch-2')
bm.workon_branch('master')
q = f"""
{{
labbook(name: "{UT_LBNAME}", owner: "{UT_USERNAME}") {{
branches {{
branchName
isLocal
isRemote
isActive
commitsAhead
commitsBehind
}}
}}
}}
"""
r = client.execute(q)
assert 'errors' not in r
assert len(r['data']['labbook']['branches']) == 3
assert r['data']['labbook']['branches'][0][
'branchName'] == 'master'
assert r['data']['labbook']['branches'][0][
'isLocal'] is True, "Should be local"
assert r['data']['labbook']['branches'][0][
'isRemote'] is False, "not published yet"
assert r['data']['labbook']['branches'][0]['isActive'] is True
assert r['data']['labbook']['branches'][0]['commitsAhead'] == 0
assert r['data']['labbook']['branches'][0]['commitsBehind'] == 0
# Make a remote change!
username = '******'
wf = LabbookWorkflow(lb)
wf.publish(username=username)
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
remote_url=wf.remote,
username=other_user,
config_file=lb.client_config.config_file)
with open(os.path.join(wf_other.repository.root_dir, 'testfile'),
'w') as f:
f.write('filedata')
wf_other.repository.sweep_uncommitted_changes()
wf_other.sync(username=other_user)
r = client.execute(q)
assert 'errors' not in r
assert len(r['data']['labbook']['branches']) == 3
assert r['data']['labbook']['branches'][0][
'branchName'] == 'master'
assert r['data']['labbook']['branches'][0][
'isLocal'] is True, "Should be local"
assert r['data']['labbook']['branches'][0][
'isRemote'] is True, "There should be a remote"
assert r['data']['labbook']['branches'][0]['isActive'] is True
assert r['data']['labbook']['branches'][0]['commitsAhead'] == 0
assert r['data']['labbook']['branches'][0]['commitsBehind'] == 1
# Make a local change!
lb.write_readme("blah")
r = client.execute(q)
assert 'errors' not in r
assert len(r['data']['labbook']['branches']) == 3
assert r['data']['labbook']['branches'][0][
'branchName'] == 'master'
assert r['data']['labbook']['branches'][0][
'isLocal'] is True, "Should be local"
assert r['data']['labbook']['branches'][0][
'isRemote'] is True, "There should be a remote"
assert r['data']['labbook']['branches'][0]['isActive'] is True
assert r['data']['labbook']['branches'][0]['commitsAhead'] == 1
assert r['data']['labbook']['branches'][0]['commitsBehind'] == 1
# Sync
wf.sync(username=username)
r = client.execute(q)
assert 'errors' not in r
assert len(r['data']['labbook']['branches']) == 3
assert r['data']['labbook']['branches'][0][
'branchName'] == 'master'
assert r['data']['labbook']['branches'][0][
'isLocal'] is True, "Should be local"
assert r['data']['labbook']['branches'][0][
'isRemote'] is True, "There should be a remote"
assert r['data']['labbook']['branches'][0]['isActive'] is True
assert r['data']['labbook']['branches'][0]['commitsAhead'] == 0
assert r['data']['labbook']['branches'][0]['commitsBehind'] == 0
