def test_reset_branch_to_remote(self, fixture_working_dir, mock_create_labbooks_no_lfs):
# Mock the request context so a fake authorization header is present
builder = EnvironBuilder(path='/labbook', method='POST', headers={'Authorization': 'Bearer AJDFHASD'})
env = builder.get_environ()
req = Request(environ=env)
im = InventoryManager(mock_create_labbooks_no_lfs[0])
test_user_lb = im.load_labbook('default', 'default', 'labbook1')
wf = LabbookWorkflow(test_user_lb)
wf.publish(username='******')
hash_original = wf.labbook.git.commit_hash
new_file_path = os.path.join(wf.labbook.root_dir, 'input', 'new-file')
with open(new_file_path, 'w') as f: f.write('File data')
wf.labbook.sweep_uncommitted_changes()
hash_before_reset = wf.labbook.git.commit_hash
publish_query = f"""
mutation c {{
resetBranchToRemote(input: {{
labbookName: "labbook1",
owner: "default"
}}) {{
labbook {{
activeBranchName
}}
}}
}}
"""
r = mock_create_labbooks_no_lfs[2].execute(publish_query, context_value=req)
assert 'errors' not in r
assert wf.labbook.git.commit_hash == hash_original
def test_sync_1(self, mock_create_labbooks_no_lfs, mock_config_file):
# Setup responses mock for this test
responses.add(responses.GET, 'https://usersrv.gigantum.io/key',
json={'key': 'afaketoken'}, status=200)
im = InventoryManager(mock_create_labbooks_no_lfs[0])
test_user_lb = im.load_labbook('default', 'default', 'labbook1')
test_user_wf = LabbookWorkflow(test_user_lb)
test_user_wf.publish('default')
# Mock the request context so a fake authorization header is present
builder = EnvironBuilder(path='/labbook', method='POST', headers={'Authorization': 'Bearer AJDFHASD'})
env = builder.get_environ()
req = Request(environ=env)
sally_wf = LabbookWorkflow.import_from_remote(test_user_wf.remote, 'sally', config_file=mock_config_file[0])
sally_lb = sally_wf.labbook
FileOperations.makedir(sally_lb, relative_path='code/sally-dir', create_activity_record=True)
sally_wf.sync('sally')
sync_query = """
mutation x {
syncLabbook(input: {
labbookName: "labbook1",
owner: "default"
}) {
jobKey
}
}
"""
r = mock_create_labbooks_no_lfs[2].execute(sync_query, context_value=req)
assert 'errors' not in r
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
migrated = False
with lb.lock():
t0 = time.time()
workflow = LabbookWorkflow(lb)
migrated = workflow.migrate()
tf = time.time()
if migrated:
logger.info(f"Migrated {str(lb)} in {tf-t0:.2}sec")
else:
logger.info(f"No migration needed for {str(lb)}")
return MigrateLabbookSchema(
Labbook(id=f"{owner}&{labbook_name}",
name=labbook_name,
owner=owner))
def publish_repository(repository: Repository, username: str, access_token: str,
remote: Optional[str] = None, public: bool = False, id_token: str = None) -> None:
p = os.getpid()
logger = LMLogger.get_logger()
logger.info(f"(Job {p}) Starting publish_repository({str(repository)})")
def update_meta(msg):
job = get_current_job()
if not job:
return
if 'feedback' not in job.meta:
job.meta['feedback'] = msg
else:
job.meta['feedback'] = job.meta['feedback'] + f'\n{msg}'
job.save_meta()
try:
with repository.lock():
if isinstance(repository, LabBook):
wf = LabbookWorkflow(repository)
else:
wf = DatasetWorkflow(repository) # type: ignore
wf.publish(username=username, access_token=access_token, remote=remote or "origin",
public=public, feedback_callback=update_meta, id_token=id_token)
except Exception as e:
logger.exception(f"(Job {p}) Error on publish_repository: {e}")
raise
def sync_repository(repository: Repository, username: str, override: MergeOverride,
remote: str = "origin", access_token: str = None,
pull_only: bool = False, id_token: str = None) -> int:
p = os.getpid()
logger = LMLogger.get_logger()
logger.info(f"(Job {p}) Starting sync_repository({str(repository)})")
def update_meta(msg):
job = get_current_job()
if not job:
return
if 'feedback' not in job.meta:
job.meta['feedback'] = msg
else:
job.meta['feedback'] = job.meta['feedback'] + f'\n{msg}'
job.save_meta()
try:
with repository.lock():
if isinstance(repository, LabBook):
wf = LabbookWorkflow(repository)
else:
wf = DatasetWorkflow(repository) # type: ignore
cnt = wf.sync(username=username, remote=remote, override=override,
feedback_callback=update_meta, access_token=access_token,
id_token=id_token, pull_only=pull_only)
logger.info(f"(Job {p} Completed sync_repository with cnt={cnt}")
return cnt
except Exception as e:
logger.exception(f"(Job {p}) Error on sync_repository: {e}")
raise
def mutate_and_get_payload(cls, root, info, owner, labbook_name, branch_name, client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with lb.lock():
wf = LabbookWorkflow(lb)
wf.checkout(username, branch_name)
return WorkonBranch(Labbook(id="{}&{}".format(owner, labbook_name),
name=labbook_name, owner=owner))
def test_migrate_no_op_on_new_labbook(self, mock_labbook_lfs_disabled):
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
h1 = wf.labbook.git.commit_hash
wf.migrate()
# No change of git status after no-op migration
assert h1 == wf.labbook.git.commit_hash
def _stop_container(cls, lb, username):
"""Stop container and also do necessary cleanup of confhttpproxy, monitors, etc.
Currently, this supports two cases, applications monitored by MITMProxy,
and Jupyter. So, for now, if we can't find an mitmproxy endpoint, we assume
we're dealing with a jupyter container.
"""
pr = confhttpproxy.ProxyRouter.get_proxy(
lb.client_config.config['proxy'])
# Remove route from proxy
lb_name = ContainerOperations.labbook_image_name(lb, username)
if MITMProxyOperations.get_mitmendpoint(lb_name):
# there is an MITMProxy (currently only used for RStudio)
proxy_endpoint = MITMProxyOperations.stop_mitm_proxy(lb_name)
tool = 'rserver'
else:
lb_ip = ContainerOperations.get_labbook_ip(lb, username)
# The only alternative to mitmproxy (currently) is jupyter
# TODO in #453: Construction of this URL should be encapsulated in Jupyter Dev Tool logic
proxy_endpoint = f'http://{lb_ip}:8888'
tool = 'jupyter'
est_target = pr.get_matching_routes(proxy_endpoint, tool)
for i, target in enumerate(est_target):
if i == 1:
# We have > 1 entry in the router, which shouldn't happen
logger.warning(
f'Removing multiple routes for {tool} on {proxy_endpoint} during Project container stop.'
)
pr.remove(target[1:])
wf = LabbookWorkflow(lb)
wf.garbagecollect()
# Clean up empty bind mount dirs from datasets if needed
submodules = lb.git.list_submodules()
for submodule in submodules:
namespace, dataset_name = submodule['name'].split("&")
bind_location = os.path.join(lb.root_dir, 'input', dataset_name)
if os.path.isdir(bind_location):
os.rmdir(bind_location)
# stop labbook monitor
stop_labbook_monitor(lb, username)
lb, stopped = ContainerOperations.stop_container(labbook=lb,
username=username)
if not stopped:
# TODO DK: Why would stopped=False? Should this move up??
raise ValueError(f"Failed to stop labbook {lb.name}")
def sync_repository(repository: Repository,
username: str,
override: MergeOverride,
remote: str = "origin",
access_token: str = None,
pull_only: bool = False,
id_token: str = None) -> int:
p = os.getpid()
logger = LMLogger.get_logger()
logger.info(f"(Job {p}) Starting sync_repository({str(repository)})")
def update_feedback(msg: str,
has_failures: Optional[bool] = None,
failure_detail: Optional[str] = None,
percent_complete: Optional[float] = None):
"""Method to update the job's metadata and provide feedback to the UI"""
current_job = get_current_job()
if not current_job:
return
if has_failures:
current_job.meta['has_failures'] = has_failures
if failure_detail:
current_job.meta['failure_detail'] = failure_detail
if percent_complete:
current_job.meta['percent_complete'] = percent_complete
current_job.meta['feedback'] = msg
current_job.save_meta()
try:
update_feedback("Sync task in queue")
with repository.lock():
if isinstance(repository, LabBook):
wf = LabbookWorkflow(repository)
else:
wf = DatasetWorkflow(repository) # type: ignore
cnt = wf.sync(username=username,
remote=remote,
override=override,
feedback_callback=update_feedback,
access_token=access_token,
id_token=id_token,
pull_only=pull_only)
logger.info(f"(Job {p} Completed sync_repository with cnt={cnt}")
return cnt
except MergeConflict as me:
logger.exception(f"(Job {p}) Merge conflict: {me}")
raise
except IOError:
raise
except Exception as e:
logger.exception(e)
raise Exception("Could not sync - try to log out and log in again.")
def test_sync___override_merge_conflict_ours(self,
mock_labbook_lfs_disabled,
mock_config_file):
""" test sync, with override in case of merge conflict. """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
bm = BranchManager(lb, username='******')
bm.create_branch('test-conflict-branch')
fpath = os.path.join(lb.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('filedata')
lb.sweep_uncommitted_changes()
wf.sync('test')
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
wf.remote, username=other_user, config_file=mock_config_file[0])
bm_other = BranchManager(wf_other.labbook, username=other_user)
bm_other.workon_branch('test-conflict-branch')
with open(os.path.join(wf_other.labbook.root_dir, 'input', 'testfile'),
'w') as f:
f.write('conflicting-change-other-user')
wf_other.labbook.sweep_uncommitted_changes()
wf_other.sync(username=username)
fpath = os.path.join(wf.labbook.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('conflicting-change-original-user')
wf.labbook.sweep_uncommitted_changes()
n = wf.sync(username=username, override=MergeOverride.OURS)
flines = open(os.path.join(wf.labbook.root_dir, 'input',
'testfile')).read()
assert 'conflicting-change-original-user' == flines
def test_sync___detect_merge_conflict(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test import_from_remote method """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
bm = BranchManager(lb, username='******')
bm.create_branch('test-conflict-branch')
fpath = os.path.join(lb.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('filedata')
lb.sweep_uncommitted_changes()
wf.sync('test')
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
wf.remote, username=other_user, config_file=mock_config_file[0])
bm_other = BranchManager(wf_other.labbook, username=other_user)
bm_other.workon_branch('test-conflict-branch')
with open(os.path.join(wf_other.labbook.root_dir, 'input', 'testfile'),
'w') as f:
f.write('conflicting-change-other-user')
wf_other.labbook.sweep_uncommitted_changes()
wf_other.sync(username=username)
with open(fpath, 'w') as f:
f.write('conflicting-change-original-user')
wf.labbook.sweep_uncommitted_changes()
h = wf.labbook.git.commit_hash
with pytest.raises(MergeConflict):
n = wf.sync(username=username)
assert h == wf.labbook.git.commit_hash
def test_publish__publish_then_import_with_another_user(
self, mock_labbook_lfs_disabled, mock_config_file):
""" Test cannot publish a project already published. """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
remote_url=wf.remote,
username=other_user,
config_file=mock_config_file[0])
lb_other = wf_other.repository
assert lb_other.root_dir != lb.root_dir
def publish_repository(repository: Repository,
username: str,
access_token: str,
remote: Optional[str] = None,
public: bool = False,
id_token: str = None) -> None:
p = os.getpid()
logger = LMLogger.get_logger()
logger.info(f"(Job {p}) Starting publish_repository({str(repository)})")
def update_feedback(msg: str,
has_failures: Optional[bool] = None,
failure_detail: Optional[str] = None,
percent_complete: Optional[float] = None):
"""Method to update the job's metadata and provide feedback to the UI"""
current_job = get_current_job()
if not current_job:
return
if has_failures:
current_job.meta['has_failures'] = has_failures
if failure_detail:
current_job.meta['failure_detail'] = failure_detail
if percent_complete:
current_job.meta['percent_complete'] = percent_complete
current_job.meta['feedback'] = msg
current_job.save_meta()
logger = LMLogger.get_logger()
try:
update_feedback("Publish task in queue")
with repository.lock():
if isinstance(repository, LabBook):
wf = LabbookWorkflow(repository)
else:
wf = DatasetWorkflow(repository) # type: ignore
wf.publish(username=username,
access_token=access_token,
remote=remote or "origin",
public=public,
feedback_callback=update_feedback,
id_token=id_token)
except IOError:
raise
except Exception as e:
logger.exception(e)
raise Exception("Could not publish - try to log out and log in again.")
def _mock_import_labbook_from_remote(remote_url, username,
config_file):
print('X' * 200)
lb = InventoryManager(config_file).create_labbook(
username, username,
remote_url.split('/')[-1])
return LabbookWorkflow(lb)
def import_labbook_from_remote(remote_url: str,
username: str,
config_file: str = None) -> str:
"""Return the root directory of the newly imported Project"""
p = os.getpid()
logger = LMLogger.get_logger()
logger.info(
f"(Job {p}) Starting import_labbook_from_remote({remote_url}, {username})"
)
def update_meta(msg):
job = get_current_job()
if not job:
return
if 'feedback' not in job.meta:
job.meta['feedback'] = msg
else:
job.meta['feedback'] = job.meta['feedback'] + f'\n{msg}'
job.save_meta()
try:
toks = remote_url.split("/")
if len(toks) > 1:
proj_path = f'{toks[-2]}/{toks[-1].replace(".git", "")}'
else:
proj_path = remote_url
update_meta(f"Importing Project from {proj_path!r}...")
wf = LabbookWorkflow.import_from_remote(remote_url, username,
config_file)
update_meta(f"Imported Project {wf.labbook.name}!")
return wf.labbook.root_dir
except Exception as e:
update_meta(f"Could not import Project from {remote_url}.")
logger.exception(f"(Job {p}) Error on import_labbook_from_remote: {e}")
raise
def test_import_from_remote__nominal(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test import_from_remote method """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
wf.remote, username=other_user, config_file=mock_config_file[0])
# The remotes must be the same, cause it's the same remote repo
assert wf_other.remote == wf.remote
# The actual path on disk will be different, though
assert wf_other.repository != wf.repository
# Check imported into namespace of original owner (testuser)
assert f'{other_user}/{username}/labbooks/labbook1' in wf_other.repository.root_dir
def test_publish__publish_then_import_then_sync(self,
mock_labbook_lfs_disabled,
mock_config_file):
""" Test cannot publish a project already published. """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
remote_url=wf.remote,
username=other_user,
config_file=mock_config_file[0])
with open(os.path.join(wf_other.repository.root_dir, 'testfile'),
'w') as f:
f.write('filedata')
wf_other.repository.sweep_uncommitted_changes()
wf_other.sync(username=other_user)
commit_hash = wf_other.repository.git.commit_hash
assert wf.repository.git.commit_hash != commit_hash
wf.sync(username=username)
assert len(wf.repository.git.commit_hash) == len(commit_hash)
assert wf.repository.git.commit_hash == commit_hash
def test_reset__reset_local_change_same_owner(self,
mock_labbook_lfs_disabled,
mock_config_file):
""" test reset performs no operation when there's nothing to do """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
commit_to_check = lb.git.commit_hash
# Make some change locally and commit
fpath = os.path.join(lb.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('filedata')
lb.sweep_uncommitted_changes()
assert lb.git.commit_hash != commit_to_check
# Make an UNTRACKED change locally, make sure it gets clared up
untracked_file = os.path.join(lb.root_dir, 'output', 'untracked-file')
with open(untracked_file, 'w') as f:
f.write('untracked data')
# Do a reset and make sure state resets appropriately
wf.reset(username=username)
assert lb.git.commit_hash == commit_to_check
assert not os.path.exists(fpath)
assert not os.path.exists(untracked_file)
remote_hash = call_subprocess('git log -n 1 --oneline'.split(),
cwd=wf.remote).split()[0]
assert remote_hash in lb.git.commit_hash
def test_reset__no_op(self, mock_labbook_lfs_disabled, mock_config_file):
""" test reset performs no operation when there's nothing to do """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.reset(username=username)
wf.publish(username=username)
def test_publish__cannot_overwrite(self, mock_labbook_lfs_disabled):
""" Test cannot publish a project already published. """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
with pytest.raises(GitWorkflowException):
wf.publish(username=username)
def test_sync___push_up_then_sync(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test import_from_remote method """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
bm = BranchManager(lb, username='******')
bm.create_branch('new-branch-to-push')
wf.sync('test')
# Make some change locally and commit, then sync.
fpath = os.path.join(lb.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('filedata')
lb.sweep_uncommitted_changes()
wf.sync(username=username)
def test_migrate_old_schema_1_project(self, mock_config_file):
""" Test migrating a very old schema 1/gm.workspace LabBook """
p = resource_filename('gtmcore', 'workflows')
p2 = os.path.join(p, 'tests', 'snappy.zip')
with tempfile.TemporaryDirectory() as td:
call_subprocess(f"unzip {p2} -d {td}".split(), cwd=td)
temp_lb_path = os.path.join(td, 'snappy')
# Tests backwards compatibility (test.zip is a very old schema 1 LabBook)
lb = InventoryManager(
mock_config_file[0]).load_labbook_from_directory(temp_lb_path)
wf = LabbookWorkflow(lb)
wf.labbook.remove_remote()
wf.migrate()
# Test that current branch is as appropriate
assert wf.labbook.active_branch == 'master'
# Test that there is an activity record indicate migration
assert any([
'Migrate schema to 2' in c['message']
for c in wf.labbook.git.log()[:5]
])
# Test schema has successfully rolled to 2
assert wf.labbook.schema == 2
# Test that untracked space exists (if we add something to untracked space)
assert wf.labbook.is_repo_clean
with open(
os.path.join(lb.root_dir, 'output/untracked',
'untracked-file'), 'wb') as fb:
fb.write(b'cat' * 100)
assert wf.labbook.is_repo_clean
def test_sync___push_up_new_branch(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test import_from_remote method """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
bm = BranchManager(lb, username='******')
bm.create_branch('new-branch-to-push')
assert 'new-branch-to-push' not in bm.branches_remote
wf.sync('test')
assert 'new-branch-to-push' in bm.branches_remote
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
remote_url,
client_mutation_id=None):
username = get_logged_in_username()
logger.info(f"Importing remote labbook from {remote_url}")
lb = LabBook(author=get_logged_in_author())
default_remote = lb.client_config.config['git']['default_remote']
admin_service = None
for remote in lb.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = lb.client_config.config['git']['remotes'][
remote]['admin_service']
break
# Extract valid Bearer token
if hasattr(info.context, 'headers'
) and "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
else:
raise ValueError(
"Authorization header not provided. Must have a valid session to query for collaborators"
)
gl_mgr = GitLabManager(default_remote,
admin_service=admin_service,
access_token=token)
gl_mgr.configure_git_credentials(default_remote, username)
wf = LabbookWorkflow.import_from_remote(remote_url, username=username)
import_owner = InventoryManager().query_owner(wf.labbook)
# TODO: Fix cursor implementation, this currently doesn't make sense
cursor = base64.b64encode(f"{0}".encode('utf-8'))
lbedge = LabbookConnection.Edge(node=Labbook(owner=import_owner,
name=wf.labbook.name),
cursor=cursor)
return ImportRemoteLabbook(new_labbook_edge=lbedge)
def test_sync___simple_push_to_master(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test import_from_remote method """
username = '******'
lb = mock_labbook_lfs_disabled[2]
wf = LabbookWorkflow(lb)
wf.publish(username=username)
fpath = os.path.join(lb.root_dir, 'input', 'testfile')
with open(fpath, 'w') as f:
f.write('filedata')
lb.sweep_uncommitted_changes()
wf.sync(username=username)
# Check hash on remote - make sure it matches local.
remote_hash = call_subprocess('git log -n 1 --oneline'.split(),
cwd=wf.remote).split()[0]
assert remote_hash in lb.git.commit_hash
def test_should_migrate_on_old_project(self, mock_config_file):
p = resource_filename('gtmcore', 'workflows')
p2 = os.path.join(p, 'tests', 'lb-to-migrate-197b6a.zip')
with tempfile.TemporaryDirectory() as tempdir:
lbp = shutil.copyfile(p2, os.path.join(tempdir,
'lb-to-migrate.zip'))
subprocess.run(f'unzip lb-to-migrate.zip'.split(),
check=True,
cwd=tempdir)
im = InventoryManager(mock_config_file[0])
lb = im.load_labbook_from_directory(
os.path.join(tempdir, 'lb-to-migrate'))
wf = LabbookWorkflow(lb)
assert wf.should_migrate() is True
wf.migrate()
assert wf.labbook.active_branch == 'master'
assert wf.should_migrate() is False
wf.labbook.git.checkout('gm.workspace')
assert wf.should_migrate() is False
def test_publish__simple(self, mock_labbook_lfs_disabled):
"""Test a simple publish and ensuring master is active branch. """
username = '******'
lb = mock_labbook_lfs_disabled[2]
bm = BranchManager(lb, username)
bm.create_branch('test-local-only')
assert bm.branches_remote == []
assert bm.branches_local == ['master', 'test-local-only']
wf = LabbookWorkflow(lb)
# Test you can only publish on master.
with pytest.raises(GitWorkflowException):
wf.publish(username=username)
assert wf.remote is None
# Once we return to master branch, then we can publish.
bm.workon_branch(bm.workspace_branch)
wf.publish(username=username)
assert os.path.exists(wf.remote)
# Assert that publish only pushes up the master branch.
assert bm.branches_local == ['master', 'test-local-only']
assert bm.branches_remote == ['master']
def test_import_from_remote__linked_dataset(self,
mock_labbook_lfs_disabled,
mock_config_file):
""" test importing a project with a linked dataset"""
def dispatcher_mock(self, function_ref, kwargs, metadata):
assert kwargs['logged_in_username'] == 'other-test-user2'
assert kwargs['dataset_owner'] == 'testuser'
assert kwargs['dataset_name'] == 'test-ds'
# Inject mocked config file
kwargs['config_file'] = mock_config_file[0]
# Stop patching so job gets scheduled for real
dispatcher_patch.stop()
# Call same method as in mutation
d = Dispatcher()
res = d.dispatch_task(
gtmcore.dispatcher.dataset_jobs.check_and_import_dataset,
kwargs=kwargs,
metadata=metadata)
return res
username = '******'
lb = mock_labbook_lfs_disabled[2]
im = InventoryManager(config_file=mock_labbook_lfs_disabled[0])
ds = im.create_dataset(username,
username,
'test-ds',
storage_type='gigantum_object_v1')
# Publish dataset
dataset_wf = DatasetWorkflow(ds)
dataset_wf.publish(username=username)
# Link to project
im.link_dataset_to_labbook(dataset_wf.remote, username, username, lb)
# Publish project
labbook_wf = LabbookWorkflow(lb)
labbook_wf.publish(username=username)
# Patch dispatch_task so you can inject the mocked config file
dispatcher_patch = patch.object(Dispatcher, 'dispatch_task',
dispatcher_mock)
dispatcher_patch.start()
# Import project, triggering an auto-import of the dataset
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
labbook_wf.remote,
username=other_user,
config_file=mock_config_file[0])
# The remotes must be the same, cause it's the same remote repo
assert wf_other.remote == labbook_wf.remote
# The actual path on disk will be different, though
assert wf_other.repository != labbook_wf.repository
# Check imported into namespace of original owner (testuser)
assert f'{other_user}/{username}/labbooks/labbook1' in wf_other.repository.root_dir
cnt = 0
while cnt < 20:
try:
im_other_user = InventoryManager(
config_file=mock_config_file[0])
ds = im_other_user.load_dataset(other_user, username,
'test-ds')
break
except InventoryException:
cnt += 1
time.sleep(1)
assert cnt < 20
assert ds.name == 'test-ds'
assert ds.namespace == username
assert mock_config_file[1] in ds.root_dir
def _should_migrate(lb):
wf = LabbookWorkflow(lb)
return wf.should_migrate()
def test_checkout__linked_dataset(self, mock_labbook_lfs_disabled,
mock_config_file):
""" test checking out a branch in a project that pulls in a linked dataset"""
def dispatcher_mock(self, function_ref, kwargs, metadata):
assert kwargs['logged_in_username'] == 'other-test-user2'
assert kwargs['dataset_owner'] == 'testuser'
assert kwargs['dataset_name'] == 'test-ds'
# Inject mocked config file
kwargs['config_file'] = mock_config_file[0]
# Stop patching so job gets scheduled for real
dispatcher_patch.stop()
# Call same method as in mutation
d = Dispatcher()
res = d.dispatch_task(
gtmcore.dispatcher.dataset_jobs.check_and_import_dataset,
kwargs=kwargs,
metadata=metadata)
return res
username = '******'
lb = mock_labbook_lfs_disabled[2]
im = InventoryManager(config_file=mock_labbook_lfs_disabled[0])
ds = im.create_dataset(username,
username,
'test-ds',
storage_type='gigantum_object_v1')
# Publish dataset
dataset_wf = DatasetWorkflow(ds)
dataset_wf.publish(username=username)
# Publish project
labbook_wf = LabbookWorkflow(lb)
labbook_wf.publish(username=username)
# Switch branches
labbook_wf.labbook.checkout_branch(branch_name="dataset-branch",
new=True)
# Link to project
im.link_dataset_to_labbook(dataset_wf.remote, username, username,
labbook_wf.labbook)
# Publish branch
labbook_wf.sync(username=username)
# Import project
other_user = '******'
wf_other = LabbookWorkflow.import_from_remote(
labbook_wf.remote,
username=other_user,
config_file=mock_config_file[0])
# The remotes must be the same, cause it's the same remote repo
assert wf_other.remote == labbook_wf.remote
assert wf_other.repository != labbook_wf.repository
assert f'{other_user}/{username}/labbooks/labbook1' in wf_other.repository.root_dir
with pytest.raises(InventoryException):
im_other_user = InventoryManager(config_file=mock_config_file[0])
ds = im_other_user.load_dataset(other_user, username, 'test-ds')
# Patch dispatch_task so you can inject the mocked config file
dispatcher_patch = patch.object(Dispatcher, 'dispatch_task',
dispatcher_mock)
dispatcher_patch.start()
# Checkout the branch
assert wf_other.labbook.active_branch == "master"
wf_other.checkout(username=other_user, branch_name="dataset-branch")
cnt = 0
while cnt < 20:
try:
im_other_user = InventoryManager(
config_file=mock_config_file[0])
ds = im_other_user.load_dataset(other_user, username,
'test-ds')
break
except InventoryException:
cnt += 1
time.sleep(1)
assert cnt < 20
assert ds.name == 'test-ds'
assert ds.namespace == username
assert mock_config_file[1] in ds.root_dir
assert wf_other.labbook.active_branch == "dataset-branch"
