def test_success_export_then_import_different_users(
self, mock_config_file):
inv_manager = InventoryManager(mock_config_file[0])
ds = inv_manager.create_dataset('unittester', 'unittester',
'unittest-zip', 'gigantum_object_v1')
with tempfile.TemporaryDirectory() as tempd:
path = ZipExporter.export_dataset(ds.root_dir, tempd)
newds = ZipExporter.import_dataset(path, "unittester2",
"unittester2",
mock_config_file[0])
assert not os.path.exists(path)
assert 'unittester2' == InventoryManager(
mock_config_file[0]).query_owner(newds)
assert newds.is_repo_clean
assert newds.active_branch == 'master'
# Now try with same user as exporter
path2 = ZipExporter.export_dataset(ds.root_dir, tempd)
shutil.rmtree(ds.root_dir)
lb2 = ZipExporter.import_dataset(path2, "unittester", "unittester",
mock_config_file[0])
assert 'unittester' == InventoryManager(
mock_config_file[0]).query_owner(lb2)
assert lb2.is_repo_clean
assert lb2.active_branch == 'master'
def test_fail_cannot_import_labbook_to_overwrite_name(self, mock_config_file):
import_zip = os.path.join(resource_filename('gtmcore','workflows'),
'tests', 'snappy.zip')
dup_import = shutil.copy(import_zip, '/tmp/copy-of-snappy.zip')
z = ZipExporter()
x = z.import_labbook(dup_import, 'test', 'test', mock_config_file[0])
dup_import = shutil.copy(import_zip, '/tmp/copy-of-snappy.zip')
# Now try to import that again and it should fail, cause a
# project by that name already exists.
with pytest.raises(ZipWorkflowException):
y = z.import_labbook(dup_import, 'test', 'test', mock_config_file[0])
def test_success_import_valid_labbook_from_windows(self, mock_config_file):
import_zip = os.path.join(resource_filename('gtmcore','workflows/tests'),
'test_from_windows.zip')
dup_import = shutil.copy(import_zip, '/tmp/copy-of-test_from_windows.zip')
workspace = Configuration(mock_config_file[0]).config['git']['working_directory']
# Snapshots of directories before and after import - assert different
pre_snapshot = str(list(sorted(os.walk(workspace))))
z = ZipExporter()
x = z.import_labbook(dup_import, 'test', 'test', mock_config_file[0])
post_snapshot = str(list(sorted(os.walk(workspace))))
assert pre_snapshot != post_snapshot
def test_fail_import_non_labbook_zip_single_file(self, mock_config_file):
# Test a valid zip file but one that does not contain project
# make a zip of a single file
with open('/tmp/single_file_zip.file', 'wb') as f:
f.write(b'abc123' * 500)
subprocess.run('zip -r single_file_zip.zip single_file_zip.file'.split(),
check=True, cwd='/tmp')
assert os.path.isfile('/tmp/single_file_zip.zip')
# Baseline state of gigantum dir (used to check no new files created)
snapshot = str(list(sorted(os.walk(mock_config_file[1]))))
z = ZipExporter()
with pytest.raises(ZipWorkflowException):
z.import_labbook('/tmp/single_file_zip.zip', 'test', 'test', mock_config_file[0])
def test_fail_import_non_labbook_zip_directory(self, mock_config_file):
# Test a valid zip file but one that does not contain project
# make a zip of a directory
dirs = ['code', '.git', '.gigantum', '.labbook', 'input', 'output']
for d in dirs:
os.makedirs(os.path.join('/tmp/non-lb-dir', d), exist_ok=True)
with open(os.path.join('/tmp/non-lb-dir', d, 'file.f'), 'wb') as f:
f.write(b'file content. ' * 40)
subprocess.run('zip -r non-lb-dir.zip non-lb-dir'.split(),
check=True, cwd='/tmp')
# Baseline state of gigantum dir (used to check no new files created)
pre_snapshot = str(list(sorted(os.walk(mock_config_file[1]))))
z = ZipExporter()
with pytest.raises(ZipWorkflowException):
z.import_labbook('/tmp/non-lb-dir.zip', 'test', 'test', mock_config_file[0])
post_snapshot = str(list(sorted(os.walk(mock_config_file[1]))))
assert pre_snapshot == post_snapshot
def test_success_export_then_import_different_users(self, mock_config_file):
inv_manager = InventoryManager(mock_config_file[0])
lb = inv_manager.create_labbook('unittester', 'unittester', 'unittest-zip')
with tempfile.TemporaryDirectory() as tempd:
path = ZipExporter.export_labbook(lb.root_dir, tempd)
newlb = ZipExporter.import_labbook(path, "unittester2", "unittester2",
mock_config_file[0])
assert not os.path.exists(path)
assert 'unittester2' == InventoryManager(mock_config_file[0]).query_owner(newlb)
assert newlb.is_repo_clean
# Now try with same user as exporter
path2 = ZipExporter.export_labbook(lb.root_dir, tempd)
shutil.rmtree(lb.root_dir)
lb2 = ZipExporter.import_labbook(path2, "unittester", "unittester",
mock_config_file[0])
assert 'unittester' == InventoryManager(mock_config_file[0]).query_owner(lb2)
assert lb2.is_repo_clean
def export_dataset_as_zip(dataset_path: str, ds_export_directory: str) -> str:
"""Return path to archive file of exported dataset. """
p = os.getpid()
logger = LMLogger.get_logger()
logger.info(f"(Job {p}) Starting export_dataset_as_zip({dataset_path})")
try:
ds = InventoryManager().load_dataset_from_directory(dataset_path)
with ds.lock():
path = ZipExporter.export_dataset(ds.root_dir, ds_export_directory)
return path
except Exception as e:
logger.exception(f"(Job {p}) Error on export_dataset_as_zip: {e}")
raise
def export_labbook_as_zip(labbook_path: str, lb_export_directory: str) -> str:
"""Return path to archive file of exported labbook. """
p = os.getpid()
logger = LMLogger.get_logger()
logger.info(f"(Job {p}) Starting export_labbook_as_zip({labbook_path})")
try:
lb = InventoryManager().load_labbook_from_directory(labbook_path)
with lb.lock():
path = ZipExporter.export_labbook(lb.root_dir, lb_export_directory)
return path
except Exception as e:
logger.exception(f"(Job {p}) Error on export_labbook_as_zip: {e}")
raise
def import_dataset_from_zip(archive_path: str,
username: str,
owner: str,
config_file: Optional[str] = None) -> str:
"""Method to import a dataset from a zip file
Args:
archive_path(str): Path to the uploaded zip
username(str): Username
owner(str): Owner username
config_file(str): Optional path to a labmanager config file
Returns:
str: directory path of imported labbook
"""
def update_meta(msg):
job = get_current_job()
if not job:
return
job.meta['feedback'] = msg
job.save_meta()
p = os.getpid()
logger = LMLogger.get_logger()
logger.info(
f"(Job {p}) Starting import_dataset_from_zip(archive_path={archive_path},"
f"username={username}, owner={owner}, config_file={config_file})")
try:
lb = ZipExporter.import_dataset(archive_path,
username,
owner,
config_file=config_file,
update_meta=update_meta)
return lb.root_dir
except Exception as e:
logger.exception(
f"(Job {p}) Error on import_dataset_from_zip({archive_path}): {e}")
raise
finally:
if os.path.exists(archive_path):
os.remove(archive_path)
def test_fail_import_garbled(self, mock_config_file):
# Baseline state of gigantum dir (used to check no new files created)
snapshot = str(list(sorted(os.walk(mock_config_file[1]))))
z = ZipExporter()
# Test with a non-existing file
with pytest.raises(ZipWorkflowException):
z.import_labbook('nonsense.zip', 'none', 'none', mock_config_file[0])
assert snapshot == str(list(sorted(os.walk(mock_config_file[1]))))
# Test with an invalid zip file
with open('/tmp/invalid.zip', 'wb') as x:
x.write(b'Invalid zip file content. ' * 500)
with pytest.raises(ZipWorkflowException):
z.import_labbook(x.name, 'none', 'none', mock_config_file[0])
assert snapshot == str(list(sorted(os.walk(mock_config_file[1]))))
def test_fail_export_not_a_labbook(self):
# Pass in a valid directory, but one that is not an LB
with pytest.raises(ZipWorkflowException):
ZipExporter.export_labbook('/var', '.')
def test_fail_export_garbled_export(self):
# Test giving a path that doesn't exist
with pytest.raises(ZipWorkflowException):
ZipExporter.export_labbook('/not/a/real/path', '.')