def test_fail_import_export_zip(self, mock_config_with_repo):
# Create new LabBook to be exported
lb = InventoryManager(mock_config_with_repo[0]).create_labbook(
'test',
'test',
"lb-fail-export-import-test",
description="Failing import-export.")
cm = ComponentManager(lb)
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
gtmcore.fixtures.ENV_UNIT_TEST_BASE,
gtmcore.fixtures.ENV_UNIT_TEST_REV)
lb_root = lb.root_dir
with tempfile.TemporaryDirectory() as temp_dir_path:
# Export the labbook
export_dir = os.path.join(mock_config_with_repo[1], "export")
try:
exported_archive_path = jobs.export_labbook_as_zip(
"/tmp", export_dir)
assert False, "Exporting /tmp should fail"
except Exception as e:
pass
# Export the labbook, then remove before re-importing
exported_archive_path = jobs.export_labbook_as_zip(
lb.root_dir, export_dir)
try:
imported_lb_path = jobs.import_labboook_from_zip(
archive_path=exported_archive_path,
username="******",
owner="test",
config_file=mock_config_with_repo[0])
assert False, f"Should not be able to import LabBook because it already exited at {lb_root}"
except Exception as e:
pass
try:
imported_lb_path = jobs.import_labboook_from_zip(
archive_path="/t",
username="******",
owner="test",
config_file=mock_config_with_repo[0])
assert False, f"Should not be able to import LabBook from strange directory /t"
except Exception as e:
pass
def test_import_labbook(self, fixture_working_dir):
"""Test batch uploading, but not full import"""
class DummyContext(object):
def __init__(self, file_handle):
self.labbook_loader = None
self.files = {'uploadChunk': file_handle}
client = Client(fixture_working_dir[3], middleware=[DataloaderMiddleware()])
# Create a temporary labbook
lb = InventoryManager(fixture_working_dir[0]).create_labbook("default", "default", "test-export",
description="Tester")
# Create a largeish file in the dir
with open(os.path.join(fixture_working_dir[1], 'testfile.bin'), 'wb') as testfile:
testfile.write(os.urandom(9000000))
FileOperations.insert_file(lb, 'input', testfile.name)
# Export labbook
zip_file = export_labbook_as_zip(lb.root_dir, tempfile.gettempdir())
lb_dir = lb.root_dir
# Get upload params
chunk_size = 4194304
file_info = os.stat(zip_file)
file_size = int(file_info.st_size / 1000)
total_chunks = int(math.ceil(file_info.st_size/chunk_size))
with open(zip_file, 'rb') as tf:
for chunk_index in range(total_chunks):
chunk = io.BytesIO()
chunk.write(tf.read(chunk_size))
chunk.seek(0)
file = FileStorage(chunk)
query = f"""
mutation myMutation{{
importLabbook(input:{{
chunkUploadParams:{{
uploadId: "jfdjfdjdisdjwdoijwlkfjd",
chunkSize: {chunk_size},
totalChunks: {total_chunks},
chunkIndex: {chunk_index},
fileSize: "{file_size}",
filename: "{os.path.basename(zip_file)}"
}}
}}) {{
importJobKey
}}
}}
"""
result = client.execute(query, context_value=DummyContext(file))
assert "errors" not in result
if chunk_index == total_chunks - 1:
assert type(result['data']['importLabbook']['importJobKey']) == str
assert "rq:job:" in result['data']['importLabbook']['importJobKey']
chunk.close()
def test_success_import_export_zip(self, mock_config_with_repo):
# Create new LabBook to be exported
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook('unittester',
'unittester',
"unittest-lb-for-export-import-test",
description="Testing import-export.")
cm = ComponentManager(lb)
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
gtmcore.fixtures.ENV_UNIT_TEST_BASE,
gtmcore.fixtures.ENV_UNIT_TEST_REV)
ib = ImageBuilder(lb)
ib.assemble_dockerfile()
# Make sure the destination user exists locally
working_dir = lb.client_config.config['git']['working_directory']
os.makedirs(os.path.join(working_dir, 'unittester2', 'unittester2',
'labbooks'),
exist_ok=True)
lb_root = lb.root_dir
with tempfile.TemporaryDirectory() as temp_dir_path:
# Export the labbook
export_dir = os.path.join(mock_config_with_repo[1], "export")
exported_archive_path = jobs.export_labbook_as_zip(
lb.root_dir, export_dir)
tmp_archive_path = shutil.copy(exported_archive_path, '/tmp')
# Delete the labbook
shutil.rmtree(lb.root_dir)
assert not os.path.exists(
lb_root), f"LabBook at {lb_root} should not exist."
assert os.path.exists(tmp_archive_path)
# Now import the labbook as a new user, validating that the change of namespace works properly.
imported_lb_path = jobs.import_labboook_from_zip(
archive_path=tmp_archive_path,
username='******',
owner='unittester2',
config_file=mock_config_with_repo[0])
assert not os.path.exists(tmp_archive_path)
tmp_archive_path = shutil.copy(exported_archive_path, '/tmp')
assert os.path.exists(tmp_archive_path)
# New path should reflect username of new owner and user.
assert imported_lb_path == lb_root.replace(
'/unittester/unittester/', '/unittester2/unittester2/')
import_lb = InventoryManager(
mock_config_with_repo[0]).load_labbook_from_directory(
imported_lb_path)
ib = ImageBuilder(import_lb)
ib.assemble_dockerfile(write=True)
assert os.path.exists(
os.path.join(imported_lb_path, '.gigantum', 'env',
'Dockerfile'))
assert not import_lb.has_remote
# Repeat the above, except with the original user (e.g., re-importing their own labbook)
user_import_lb = jobs.import_labboook_from_zip(
archive_path=tmp_archive_path,
username="******",
owner="unittester",
config_file=mock_config_with_repo[0])
assert not os.path.exists(tmp_archive_path)
# New path should reflect username of new owner and user.
assert user_import_lb
import_lb2 = InventoryManager(
mock_config_with_repo[0]).load_labbook_from_directory(
user_import_lb)
# After importing, the new user (in this case "cat") should be the current, active workspace.
# And be created, if necessary.
assert not import_lb2.has_remote
build_kwargs = {
'path': lb.root_dir,
'username': '******',
'nocache': True
}
docker_image_id = jobs.build_labbook_image(**build_kwargs)
try:
client = get_docker_client()
client.images.remove(docker_image_id)
except Exception as e:
pprint.pprint(e)
raise
def test_success_import_export_lbk(self, mock_config_with_repo):
"""Test legacy .lbk extension still works"""
# Create new LabBook to be exported
lb = InventoryManager(mock_config_with_repo[0]).create_labbook(
'unittester',
'unittester',
"unittest-lb-for-export-import-test-lbk",
description="Testing import-export.")
cm = ComponentManager(lb)
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
gtmcore.fixtures.ENV_UNIT_TEST_BASE,
gtmcore.fixtures.ENV_UNIT_TEST_REV)
ib = ImageBuilder(lb)
ib.assemble_dockerfile()
# Make sure the destination user exists locally
working_dir = lb.client_config.config['git']['working_directory']
os.makedirs(os.path.join(working_dir, 'unittester2', 'unittester2',
'labbooks'),
exist_ok=True)
lb_root = lb.root_dir
with tempfile.TemporaryDirectory() as temp_dir_path:
# Export the labbook
export_dir = os.path.join(mock_config_with_repo[1], "export")
exported_archive_path = jobs.export_labbook_as_zip(
lb.root_dir, export_dir)
tmp_archive_path = shutil.copy(exported_archive_path, '/tmp')
lbk_archive_path = tmp_archive_path.replace(".zip", ".lbk")
lbk_archive_path = shutil.copy(tmp_archive_path, lbk_archive_path)
print(lbk_archive_path)
# Delete the labbook
shutil.rmtree(lb.root_dir)
assert not os.path.exists(
lb_root), f"LabBook at {lb_root} should not exist."
assert os.path.exists(tmp_archive_path)
# Now import the labbook as a new user, validating that the change of namespace works properly.
imported_lb_path = jobs.import_labboook_from_zip(
archive_path=lbk_archive_path,
username='******',
owner='unittester2',
config_file=mock_config_with_repo[0])
assert not os.path.exists(lbk_archive_path)
tmp_archive_path = shutil.copy(exported_archive_path, '/tmp')
assert os.path.exists(tmp_archive_path)
# New path should reflect username of new owner and user.
assert imported_lb_path == lb_root.replace(
'/unittester/unittester/', '/unittester2/unittester2/')
import_lb = InventoryManager(
mock_config_with_repo[0]).load_labbook_from_directory(
imported_lb_path)
ib = ImageBuilder(import_lb)
ib.assemble_dockerfile(write=True)
assert os.path.exists(
os.path.join(imported_lb_path, '.gigantum', 'env',
'Dockerfile'))
assert not import_lb.has_remote