def test_fail_import_export_zip(self, mock_config_with_repo): # Create new LabBook to be exported lb = InventoryManager(mock_config_with_repo[0]).create_labbook( 'test', 'test', "lb-fail-export-import-test", description="Failing import-export.") cm = ComponentManager(lb) cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO, gtmcore.fixtures.ENV_UNIT_TEST_BASE, gtmcore.fixtures.ENV_UNIT_TEST_REV) lb_root = lb.root_dir with tempfile.TemporaryDirectory() as temp_dir_path: # Export the labbook export_dir = os.path.join(mock_config_with_repo[1], "export") try: exported_archive_path = jobs.export_labbook_as_zip( "/tmp", export_dir) assert False, "Exporting /tmp should fail" except Exception as e: pass # Export the labbook, then remove before re-importing exported_archive_path = jobs.export_labbook_as_zip( lb.root_dir, export_dir) try: imported_lb_path = jobs.import_labboook_from_zip( archive_path=exported_archive_path, username="******", owner="test", config_file=mock_config_with_repo[0]) assert False, f"Should not be able to import LabBook because it already exited at {lb_root}" except Exception as e: pass try: imported_lb_path = jobs.import_labboook_from_zip( archive_path="/t", username="******", owner="test", config_file=mock_config_with_repo[0]) assert False, f"Should not be able to import LabBook from strange directory /t" except Exception as e: pass
def test_import_labbook(self, fixture_working_dir): """Test batch uploading, but not full import""" class DummyContext(object): def __init__(self, file_handle): self.labbook_loader = None self.files = {'uploadChunk': file_handle} client = Client(fixture_working_dir[3], middleware=[DataloaderMiddleware()]) # Create a temporary labbook lb = InventoryManager(fixture_working_dir[0]).create_labbook("default", "default", "test-export", description="Tester") # Create a largeish file in the dir with open(os.path.join(fixture_working_dir[1], 'testfile.bin'), 'wb') as testfile: testfile.write(os.urandom(9000000)) FileOperations.insert_file(lb, 'input', testfile.name) # Export labbook zip_file = export_labbook_as_zip(lb.root_dir, tempfile.gettempdir()) lb_dir = lb.root_dir # Get upload params chunk_size = 4194304 file_info = os.stat(zip_file) file_size = int(file_info.st_size / 1000) total_chunks = int(math.ceil(file_info.st_size/chunk_size)) with open(zip_file, 'rb') as tf: for chunk_index in range(total_chunks): chunk = io.BytesIO() chunk.write(tf.read(chunk_size)) chunk.seek(0) file = FileStorage(chunk) query = f""" mutation myMutation{{ importLabbook(input:{{ chunkUploadParams:{{ uploadId: "jfdjfdjdisdjwdoijwlkfjd", chunkSize: {chunk_size}, totalChunks: {total_chunks}, chunkIndex: {chunk_index}, fileSize: "{file_size}", filename: "{os.path.basename(zip_file)}" }} }}) {{ importJobKey }} }} """ result = client.execute(query, context_value=DummyContext(file)) assert "errors" not in result if chunk_index == total_chunks - 1: assert type(result['data']['importLabbook']['importJobKey']) == str assert "rq:job:" in result['data']['importLabbook']['importJobKey'] chunk.close()
def test_success_import_export_zip(self, mock_config_with_repo): # Create new LabBook to be exported im = InventoryManager(mock_config_with_repo[0]) lb = im.create_labbook('unittester', 'unittester', "unittest-lb-for-export-import-test", description="Testing import-export.") cm = ComponentManager(lb) cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO, gtmcore.fixtures.ENV_UNIT_TEST_BASE, gtmcore.fixtures.ENV_UNIT_TEST_REV) ib = ImageBuilder(lb) ib.assemble_dockerfile() # Make sure the destination user exists locally working_dir = lb.client_config.config['git']['working_directory'] os.makedirs(os.path.join(working_dir, 'unittester2', 'unittester2', 'labbooks'), exist_ok=True) lb_root = lb.root_dir with tempfile.TemporaryDirectory() as temp_dir_path: # Export the labbook export_dir = os.path.join(mock_config_with_repo[1], "export") exported_archive_path = jobs.export_labbook_as_zip( lb.root_dir, export_dir) tmp_archive_path = shutil.copy(exported_archive_path, '/tmp') # Delete the labbook shutil.rmtree(lb.root_dir) assert not os.path.exists( lb_root), f"LabBook at {lb_root} should not exist." assert os.path.exists(tmp_archive_path) # Now import the labbook as a new user, validating that the change of namespace works properly. imported_lb_path = jobs.import_labboook_from_zip( archive_path=tmp_archive_path, username='******', owner='unittester2', config_file=mock_config_with_repo[0]) assert not os.path.exists(tmp_archive_path) tmp_archive_path = shutil.copy(exported_archive_path, '/tmp') assert os.path.exists(tmp_archive_path) # New path should reflect username of new owner and user. assert imported_lb_path == lb_root.replace( '/unittester/unittester/', '/unittester2/unittester2/') import_lb = InventoryManager( mock_config_with_repo[0]).load_labbook_from_directory( imported_lb_path) ib = ImageBuilder(import_lb) ib.assemble_dockerfile(write=True) assert os.path.exists( os.path.join(imported_lb_path, '.gigantum', 'env', 'Dockerfile')) assert not import_lb.has_remote # Repeat the above, except with the original user (e.g., re-importing their own labbook) user_import_lb = jobs.import_labboook_from_zip( archive_path=tmp_archive_path, username="******", owner="unittester", config_file=mock_config_with_repo[0]) assert not os.path.exists(tmp_archive_path) # New path should reflect username of new owner and user. assert user_import_lb import_lb2 = InventoryManager( mock_config_with_repo[0]).load_labbook_from_directory( user_import_lb) # After importing, the new user (in this case "cat") should be the current, active workspace. # And be created, if necessary. assert not import_lb2.has_remote build_kwargs = { 'path': lb.root_dir, 'username': '******', 'nocache': True } docker_image_id = jobs.build_labbook_image(**build_kwargs) try: client = get_docker_client() client.images.remove(docker_image_id) except Exception as e: pprint.pprint(e) raise
def test_success_import_export_lbk(self, mock_config_with_repo): """Test legacy .lbk extension still works""" # Create new LabBook to be exported lb = InventoryManager(mock_config_with_repo[0]).create_labbook( 'unittester', 'unittester', "unittest-lb-for-export-import-test-lbk", description="Testing import-export.") cm = ComponentManager(lb) cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO, gtmcore.fixtures.ENV_UNIT_TEST_BASE, gtmcore.fixtures.ENV_UNIT_TEST_REV) ib = ImageBuilder(lb) ib.assemble_dockerfile() # Make sure the destination user exists locally working_dir = lb.client_config.config['git']['working_directory'] os.makedirs(os.path.join(working_dir, 'unittester2', 'unittester2', 'labbooks'), exist_ok=True) lb_root = lb.root_dir with tempfile.TemporaryDirectory() as temp_dir_path: # Export the labbook export_dir = os.path.join(mock_config_with_repo[1], "export") exported_archive_path = jobs.export_labbook_as_zip( lb.root_dir, export_dir) tmp_archive_path = shutil.copy(exported_archive_path, '/tmp') lbk_archive_path = tmp_archive_path.replace(".zip", ".lbk") lbk_archive_path = shutil.copy(tmp_archive_path, lbk_archive_path) print(lbk_archive_path) # Delete the labbook shutil.rmtree(lb.root_dir) assert not os.path.exists( lb_root), f"LabBook at {lb_root} should not exist." assert os.path.exists(tmp_archive_path) # Now import the labbook as a new user, validating that the change of namespace works properly. imported_lb_path = jobs.import_labboook_from_zip( archive_path=lbk_archive_path, username='******', owner='unittester2', config_file=mock_config_with_repo[0]) assert not os.path.exists(lbk_archive_path) tmp_archive_path = shutil.copy(exported_archive_path, '/tmp') assert os.path.exists(tmp_archive_path) # New path should reflect username of new owner and user. assert imported_lb_path == lb_root.replace( '/unittester/unittester/', '/unittester2/unittester2/') import_lb = InventoryManager( mock_config_with_repo[0]).load_labbook_from_directory( imported_lb_path) ib = ImageBuilder(import_lb) ib.assemble_dockerfile(write=True) assert os.path.exists( os.path.join(imported_lb_path, '.gigantum', 'env', 'Dockerfile')) assert not import_lb.has_remote