def remote_labbook_repo(): # TODO: Remove after integration tests with LFS support are available conf_file, working_dir = _create_temp_work_dir(lfs_enabled=False) im = InventoryManager(conf_file) lb = im.create_labbook('test', 'test', 'sample-repo-lb', description="my first labbook") bm = BranchManager(lb, username='******') bm.create_branch('testing-branch') #with tempfile.TemporaryDirectory() as tmpdirname: with open(os.path.join('/tmp', 'codefile.c'), 'wb') as codef: codef.write(b'// Cody McCodeface ...') FileOperations.insert_file(lb, "code", "/tmp/codefile.c") assert lb.is_repo_clean bm.workon_branch('master') # Location of the repo to push/pull from yield lb.root_dir shutil.rmtree(working_dir) try: os.remove('/tmp/codefile.c') except: pass
def test_sync_1(self, mock_create_labbooks_no_lfs, mock_config_file): # Setup responses mock for this test responses.add(responses.GET, 'https://usersrv.gigantum.io/key', json={'key': 'afaketoken'}, status=200) im = InventoryManager(mock_create_labbooks_no_lfs[0]) test_user_lb = im.load_labbook('default', 'default', 'labbook1') test_user_wf = LabbookWorkflow(test_user_lb) test_user_wf.publish('default') # Mock the request context so a fake authorization header is present builder = EnvironBuilder(path='/labbook', method='POST', headers={'Authorization': 'Bearer AJDFHASD'}) env = builder.get_environ() req = Request(environ=env) sally_wf = LabbookWorkflow.import_from_remote(test_user_wf.remote, 'sally', config_file=mock_config_file[0]) sally_lb = sally_wf.labbook FileOperations.makedir(sally_lb, relative_path='code/sally-dir', create_activity_record=True) sally_wf.sync('sally') sync_query = """ mutation x { syncLabbook(input: { labbookName: "labbook1", owner: "default" }) { jobKey } } """ r = mock_create_labbooks_no_lfs[2].execute(sync_query, context_value=req) assert 'errors' not in r
def test_reflect_deleted_files_on_merge_in(self, mock_create_labbooks): lb, client = mock_create_labbooks[0], mock_create_labbooks[1] with open('/tmp/s1.txt', 'w') as s1: s1.write('original-file\ndata') FileOperations.insert_file(lb, section='code', src_file=s1.name) bm = BranchManager(lb, username=UT_USERNAME) nb = bm.create_branch(f'new-branch') assert os.path.exists(os.path.join(lb.root_dir, 'code', 's1.txt')) FileOperations.delete_files(lb, 'code', ['s1.txt']) assert lb.is_repo_clean assert not os.path.exists(os.path.join(lb.root_dir, 'code', 's1.txt')) bm.workon_branch(bm.workspace_branch) assert os.path.exists(os.path.join(lb.root_dir, 'code', 's1.txt')) merge_q = f""" mutation x {{ mergeFromBranch(input: {{ owner: "{UT_USERNAME}", labbookName: "{UT_LBNAME}", otherBranchName: "{nb}" }}) {{ labbook{{ name description activeBranchName }} }} }} """ r = client.execute(merge_q) assert 'errors' not in r assert r['data']['mergeFromBranch']['labbook']['activeBranchName'] == 'master' assert not os.path.exists(os.path.join(lb.root_dir, 'code', 's1.txt'))
def test_insert_and_make_intermediary_directories(self, mock_labbook, sample_src_file): lb = mock_labbook[2] FO.insert_file(lb, "code", sample_src_file, "/super/random/dir/inside.file") p = os.path.join(lb.root_dir, 'code', "super/random/dir/inside.file") assert os.path.isfile(p)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, directory, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): FileOperations.makedir(lb, os.path.join(section, directory), create_activity_record=True) # Prime dataloader with labbook you already loaded dataloader = LabBookLoader() dataloader.prime(f"{owner}&{labbook_name}&{lb.name}", lb) # Create data to populate edge file_info = FileOperations.get_file_info(lb, section, directory) create_data = {'owner': owner, 'name': labbook_name, 'section': section, 'key': file_info['key'], '_file_info': file_info} # TODO: Fix cursor implementation, this currently doesn't make sense cursor = base64.b64encode(f"{0}".encode('utf-8')) return MakeLabbookDirectory( new_labbook_file_edge=LabbookFileConnection.Edge( node=LabbookFile(**create_data), cursor=cursor))
def mock_create_labbooks(fixture_working_dir): # Create a labbook in the temporary directory config_file = fixture_working_dir[0] im = InventoryManager(fixture_working_dir[0]) lb = im.create_labbook(UT_USERNAME, UT_USERNAME, UT_LBNAME, description="Cats labbook 1") # Create a file in the dir with open(os.path.join(fixture_working_dir[1], 'unittest-examplefile'), 'w') as sf: sf.write("test data") sf.seek(0) FileOperations.insert_file(lb, 'code', sf.name) assert os.path.isfile( os.path.join(lb.root_dir, 'code', 'unittest-examplefile')) # Create test client schema = graphene.Schema(query=LabbookQuery, mutation=LabbookMutations) with patch.object(Configuration, 'find_default_config', lambda self: config_file): app = Flask("lmsrvlabbook") app.config["LABMGR_CONFIG"] = Configuration() app.config["LABMGR_ID_MGR"] = get_identity_manager(Configuration()) with app.app_context(): flask.g.user_obj = app.config["LABMGR_ID_MGR"].get_user_profile() client = Client( schema, middleware=[DataloaderMiddleware(), error_middleware], context_value=ContextMock()) yield lb, client, schema shutil.rmtree(fixture_working_dir, ignore_errors=True)
def test_move_loaded_directory_with_one_file(self, mock_labbook, mock_config_file, sample_src_file): lb = mock_labbook[2] new_file_data = FO.insert_file(lb, "code", sample_src_file) base_name = os.path.basename(new_file_data['key']) assert os.path.exists(os.path.join(lb.root_dir, 'code', base_name)) # make new subdir os.makedirs(os.path.join(lb.root_dir, 'code', 'subdir')) # .. and then put a file in it mv_file_res = FO.move_file(lb, "code", base_name, os.path.join('subdir', base_name)) # Should be 2, because it returns the info of the directory it was moved into assert len(mv_file_res) == 1 assert mv_file_res[0]['key'] == f'subdir/{base_name}' assert mv_file_res[0]['is_dir'] == False # Move "subdir" into "target_dir", there should be two activity records FO.makedir(lb, "code/target_dir", create_activity_record=True) mv_dir_res = FO.move_file(lb, "code", 'subdir', 'target_dir') assert len(mv_dir_res) == 2 assert mv_dir_res[0]['key'] == 'target_dir/subdir/' assert mv_dir_res[0]['is_dir'] is True assert mv_dir_res[1]['key'] == f'target_dir/subdir/{base_name}' assert mv_dir_res[1]['is_dir'] is False assert not os.path.exists(os.path.join(lb.root_dir, 'code', 'subdir')) assert os.path.exists(os.path.join(lb.root_dir, 'code', 'target_dir/subdir'))
def test_remove_file_success(self, mock_labbook, sample_src_file): lb = mock_labbook[2] new_file_data = FO.insert_file(lb, "code", sample_src_file) base_name = os.path.basename(new_file_data['key']) assert os.path.exists(os.path.join(lb.root_dir, 'code', base_name)) FO.delete_files(lb, 'code', [base_name]) assert not os.path.exists(os.path.join(lb.root_dir, 'code', base_name))
def test_listdir(self, mock_labbook, sample_src_file): def write_test_file(base, name): with open(os.path.join(base, name), 'wt') as f: f.write("Blah blah") lb = mock_labbook[2] dirs = ["code/new_dir", ".hidden_dir"] for d in dirs: FO.makedir(lb, d) write_test_file(lb.root_dir, 'test1.txt') write_test_file(lb.root_dir, 'test2.txt') write_test_file(lb.root_dir, '.hidden.txt') write_test_file(lb.root_dir, 'code/test_subdir1.txt') write_test_file(lb.root_dir, 'code/test_subdir2.txt') write_test_file(lb.root_dir, 'code/new_dir/tester.txt') # List just the code dir data = FO.listdir(lb, "code", base_path='') assert len(data) == 3 assert data[0]['key'] == 'new_dir/' assert data[1]['key'] == 'test_subdir1.txt' assert data[2]['key'] == 'test_subdir2.txt' data = FO.listdir(lb, "input", base_path='') assert len(data) == 0 # List just the code/subdir dir data = FO.listdir(lb, "code", base_path='new_dir') assert len(data) == 1 assert data[0]['key'] == 'new_dir/tester.txt'
def test_insert_file_success_2(self, mock_labbook, sample_src_file): lb = mock_labbook[2] FO.makedir(lb, "output/testdir") new_file_data = FO.insert_file(lb, "output", sample_src_file, "testdir") base_name = os.path.basename(new_file_data['key']) assert os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir', base_name)) assert new_file_data['key'] == f'testdir/{base_name}' assert new_file_data['is_dir'] is False
def test_detail_record_node(self, fixture_working_dir, fixture_test_file): """Test getting an detail record by node ID""" im = InventoryManager(fixture_working_dir[0]) lb = im.create_labbook("default", "default", "labbook1", description="my test description") FileOperations.insert_file(lb, "code", fixture_test_file) # Get activity record to query = """ { labbook(name: "labbook1", owner: "default") { activityRecords { edges{ node{ id commit linkedCommit message type show importance tags detailObjects{ id key type data show importance tags } } } } } } """ result1 = fixture_working_dir[2].execute(query) assert 'errors' not in result1 query = """ {{ node(id: "{}") {{ ... on ActivityDetailObject {{ id key type data show importance tags }} }} }} """.format(result1['data']['labbook']['activityRecords']['edges'][0]['node']['detailObjects'][0]['id']) result2 = fixture_working_dir[2].execute(query) assert 'errors' not in result2 assert result2['data']['node'] == result1['data']['labbook']['activityRecords']['edges'][0]['node']['detailObjects'][0]
def test_import_labbook(self, fixture_working_dir): """Test batch uploading, but not full import""" class DummyContext(object): def __init__(self, file_handle): self.labbook_loader = None self.files = {'uploadChunk': file_handle} client = Client(fixture_working_dir[3], middleware=[DataloaderMiddleware()]) # Create a temporary labbook lb = InventoryManager(fixture_working_dir[0]).create_labbook("default", "default", "test-export", description="Tester") # Create a largeish file in the dir with open(os.path.join(fixture_working_dir[1], 'testfile.bin'), 'wb') as testfile: testfile.write(os.urandom(9000000)) FileOperations.insert_file(lb, 'input', testfile.name) # Export labbook zip_file = export_labbook_as_zip(lb.root_dir, tempfile.gettempdir()) lb_dir = lb.root_dir # Get upload params chunk_size = 4194304 file_info = os.stat(zip_file) file_size = int(file_info.st_size / 1000) total_chunks = int(math.ceil(file_info.st_size/chunk_size)) with open(zip_file, 'rb') as tf: for chunk_index in range(total_chunks): chunk = io.BytesIO() chunk.write(tf.read(chunk_size)) chunk.seek(0) file = FileStorage(chunk) query = f""" mutation myMutation{{ importLabbook(input:{{ chunkUploadParams:{{ uploadId: "jfdjfdjdisdjwdoijwlkfjd", chunkSize: {chunk_size}, totalChunks: {total_chunks}, chunkIndex: {chunk_index}, fileSize: "{file_size}", filename: "{os.path.basename(zip_file)}" }} }}) {{ importJobKey }} }} """ result = client.execute(query, context_value=DummyContext(file)) assert "errors" not in result if chunk_index == total_chunks - 1: assert type(result['data']['importLabbook']['importJobKey']) == str assert "rq:job:" in result['data']['importLabbook']['importJobKey'] chunk.close()
def test_remove_file_fail_old_prototype(self, mock_labbook, sample_src_file): lb = mock_labbook[2] new_file_data = FO.insert_file(lb, "code", sample_src_file) base_name = os.path.basename(new_file_data['key']) assert os.path.exists(os.path.join(lb.root_dir, 'code', base_name)) with pytest.raises(ValueError): FO.delete_files(lb, 'code', base_name)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, file_paths, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): FileOperations.delete_files(lb, section, file_paths) return DeleteLabbookFiles(success=True)
def test_move_single_file(self, mock_labbook, mock_config_file, sample_src_file): lb = mock_labbook[2] f = FO.insert_file(lb, 'code', sample_src_file)['key'] FO.makedir(lb, 'code/target_dir') results = FO.move_file(lb, 'code', f, 'target_dir') assert len(results) == 1 pprint.pprint(results) assert results[0]['is_dir'] == False assert results[0]['key'] == 'target_dir/' + os.path.basename(sample_src_file)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, transaction_id, cancel=False, rollback=False, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): FileOperations.complete_batch(lb, transaction_id, cancel=cancel, rollback=rollback) return CompleteBatchUploadTransaction(success=True)
def test_move_empty_directory(self, mock_labbook, mock_config_file, sample_src_file): lb = mock_labbook[2] FO.makedir(lb, 'code/stable_dir') FO.makedir(lb, 'code/empty_dir') # We'll move "empty_dir" into "stable_dir" - there should only be one element in returned list res = FO.move_file(lb, 'code', 'empty_dir', 'stable_dir') assert len(res) == 1 assert res[0]['is_dir'] is True assert res[0]['key'] == 'stable_dir/empty_dir/'
def test_move_single_file_to_section_top(self, mock_labbook, mock_config_file, sample_src_file): lb = mock_labbook[2] FO.makedir(lb, 'code/inner_dir') f = FO.insert_file(lb, 'code', sample_src_file, 'inner_dir')['key'] # Move file to top of code section results = FO.move_file(lb, 'code', f, dst_rel_path='') # Results should be returned for "code" -- the file just moved there and the assert len(results) == 1 assert results[0]['is_dir'] == False assert results[0]['key'] == os.path.basename(f)
def test_remove_dir(self, mock_labbook, sample_src_file): lb = mock_labbook[2] FO.makedir(lb, "output/testdir") new_file_path = FO.insert_file(lb, "output", sample_src_file, "testdir") base_name = os.path.basename(new_file_path['key']) assert os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir', base_name)) # Note! Now that remove() uses force=True, no special action is needed for directories. # Delete the directory FO.delete_files(lb, "output", ["testdir"]) assert not os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir', base_name)) assert not os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir'))
def test_makedir_record(self, mock_labbook): # Note that "score" refers to the count of .gitkeep files. lb = mock_labbook[2] assert os.path.exists(os.path.join(lb.root_dir, 'code', 'test')) is False FO.makedir(lb, "code/test", create_activity_record=True) assert os.path.exists(os.path.join(lb.root_dir, 'code', 'test')) is True assert lb.is_repo_clean is True FO.makedir(lb, "code/test2", create_activity_record=False) assert os.path.exists(os.path.join(lb.root_dir, 'code', 'test2')) is True assert lb.is_repo_clean is False
def test_remove_empty_dir(self, mock_labbook, sample_src_file): lb = mock_labbook[2] FO.makedir(lb, "output/testdir") new_file_path = FO.insert_file(lb, "output", sample_src_file, "testdir") base_name = os.path.basename(new_file_path['key']) assert os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir', base_name)) # Delete the directory FO.delete_files(lb, "output", ["testdir"]) assert not os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir', base_name)) assert not os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir'))
def test_merge_into_feature_from_workspace_simple_success( self, mock_create_labbooks): lb, client = mock_create_labbooks[0], mock_create_labbooks[1] bm = BranchManager(lb, username=UT_USERNAME) og_hash = lb.git.commit_hash b1 = bm.create_branch(f"test-branch") bm.workon_branch(bm.workspace_branch) assert lb.active_branch == bm.workspace_branch og2_hash = lb.git.commit_hash assert lb.git.commit_hash == og_hash FileOperations.makedir(lb, 'code/main-branch-dir1', create_activity_record=True) FileOperations.makedir(lb, 'code/main-branch-dir2', create_activity_record=True) next_main_hash = lb.git.commit_hash assert og_hash != next_main_hash bm.workon_branch(b1) assert not os.path.exists( os.path.join(lb.root_dir, 'code/main-branch-dir1')) merge_q = f""" mutation x {{ mergeFromBranch(input: {{ owner: "{UT_USERNAME}", labbookName: "{UT_LBNAME}", otherBranchName: "{bm.workspace_branch}" }}) {{ labbook{{ name description activeBranchName }} }} }} """ r = client.execute(merge_q) assert 'errors' not in r assert r['data']['mergeFromBranch']['labbook'][ 'activeBranchName'] == 'test-branch' assert r['data']['mergeFromBranch']['labbook'][ 'name'] == 'unittest-workflow-branch-1' assert lb.active_branch == b1 assert os.path.exists( os.path.join(lb.root_dir, 'code/main-branch-dir1')) assert lb.is_repo_clean
def mock_create_labbooks(fixture_working_dir): # Create a labbook in the temporary directory lb = InventoryManager(fixture_working_dir[0]).create_labbook( "default", "default", "sample-repo-lb", description="Cats labbook 1") # Create a file in the dir with open(os.path.join(fixture_working_dir[1], 'codefile.c'), 'w') as sf: sf.write("1234567") sf.seek(0) FileOperations.insert_file(lb, 'code', sf.name) assert os.path.isfile(os.path.join(lb.root_dir, 'code', 'codefile.c')) # name of the config file, temporary working directory, the schema yield fixture_working_dir, lb
def test_move_file_as_rename_in_same_dir(self, mock_labbook, sample_src_file): lb = mock_labbook[2] # insert file new_file_data = FO.insert_file(lb, "code", sample_src_file, '') base_name = os.path.basename(new_file_data['key']) assert os.path.exists(os.path.join(lb.root_dir, 'code', base_name)) assert new_file_data['key'] == base_name # move to rename moved_rel_path = os.path.join(f'{base_name}.MOVED') r = FO.move_file(lb, 'code', new_file_data['key'], moved_rel_path) assert len(r) == 1 assert not os.path.exists(os.path.join(lb.root_dir, 'code', base_name)) assert os.path.exists(os.path.join(lb.root_dir, 'code', f'{base_name}.MOVED')) assert os.path.isfile(os.path.join(lb.root_dir, 'code', f'{base_name}.MOVED'))
def mock_create_labbooks_no_lfs(fixture_working_dir_lfs_disabled): # Create a labbook in the temporary directory im = InventoryManager(fixture_working_dir_lfs_disabled[0]) lb = im.create_labbook("default", "default", "labbook1", description="Cats labbook 1") # Create a file in the dir with open(os.path.join(fixture_working_dir_lfs_disabled[1], 'sillyfile'), 'w') as sf: sf.write("1234567") sf.seek(0) FileOperations.insert_file(lb, 'code', sf.name) assert os.path.isfile(os.path.join(lb.root_dir, 'code', 'sillyfile')) # name of the config file, temporary working directory, the schema yield fixture_working_dir_lfs_disabled
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, src_path, dst_path, client_mutation_id=None, **kwargs): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): mv_results = FileOperations.move_file(lb, section, src_path, dst_path) file_edges = list() for file_dict in mv_results: file_edges.append(LabbookFile(owner=owner, name=labbook_name, section=section, key=file_dict['key'], is_dir=file_dict['is_dir'], is_favorite=file_dict['is_favorite'], modified_at=file_dict['modified_at'], size=str(file_dict['size']))) cursors = [base64.b64encode("{}".format(cnt).encode("UTF-8")).decode("UTF-8") for cnt, x in enumerate(file_edges)] edge_objs = [LabbookFileConnection.Edge(node=e, cursor=c) for e, c in zip(file_edges, cursors)] return MoveLabbookFile(updated_edges=edge_objs)
def test_labbook_content_size_simply(self, mock_labbook): x, y, lb = mock_labbook lb_size = FO.content_size(lb) # Make sure the new LB is about 10-30kB. This is about reasonable for a new, emtpy LB. assert lb_size > 10000 assert lb_size < 30000
def test_insert_file_success_1(self, mock_labbook, sample_src_file): lb = mock_labbook[2] new_file_data = FO.insert_file(lb, "code", sample_src_file) base_name = os.path.basename(sample_src_file) assert os.path.exists(os.path.join(lb.root_dir, 'code', base_name)) assert new_file_data['key'] == f'{base_name}' assert new_file_data['is_dir'] is False
def test_remove_many_files(self, mock_labbook, sample_src_file): lb = mock_labbook[2] test_files = [f"testfile{x}.txt" for x in range(15)] for test_file in test_files: with open(os.path.join(lb.root_dir, 'code', test_file), 'wt') as sample_f: sample_f.write("blah") assert os.path.exists(os.path.join(lb.root_dir, 'code', test_file)) lb.git.add_all() lb.git.commit("making test data") FO.delete_files(lb, "code", test_files) for test_file in test_files: assert not os.path.exists(os.path.join(lb.root_dir, 'code', test_file))
def test_makedir_simple(self, mock_labbook): # Note that "score" refers to the count of .gitkeep files. lb = mock_labbook[2] long_dir = "code/non/existant/dir/should/now/be/made" dirs = ["code/cat_dir", "code/dog_dir", "code/mouse_dir/", "code/mouse_dir/new_dir", long_dir] for d in dirs: FO.makedir(lb, d) assert os.path.isdir(os.path.join(lb.root_dir, d)) assert os.path.isfile(os.path.join(lb.root_dir, d, '.gitkeep')) score = 0 for root, dirs, files in os.walk(os.path.join(lb.root_dir, 'code', 'non')): for f in files: if f == '.gitkeep': score += 1 # Ensure that count of .gitkeep files equals the number of subdirs, excluding the code dir. assert score == len(LabBook.make_path_relative(long_dir).split(os.sep)) - 1