def remote_labbook_repo():
# TODO: Remove after integration tests with LFS support are available
conf_file, working_dir = _create_temp_work_dir(lfs_enabled=False)
im = InventoryManager(conf_file)
lb = im.create_labbook('test',
'test',
'sample-repo-lb',
description="my first labbook")
bm = BranchManager(lb, username='******')
bm.create_branch('testing-branch')
#with tempfile.TemporaryDirectory() as tmpdirname:
with open(os.path.join('/tmp', 'codefile.c'), 'wb') as codef:
codef.write(b'// Cody McCodeface ...')
FileOperations.insert_file(lb, "code", "/tmp/codefile.c")
assert lb.is_repo_clean
bm.workon_branch('master')
# Location of the repo to push/pull from
yield lb.root_dir
shutil.rmtree(working_dir)
try:
os.remove('/tmp/codefile.c')
except:
pass
def test_sync_1(self, mock_create_labbooks_no_lfs, mock_config_file):
# Setup responses mock for this test
responses.add(responses.GET, 'https://usersrv.gigantum.io/key',
json={'key': 'afaketoken'}, status=200)
im = InventoryManager(mock_create_labbooks_no_lfs[0])
test_user_lb = im.load_labbook('default', 'default', 'labbook1')
test_user_wf = LabbookWorkflow(test_user_lb)
test_user_wf.publish('default')
# Mock the request context so a fake authorization header is present
builder = EnvironBuilder(path='/labbook', method='POST', headers={'Authorization': 'Bearer AJDFHASD'})
env = builder.get_environ()
req = Request(environ=env)
sally_wf = LabbookWorkflow.import_from_remote(test_user_wf.remote, 'sally', config_file=mock_config_file[0])
sally_lb = sally_wf.labbook
FileOperations.makedir(sally_lb, relative_path='code/sally-dir', create_activity_record=True)
sally_wf.sync('sally')
sync_query = """
mutation x {
syncLabbook(input: {
labbookName: "labbook1",
owner: "default"
}) {
jobKey
}
}
"""
r = mock_create_labbooks_no_lfs[2].execute(sync_query, context_value=req)
assert 'errors' not in r
def test_reflect_deleted_files_on_merge_in(self, mock_create_labbooks):
lb, client = mock_create_labbooks[0], mock_create_labbooks[1]
with open('/tmp/s1.txt', 'w') as s1:
s1.write('original-file\ndata')
FileOperations.insert_file(lb, section='code', src_file=s1.name)
bm = BranchManager(lb, username=UT_USERNAME)
nb = bm.create_branch(f'new-branch')
assert os.path.exists(os.path.join(lb.root_dir, 'code', 's1.txt'))
FileOperations.delete_files(lb, 'code', ['s1.txt'])
assert lb.is_repo_clean
assert not os.path.exists(os.path.join(lb.root_dir, 'code', 's1.txt'))
bm.workon_branch(bm.workspace_branch)
assert os.path.exists(os.path.join(lb.root_dir, 'code', 's1.txt'))
merge_q = f"""
mutation x {{
mergeFromBranch(input: {{
owner: "{UT_USERNAME}",
labbookName: "{UT_LBNAME}",
otherBranchName: "{nb}"
}}) {{
labbook{{
name
description
activeBranchName
}}
}}
}}
"""
r = client.execute(merge_q)
assert 'errors' not in r
assert r['data']['mergeFromBranch']['labbook']['activeBranchName'] == 'master'
assert not os.path.exists(os.path.join(lb.root_dir, 'code', 's1.txt'))
def test_insert_and_make_intermediary_directories(self, mock_labbook,
sample_src_file):
lb = mock_labbook[2]
FO.insert_file(lb, "code", sample_src_file,
"/super/random/dir/inside.file")
p = os.path.join(lb.root_dir, 'code', "super/random/dir/inside.file")
assert os.path.isfile(p)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, directory,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with lb.lock():
FileOperations.makedir(lb, os.path.join(section, directory), create_activity_record=True)
# Prime dataloader with labbook you already loaded
dataloader = LabBookLoader()
dataloader.prime(f"{owner}&{labbook_name}&{lb.name}", lb)
# Create data to populate edge
file_info = FileOperations.get_file_info(lb, section, directory)
create_data = {'owner': owner,
'name': labbook_name,
'section': section,
'key': file_info['key'],
'_file_info': file_info}
# TODO: Fix cursor implementation, this currently doesn't make sense
cursor = base64.b64encode(f"{0}".encode('utf-8'))
return MakeLabbookDirectory(
new_labbook_file_edge=LabbookFileConnection.Edge(
node=LabbookFile(**create_data),
cursor=cursor))
def mock_create_labbooks(fixture_working_dir):
# Create a labbook in the temporary directory
config_file = fixture_working_dir[0]
im = InventoryManager(fixture_working_dir[0])
lb = im.create_labbook(UT_USERNAME,
UT_USERNAME,
UT_LBNAME,
description="Cats labbook 1")
# Create a file in the dir
with open(os.path.join(fixture_working_dir[1], 'unittest-examplefile'),
'w') as sf:
sf.write("test data")
sf.seek(0)
FileOperations.insert_file(lb, 'code', sf.name)
assert os.path.isfile(
os.path.join(lb.root_dir, 'code', 'unittest-examplefile'))
# Create test client
schema = graphene.Schema(query=LabbookQuery, mutation=LabbookMutations)
with patch.object(Configuration, 'find_default_config',
lambda self: config_file):
app = Flask("lmsrvlabbook")
app.config["LABMGR_CONFIG"] = Configuration()
app.config["LABMGR_ID_MGR"] = get_identity_manager(Configuration())
with app.app_context():
flask.g.user_obj = app.config["LABMGR_ID_MGR"].get_user_profile()
client = Client(
schema,
middleware=[DataloaderMiddleware(), error_middleware],
context_value=ContextMock())
yield lb, client, schema
shutil.rmtree(fixture_working_dir, ignore_errors=True)
def test_move_loaded_directory_with_one_file(self, mock_labbook, mock_config_file, sample_src_file):
lb = mock_labbook[2]
new_file_data = FO.insert_file(lb, "code", sample_src_file)
base_name = os.path.basename(new_file_data['key'])
assert os.path.exists(os.path.join(lb.root_dir, 'code', base_name))
# make new subdir
os.makedirs(os.path.join(lb.root_dir, 'code', 'subdir'))
# .. and then put a file in it
mv_file_res = FO.move_file(lb, "code", base_name, os.path.join('subdir', base_name))
# Should be 2, because it returns the info of the directory it was moved into
assert len(mv_file_res) == 1
assert mv_file_res[0]['key'] == f'subdir/{base_name}'
assert mv_file_res[0]['is_dir'] == False
# Move "subdir" into "target_dir", there should be two activity records
FO.makedir(lb, "code/target_dir", create_activity_record=True)
mv_dir_res = FO.move_file(lb, "code", 'subdir', 'target_dir')
assert len(mv_dir_res) == 2
assert mv_dir_res[0]['key'] == 'target_dir/subdir/'
assert mv_dir_res[0]['is_dir'] is True
assert mv_dir_res[1]['key'] == f'target_dir/subdir/{base_name}'
assert mv_dir_res[1]['is_dir'] is False
assert not os.path.exists(os.path.join(lb.root_dir, 'code', 'subdir'))
assert os.path.exists(os.path.join(lb.root_dir, 'code', 'target_dir/subdir'))
def test_remove_file_success(self, mock_labbook, sample_src_file):
lb = mock_labbook[2]
new_file_data = FO.insert_file(lb, "code", sample_src_file)
base_name = os.path.basename(new_file_data['key'])
assert os.path.exists(os.path.join(lb.root_dir, 'code', base_name))
FO.delete_files(lb, 'code', [base_name])
assert not os.path.exists(os.path.join(lb.root_dir, 'code', base_name))
def test_listdir(self, mock_labbook, sample_src_file):
def write_test_file(base, name):
with open(os.path.join(base, name), 'wt') as f:
f.write("Blah blah")
lb = mock_labbook[2]
dirs = ["code/new_dir", ".hidden_dir"]
for d in dirs:
FO.makedir(lb, d)
write_test_file(lb.root_dir, 'test1.txt')
write_test_file(lb.root_dir, 'test2.txt')
write_test_file(lb.root_dir, '.hidden.txt')
write_test_file(lb.root_dir, 'code/test_subdir1.txt')
write_test_file(lb.root_dir, 'code/test_subdir2.txt')
write_test_file(lb.root_dir, 'code/new_dir/tester.txt')
# List just the code dir
data = FO.listdir(lb, "code", base_path='')
assert len(data) == 3
assert data[0]['key'] == 'new_dir/'
assert data[1]['key'] == 'test_subdir1.txt'
assert data[2]['key'] == 'test_subdir2.txt'
data = FO.listdir(lb, "input", base_path='')
assert len(data) == 0
# List just the code/subdir dir
data = FO.listdir(lb, "code", base_path='new_dir')
assert len(data) == 1
assert data[0]['key'] == 'new_dir/tester.txt'
def test_insert_file_success_2(self, mock_labbook, sample_src_file):
lb = mock_labbook[2]
FO.makedir(lb, "output/testdir")
new_file_data = FO.insert_file(lb, "output", sample_src_file, "testdir")
base_name = os.path.basename(new_file_data['key'])
assert os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir', base_name))
assert new_file_data['key'] == f'testdir/{base_name}'
assert new_file_data['is_dir'] is False
def test_detail_record_node(self, fixture_working_dir, fixture_test_file):
"""Test getting an detail record by node ID"""
im = InventoryManager(fixture_working_dir[0])
lb = im.create_labbook("default", "default", "labbook1",
description="my test description")
FileOperations.insert_file(lb, "code", fixture_test_file)
# Get activity record to
query = """
{
labbook(name: "labbook1", owner: "default") {
activityRecords {
edges{
node{
id
commit
linkedCommit
message
type
show
importance
tags
detailObjects{
id
key
type
data
show
importance
tags
}
}
}
}
}
}
"""
result1 = fixture_working_dir[2].execute(query)
assert 'errors' not in result1
query = """
{{
node(id: "{}") {{
... on ActivityDetailObject {{
id
key
type
data
show
importance
tags
}}
}}
}}
""".format(result1['data']['labbook']['activityRecords']['edges'][0]['node']['detailObjects'][0]['id'])
result2 = fixture_working_dir[2].execute(query)
assert 'errors' not in result2
assert result2['data']['node'] == result1['data']['labbook']['activityRecords']['edges'][0]['node']['detailObjects'][0]
def test_import_labbook(self, fixture_working_dir):
"""Test batch uploading, but not full import"""
class DummyContext(object):
def __init__(self, file_handle):
self.labbook_loader = None
self.files = {'uploadChunk': file_handle}
client = Client(fixture_working_dir[3], middleware=[DataloaderMiddleware()])
# Create a temporary labbook
lb = InventoryManager(fixture_working_dir[0]).create_labbook("default", "default", "test-export",
description="Tester")
# Create a largeish file in the dir
with open(os.path.join(fixture_working_dir[1], 'testfile.bin'), 'wb') as testfile:
testfile.write(os.urandom(9000000))
FileOperations.insert_file(lb, 'input', testfile.name)
# Export labbook
zip_file = export_labbook_as_zip(lb.root_dir, tempfile.gettempdir())
lb_dir = lb.root_dir
# Get upload params
chunk_size = 4194304
file_info = os.stat(zip_file)
file_size = int(file_info.st_size / 1000)
total_chunks = int(math.ceil(file_info.st_size/chunk_size))
with open(zip_file, 'rb') as tf:
for chunk_index in range(total_chunks):
chunk = io.BytesIO()
chunk.write(tf.read(chunk_size))
chunk.seek(0)
file = FileStorage(chunk)
query = f"""
mutation myMutation{{
importLabbook(input:{{
chunkUploadParams:{{
uploadId: "jfdjfdjdisdjwdoijwlkfjd",
chunkSize: {chunk_size},
totalChunks: {total_chunks},
chunkIndex: {chunk_index},
fileSize: "{file_size}",
filename: "{os.path.basename(zip_file)}"
}}
}}) {{
importJobKey
}}
}}
"""
result = client.execute(query, context_value=DummyContext(file))
assert "errors" not in result
if chunk_index == total_chunks - 1:
assert type(result['data']['importLabbook']['importJobKey']) == str
assert "rq:job:" in result['data']['importLabbook']['importJobKey']
chunk.close()
def test_remove_file_fail_old_prototype(self, mock_labbook, sample_src_file):
lb = mock_labbook[2]
new_file_data = FO.insert_file(lb, "code", sample_src_file)
base_name = os.path.basename(new_file_data['key'])
assert os.path.exists(os.path.join(lb.root_dir, 'code', base_name))
with pytest.raises(ValueError):
FO.delete_files(lb, 'code', base_name)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, file_paths, client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with lb.lock():
FileOperations.delete_files(lb, section, file_paths)
return DeleteLabbookFiles(success=True)
def test_move_single_file(self, mock_labbook, mock_config_file, sample_src_file):
lb = mock_labbook[2]
f = FO.insert_file(lb, 'code', sample_src_file)['key']
FO.makedir(lb, 'code/target_dir')
results = FO.move_file(lb, 'code', f, 'target_dir')
assert len(results) == 1
pprint.pprint(results)
assert results[0]['is_dir'] == False
assert results[0]['key'] == 'target_dir/' + os.path.basename(sample_src_file)
def mutate_and_get_payload(cls, root, info, owner, labbook_name,
transaction_id, cancel=False, rollback=False,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with lb.lock():
FileOperations.complete_batch(lb, transaction_id, cancel=cancel,
rollback=rollback)
return CompleteBatchUploadTransaction(success=True)
def test_move_empty_directory(self, mock_labbook, mock_config_file, sample_src_file):
lb = mock_labbook[2]
FO.makedir(lb, 'code/stable_dir')
FO.makedir(lb, 'code/empty_dir')
# We'll move "empty_dir" into "stable_dir" - there should only be one element in returned list
res = FO.move_file(lb, 'code', 'empty_dir', 'stable_dir')
assert len(res) == 1
assert res[0]['is_dir'] is True
assert res[0]['key'] == 'stable_dir/empty_dir/'
def test_move_single_file_to_section_top(self, mock_labbook, mock_config_file, sample_src_file):
lb = mock_labbook[2]
FO.makedir(lb, 'code/inner_dir')
f = FO.insert_file(lb, 'code', sample_src_file, 'inner_dir')['key']
# Move file to top of code section
results = FO.move_file(lb, 'code', f, dst_rel_path='')
# Results should be returned for "code" -- the file just moved there and the
assert len(results) == 1
assert results[0]['is_dir'] == False
assert results[0]['key'] == os.path.basename(f)
def test_remove_dir(self, mock_labbook, sample_src_file):
lb = mock_labbook[2]
FO.makedir(lb, "output/testdir")
new_file_path = FO.insert_file(lb, "output", sample_src_file, "testdir")
base_name = os.path.basename(new_file_path['key'])
assert os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir', base_name))
# Note! Now that remove() uses force=True, no special action is needed for directories.
# Delete the directory
FO.delete_files(lb, "output", ["testdir"])
assert not os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir', base_name))
assert not os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir'))
def test_makedir_record(self, mock_labbook):
# Note that "score" refers to the count of .gitkeep files.
lb = mock_labbook[2]
assert os.path.exists(os.path.join(lb.root_dir, 'code', 'test')) is False
FO.makedir(lb, "code/test", create_activity_record=True)
assert os.path.exists(os.path.join(lb.root_dir, 'code', 'test')) is True
assert lb.is_repo_clean is True
FO.makedir(lb, "code/test2", create_activity_record=False)
assert os.path.exists(os.path.join(lb.root_dir, 'code', 'test2')) is True
assert lb.is_repo_clean is False
def test_remove_empty_dir(self, mock_labbook, sample_src_file):
lb = mock_labbook[2]
FO.makedir(lb, "output/testdir")
new_file_path = FO.insert_file(lb, "output", sample_src_file, "testdir")
base_name = os.path.basename(new_file_path['key'])
assert os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir', base_name))
# Delete the directory
FO.delete_files(lb, "output", ["testdir"])
assert not os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir', base_name))
assert not os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir'))
def test_merge_into_feature_from_workspace_simple_success(
self, mock_create_labbooks):
lb, client = mock_create_labbooks[0], mock_create_labbooks[1]
bm = BranchManager(lb, username=UT_USERNAME)
og_hash = lb.git.commit_hash
b1 = bm.create_branch(f"test-branch")
bm.workon_branch(bm.workspace_branch)
assert lb.active_branch == bm.workspace_branch
og2_hash = lb.git.commit_hash
assert lb.git.commit_hash == og_hash
FileOperations.makedir(lb,
'code/main-branch-dir1',
create_activity_record=True)
FileOperations.makedir(lb,
'code/main-branch-dir2',
create_activity_record=True)
next_main_hash = lb.git.commit_hash
assert og_hash != next_main_hash
bm.workon_branch(b1)
assert not os.path.exists(
os.path.join(lb.root_dir, 'code/main-branch-dir1'))
merge_q = f"""
mutation x {{
mergeFromBranch(input: {{
owner: "{UT_USERNAME}",
labbookName: "{UT_LBNAME}",
otherBranchName: "{bm.workspace_branch}"
}}) {{
labbook{{
name
description
activeBranchName
}}
}}
}}
"""
r = client.execute(merge_q)
assert 'errors' not in r
assert r['data']['mergeFromBranch']['labbook'][
'activeBranchName'] == 'test-branch'
assert r['data']['mergeFromBranch']['labbook'][
'name'] == 'unittest-workflow-branch-1'
assert lb.active_branch == b1
assert os.path.exists(
os.path.join(lb.root_dir, 'code/main-branch-dir1'))
assert lb.is_repo_clean
def mock_create_labbooks(fixture_working_dir):
# Create a labbook in the temporary directory
lb = InventoryManager(fixture_working_dir[0]).create_labbook(
"default", "default", "sample-repo-lb", description="Cats labbook 1")
# Create a file in the dir
with open(os.path.join(fixture_working_dir[1], 'codefile.c'), 'w') as sf:
sf.write("1234567")
sf.seek(0)
FileOperations.insert_file(lb, 'code', sf.name)
assert os.path.isfile(os.path.join(lb.root_dir, 'code', 'codefile.c'))
# name of the config file, temporary working directory, the schema
yield fixture_working_dir, lb
def test_move_file_as_rename_in_same_dir(self, mock_labbook, sample_src_file):
lb = mock_labbook[2]
# insert file
new_file_data = FO.insert_file(lb, "code", sample_src_file, '')
base_name = os.path.basename(new_file_data['key'])
assert os.path.exists(os.path.join(lb.root_dir, 'code', base_name))
assert new_file_data['key'] == base_name
# move to rename
moved_rel_path = os.path.join(f'{base_name}.MOVED')
r = FO.move_file(lb, 'code', new_file_data['key'], moved_rel_path)
assert len(r) == 1
assert not os.path.exists(os.path.join(lb.root_dir, 'code', base_name))
assert os.path.exists(os.path.join(lb.root_dir, 'code', f'{base_name}.MOVED'))
assert os.path.isfile(os.path.join(lb.root_dir, 'code', f'{base_name}.MOVED'))
def mock_create_labbooks_no_lfs(fixture_working_dir_lfs_disabled):
# Create a labbook in the temporary directory
im = InventoryManager(fixture_working_dir_lfs_disabled[0])
lb = im.create_labbook("default", "default", "labbook1",
description="Cats labbook 1")
# Create a file in the dir
with open(os.path.join(fixture_working_dir_lfs_disabled[1], 'sillyfile'), 'w') as sf:
sf.write("1234567")
sf.seek(0)
FileOperations.insert_file(lb, 'code', sf.name)
assert os.path.isfile(os.path.join(lb.root_dir, 'code', 'sillyfile'))
# name of the config file, temporary working directory, the schema
yield fixture_working_dir_lfs_disabled
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, src_path, dst_path,
client_mutation_id=None, **kwargs):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with lb.lock():
mv_results = FileOperations.move_file(lb, section, src_path, dst_path)
file_edges = list()
for file_dict in mv_results:
file_edges.append(LabbookFile(owner=owner,
name=labbook_name,
section=section,
key=file_dict['key'],
is_dir=file_dict['is_dir'],
is_favorite=file_dict['is_favorite'],
modified_at=file_dict['modified_at'],
size=str(file_dict['size'])))
cursors = [base64.b64encode("{}".format(cnt).encode("UTF-8")).decode("UTF-8")
for cnt, x in enumerate(file_edges)]
edge_objs = [LabbookFileConnection.Edge(node=e, cursor=c) for e, c in zip(file_edges, cursors)]
return MoveLabbookFile(updated_edges=edge_objs)
def test_labbook_content_size_simply(self, mock_labbook):
x, y, lb = mock_labbook
lb_size = FO.content_size(lb)
# Make sure the new LB is about 10-30kB. This is about reasonable for a new, emtpy LB.
assert lb_size > 10000
assert lb_size < 30000
def test_insert_file_success_1(self, mock_labbook, sample_src_file):
lb = mock_labbook[2]
new_file_data = FO.insert_file(lb, "code", sample_src_file)
base_name = os.path.basename(sample_src_file)
assert os.path.exists(os.path.join(lb.root_dir, 'code', base_name))
assert new_file_data['key'] == f'{base_name}'
assert new_file_data['is_dir'] is False
def test_remove_many_files(self, mock_labbook, sample_src_file):
lb = mock_labbook[2]
test_files = [f"testfile{x}.txt" for x in range(15)]
for test_file in test_files:
with open(os.path.join(lb.root_dir, 'code', test_file), 'wt') as sample_f:
sample_f.write("blah")
assert os.path.exists(os.path.join(lb.root_dir, 'code', test_file))
lb.git.add_all()
lb.git.commit("making test data")
FO.delete_files(lb, "code", test_files)
for test_file in test_files:
assert not os.path.exists(os.path.join(lb.root_dir, 'code', test_file))
def test_makedir_simple(self, mock_labbook):
# Note that "score" refers to the count of .gitkeep files.
lb = mock_labbook[2]
long_dir = "code/non/existant/dir/should/now/be/made"
dirs = ["code/cat_dir", "code/dog_dir", "code/mouse_dir/", "code/mouse_dir/new_dir", long_dir]
for d in dirs:
FO.makedir(lb, d)
assert os.path.isdir(os.path.join(lb.root_dir, d))
assert os.path.isfile(os.path.join(lb.root_dir, d, '.gitkeep'))
score = 0
for root, dirs, files in os.walk(os.path.join(lb.root_dir, 'code', 'non')):
for f in files:
if f == '.gitkeep':
score += 1
# Ensure that count of .gitkeep files equals the number of subdirs, excluding the code dir.
assert score == len(LabBook.make_path_relative(long_dir).split(os.sep)) - 1
