def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, directory,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with lb.lock():
FileOperations.makedir(lb, os.path.join(section, directory), create_activity_record=True)
# Prime dataloader with labbook you already loaded
dataloader = LabBookLoader()
dataloader.prime(f"{owner}&{labbook_name}&{lb.name}", lb)
# Create data to populate edge
file_info = FileOperations.get_file_info(lb, section, directory)
create_data = {'owner': owner,
'name': labbook_name,
'section': section,
'key': file_info['key'],
'_file_info': file_info}
# TODO: Fix cursor implementation, this currently doesn't make sense
cursor = base64.b64encode(f"{0}".encode('utf-8'))
return MakeLabbookDirectory(
new_labbook_file_edge=LabbookFileConnection.Edge(
node=LabbookFile(**create_data),
cursor=cursor))
def test_listdir(self, mock_labbook, sample_src_file):
def write_test_file(base, name):
with open(os.path.join(base, name), 'wt') as f:
f.write("Blah blah")
lb = mock_labbook[2]
dirs = ["code/new_dir", ".hidden_dir"]
for d in dirs:
FO.makedir(lb, d)
write_test_file(lb.root_dir, 'test1.txt')
write_test_file(lb.root_dir, 'test2.txt')
write_test_file(lb.root_dir, '.hidden.txt')
write_test_file(lb.root_dir, 'code/test_subdir1.txt')
write_test_file(lb.root_dir, 'code/test_subdir2.txt')
write_test_file(lb.root_dir, 'code/new_dir/tester.txt')
# List just the code dir
data = FO.listdir(lb, "code", base_path='')
assert len(data) == 3
assert data[0]['key'] == 'new_dir/'
assert data[1]['key'] == 'test_subdir1.txt'
assert data[2]['key'] == 'test_subdir2.txt'
data = FO.listdir(lb, "input", base_path='')
assert len(data) == 0
# List just the code/subdir dir
data = FO.listdir(lb, "code", base_path='new_dir')
assert len(data) == 1
assert data[0]['key'] == 'new_dir/tester.txt'
def test_sync_1(self, mock_create_labbooks_no_lfs, mock_config_file):
# Setup responses mock for this test
responses.add(responses.GET, 'https://usersrv.gigantum.io/key',
json={'key': 'afaketoken'}, status=200)
im = InventoryManager(mock_create_labbooks_no_lfs[0])
test_user_lb = im.load_labbook('default', 'default', 'labbook1')
test_user_wf = LabbookWorkflow(test_user_lb)
test_user_wf.publish('default')
# Mock the request context so a fake authorization header is present
builder = EnvironBuilder(path='/labbook', method='POST', headers={'Authorization': 'Bearer AJDFHASD'})
env = builder.get_environ()
req = Request(environ=env)
sally_wf = LabbookWorkflow.import_from_remote(test_user_wf.remote, 'sally', config_file=mock_config_file[0])
sally_lb = sally_wf.labbook
FileOperations.makedir(sally_lb, relative_path='code/sally-dir', create_activity_record=True)
sally_wf.sync('sally')
sync_query = """
mutation x {
syncLabbook(input: {
labbookName: "labbook1",
owner: "default"
}) {
jobKey
}
}
"""
r = mock_create_labbooks_no_lfs[2].execute(sync_query, context_value=req)
assert 'errors' not in r
def test_move_loaded_directory_with_one_file(self, mock_labbook, mock_config_file, sample_src_file):
lb = mock_labbook[2]
new_file_data = FO.insert_file(lb, "code", sample_src_file)
base_name = os.path.basename(new_file_data['key'])
assert os.path.exists(os.path.join(lb.root_dir, 'code', base_name))
# make new subdir
os.makedirs(os.path.join(lb.root_dir, 'code', 'subdir'))
# .. and then put a file in it
mv_file_res = FO.move_file(lb, "code", base_name, os.path.join('subdir', base_name))
# Should be 2, because it returns the info of the directory it was moved into
assert len(mv_file_res) == 1
assert mv_file_res[0]['key'] == f'subdir/{base_name}'
assert mv_file_res[0]['is_dir'] == False
# Move "subdir" into "target_dir", there should be two activity records
FO.makedir(lb, "code/target_dir", create_activity_record=True)
mv_dir_res = FO.move_file(lb, "code", 'subdir', 'target_dir')
assert len(mv_dir_res) == 2
assert mv_dir_res[0]['key'] == 'target_dir/subdir/'
assert mv_dir_res[0]['is_dir'] is True
assert mv_dir_res[1]['key'] == f'target_dir/subdir/{base_name}'
assert mv_dir_res[1]['is_dir'] is False
assert not os.path.exists(os.path.join(lb.root_dir, 'code', 'subdir'))
assert os.path.exists(os.path.join(lb.root_dir, 'code', 'target_dir/subdir'))
def test_insert_file_success_2(self, mock_labbook, sample_src_file):
lb = mock_labbook[2]
FO.makedir(lb, "output/testdir")
new_file_data = FO.insert_file(lb, "output", sample_src_file, "testdir")
base_name = os.path.basename(new_file_data['key'])
assert os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir', base_name))
assert new_file_data['key'] == f'testdir/{base_name}'
assert new_file_data['is_dir'] is False
def test_move_single_file(self, mock_labbook, mock_config_file, sample_src_file):
lb = mock_labbook[2]
f = FO.insert_file(lb, 'code', sample_src_file)['key']
FO.makedir(lb, 'code/target_dir')
results = FO.move_file(lb, 'code', f, 'target_dir')
assert len(results) == 1
pprint.pprint(results)
assert results[0]['is_dir'] == False
assert results[0]['key'] == 'target_dir/' + os.path.basename(sample_src_file)
def test_move_single_file_to_section_top(self, mock_labbook, mock_config_file, sample_src_file):
lb = mock_labbook[2]
FO.makedir(lb, 'code/inner_dir')
f = FO.insert_file(lb, 'code', sample_src_file, 'inner_dir')['key']
# Move file to top of code section
results = FO.move_file(lb, 'code', f, dst_rel_path='')
# Results should be returned for "code" -- the file just moved there and the
assert len(results) == 1
assert results[0]['is_dir'] == False
assert results[0]['key'] == os.path.basename(f)
def test_move_empty_directory(self, mock_labbook, mock_config_file, sample_src_file):
lb = mock_labbook[2]
FO.makedir(lb, 'code/stable_dir')
FO.makedir(lb, 'code/empty_dir')
# We'll move "empty_dir" into "stable_dir" - there should only be one element in returned list
res = FO.move_file(lb, 'code', 'empty_dir', 'stable_dir')
assert len(res) == 1
assert res[0]['is_dir'] is True
assert res[0]['key'] == 'stable_dir/empty_dir/'
def test_makedir_record(self, mock_labbook):
# Note that "score" refers to the count of .gitkeep files.
lb = mock_labbook[2]
assert os.path.exists(os.path.join(lb.root_dir, 'code', 'test')) is False
FO.makedir(lb, "code/test", create_activity_record=True)
assert os.path.exists(os.path.join(lb.root_dir, 'code', 'test')) is True
assert lb.is_repo_clean is True
FO.makedir(lb, "code/test2", create_activity_record=False)
assert os.path.exists(os.path.join(lb.root_dir, 'code', 'test2')) is True
assert lb.is_repo_clean is False
def test_move_loaded_directory_with_full_tree(self, mock_labbook, mock_config_file, sample_src_file):
lb = mock_labbook[2]
FO.makedir(lb, 'code/level_1/level_2A', create_activity_record=True)
FO.makedir(lb, 'code/level_1/level_2B', create_activity_record=True)
FO.makedir(lb, 'code/target_dir', create_activity_record=True)
FO.makedir(lb, 'code/target_dir/existing_dir_counted_anyway', create_activity_record=True)
FO.makedir(lb, 'code/this-dir-must-be-ignored', create_activity_record=True)
FO.insert_file(lb, 'code', sample_src_file, dst_path='level_1/level_2B')
# Move "level_1" into target_dir
results = FO.move_file(lb, 'code', 'level_1', 'target_dir')
assert len(results) == 4
def test_remove_empty_dir(self, mock_labbook, sample_src_file):
lb = mock_labbook[2]
FO.makedir(lb, "output/testdir")
new_file_path = FO.insert_file(lb, "output", sample_src_file, "testdir")
base_name = os.path.basename(new_file_path['key'])
assert os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir', base_name))
# Delete the directory
FO.delete_files(lb, "output", ["testdir"])
assert not os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir', base_name))
assert not os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir'))
def test_remove_dir(self, mock_labbook, sample_src_file):
lb = mock_labbook[2]
FO.makedir(lb, "output/testdir")
new_file_path = FO.insert_file(lb, "output", sample_src_file, "testdir")
base_name = os.path.basename(new_file_path['key'])
assert os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir', base_name))
# Note! Now that remove() uses force=True, no special action is needed for directories.
# Delete the directory
FO.delete_files(lb, "output", ["testdir"])
assert not os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir', base_name))
assert not os.path.exists(os.path.join(lb.root_dir, 'output', 'testdir'))
def test_merge_into_feature_from_workspace_simple_success(
self, mock_create_labbooks):
lb, client = mock_create_labbooks[0], mock_create_labbooks[1]
bm = BranchManager(lb, username=UT_USERNAME)
og_hash = lb.git.commit_hash
b1 = bm.create_branch(f"test-branch")
bm.workon_branch(bm.workspace_branch)
assert lb.active_branch == bm.workspace_branch
og2_hash = lb.git.commit_hash
assert lb.git.commit_hash == og_hash
FileOperations.makedir(lb,
'code/main-branch-dir1',
create_activity_record=True)
FileOperations.makedir(lb,
'code/main-branch-dir2',
create_activity_record=True)
next_main_hash = lb.git.commit_hash
assert og_hash != next_main_hash
bm.workon_branch(b1)
assert not os.path.exists(
os.path.join(lb.root_dir, 'code/main-branch-dir1'))
merge_q = f"""
mutation x {{
mergeFromBranch(input: {{
owner: "{UT_USERNAME}",
labbookName: "{UT_LBNAME}",
otherBranchName: "{bm.workspace_branch}"
}}) {{
labbook{{
name
description
activeBranchName
}}
}}
}}
"""
r = client.execute(merge_q)
assert 'errors' not in r
assert r['data']['mergeFromBranch']['labbook'][
'activeBranchName'] == 'test-branch'
assert r['data']['mergeFromBranch']['labbook'][
'name'] == 'unittest-workflow-branch-1'
assert lb.active_branch == b1
assert os.path.exists(
os.path.join(lb.root_dir, 'code/main-branch-dir1'))
assert lb.is_repo_clean
def test_walkdir_with_favorites(self, mock_labbook, sample_src_file):
lb = mock_labbook[2]
dirs = ["code/cat_dir", "code/dog_dir"]
for d in dirs:
FO.makedir(lb, d)
sfile = '/tmp/testwalkdirwithfavorites.file'
for d in ['', 'dog_dir', 'cat_dir']:
open(sfile, 'w').write('xxx')
FO.insert_file(lb, 'code', sfile, d)
sample_filename = os.path.basename(sfile)
# Since the file is in a hidden directory, it should not be found.
dir_walks = FO.walkdir(lb, 'code')
# Spot check some entries
assert len(dir_walks) == 5
assert dir_walks[0]['key'] == 'cat_dir/'
assert dir_walks[0]['is_dir'] is True
assert dir_walks[0]['is_favorite'] is False
assert dir_walks[1]['key'] == 'dog_dir/'
assert dir_walks[1]['is_dir'] is True
assert dir_walks[1]['is_favorite'] is False
assert dir_walks[2]['is_favorite'] is False
assert dir_walks[2]['is_dir'] is False
assert dir_walks[3]['is_favorite'] is False
assert dir_walks[3]['is_dir'] is False
assert dir_walks[4]['is_favorite'] is False
assert dir_walks[4]['is_dir'] is False
lb.create_favorite("code", sample_filename, description="Fav 1")
lb.create_favorite("code", f"dog_dir/{sample_filename}", description="Fav 2")
lb.create_favorite("code", f"cat_dir/", description="Fav 3", is_dir=True)
dir_walks = FO.walkdir(lb, 'code')
# Spot check some entries
assert len(dir_walks) == 5
assert dir_walks[0]['key'] == 'cat_dir/'
assert dir_walks[0]['is_dir'] is True
assert dir_walks[0]['is_favorite'] is True
assert dir_walks[1]['key'] == 'dog_dir/'
assert dir_walks[1]['is_dir'] is True
assert dir_walks[1]['is_favorite'] is False
assert dir_walks[2]['is_favorite'] is True
assert dir_walks[2]['is_dir'] is False
assert dir_walks[3]['is_favorite'] is False
assert dir_walks[3]['is_dir'] is False
assert dir_walks[4]['is_favorite'] is True
assert dir_walks[4]['is_dir'] is False
def test_walkdir(self, mock_labbook):
lb = mock_labbook[2]
dirs = [
"code/cat_dir", "code/dog_dir", "code/mouse_dir/",
"code/mouse_dir/new_dir", "code/.hidden_dir"
]
for d in dirs:
FO.makedir(lb, d)
for d in ['.hidden_dir/', '', 'dog_dir', 'mouse_dir/new_dir/']:
open('/tmp/myfile.c', 'w').write('data')
FO.insert_file(lb, 'code', '/tmp/myfile.c', d)
dir_walks_hidden = FO.walkdir(lb, 'code', show_hidden=True)
assert any([
os.path.basename('/tmp/myfile.c') in d['key']
for d in dir_walks_hidden
])
assert not any(
['.git' in d['key'].split(os.path.sep) for d in dir_walks_hidden])
assert not any(['.gigantum' in d['key'] for d in dir_walks_hidden])
assert all([d['key'][0] != '/' for d in dir_walks_hidden])
# Spot check some entries
assert len(dir_walks_hidden) == 15
assert dir_walks_hidden[0]['key'] == '.hidden_dir/'
assert dir_walks_hidden[0]['is_dir'] is True
assert dir_walks_hidden[3]['key'] == 'mouse_dir/'
assert dir_walks_hidden[3]['is_dir'] is True
assert dir_walks_hidden[6]['key'] == '.hidden_dir/.gitkeep'
assert dir_walks_hidden[6]['is_dir'] is False
assert dir_walks_hidden[13]['key'] == 'mouse_dir/new_dir/.gitkeep'
assert dir_walks_hidden[13]['is_dir'] is False
# Since the file is in a hidden directory, it should not be found.
dir_walks = FO.walkdir(lb, 'code')
# Spot check some entries
assert len(dir_walks) == 7
assert dir_walks[0]['key'] == 'cat_dir/'
assert dir_walks[0]['is_dir'] is True
assert dir_walks[1]['key'] == 'dog_dir/'
assert dir_walks[1]['is_dir'] is True
assert dir_walks[2]['key'] == 'mouse_dir/'
assert dir_walks[2]['is_dir'] is True
assert dir_walks[3]['is_dir'] is False
assert dir_walks[4]['is_dir'] is False
assert dir_walks[5]['is_dir'] is True
assert dir_walks[5]['key'] == 'mouse_dir/new_dir/'
assert dir_walks[6]['is_dir'] is False
def test_makedir_simple(self, mock_labbook):
# Note that "score" refers to the count of .gitkeep files.
lb = mock_labbook[2]
long_dir = "code/non/existant/dir/should/now/be/made"
dirs = ["code/cat_dir", "code/dog_dir", "code/mouse_dir/", "code/mouse_dir/new_dir", long_dir]
for d in dirs:
FO.makedir(lb, d)
assert os.path.isdir(os.path.join(lb.root_dir, d))
assert os.path.isfile(os.path.join(lb.root_dir, d, '.gitkeep'))
score = 0
for root, dirs, files in os.walk(os.path.join(lb.root_dir, 'code', 'non')):
for f in files:
if f == '.gitkeep':
score += 1
# Ensure that count of .gitkeep files equals the number of subdirs, excluding the code dir.
assert score == len(LabBook.make_path_relative(long_dir).split(os.sep)) - 1
def test_get_commits_with_local_changes(self, mock_config_file, remote_labbook_repo,
mock_labbook_lfs_disabled):
# When the branch is up to date, ensure it doesn't report being behind.
lb = mock_labbook_lfs_disabled[2]
lb.add_remote("origin", remote_labbook_repo)
bm = BranchManager(lb, username='******')
bm.workon_branch("testing-branch")
# Do some stuff to make commits locally
FileOperations.makedir(lb, 'code/rand_dir', create_activity_record=True)
FileOperations.delete_files(lb, 'code', ['rand_dir'])
behind = bm.get_commits_behind()
ahead = bm.get_commits_ahead()
assert ahead == 4
assert behind == 0
def test_get_commits_with_remote_changes(self, mock_config_file,
remote_labbook_repo,
mock_labbook_lfs_disabled):
# When the branch is up to date, ensure it doesn't report being behind.
lb = mock_labbook_lfs_disabled[2]
lb.add_remote("origin", remote_labbook_repo)
bm = BranchManager(lb, username='******')
bm.workon_branch("testing-branch")
from gtmcore.inventory.inventory import InventoryManager
remote_lb = InventoryManager(mock_config_file[0]).load_labbook_from_directory(remote_labbook_repo)
remote_bm = BranchManager(remote_lb, 'test')
remote_bm.workon_branch("testing-branch")
FileOperations.makedir(remote_lb, 'code/xyzdir', create_activity_record=True)
bm.fetch()
behind = bm.get_commits_behind()
ahead = bm.get_commits_ahead()
assert ahead == 0
assert behind == 2
def test_success_merge_from(self, mock_labbook_lfs_disabled):
""" Test merging with nonconflicting changes. """
lb = mock_labbook_lfs_disabled[2]
bm = BranchManager(lb, username=TEST_USER)
t = 'my-new-example-feature'
feature_branch_name = bm.create_branch(title=t)
assert bm.active_branch == feature_branch_name
bm.workon_branch(bm.workspace_branch)
FileOperations.makedir(lb,
'code/sillyfolder',
create_activity_record=True)
FileOperations.makedir(lb,
'input/newfolder',
create_activity_record=True)
bm.workon_branch(feature_branch_name)
FileOperations.makedir(lb,
'output/otherdir',
create_activity_record=True)
bm.merge_from(bm.workspace_branch)
# Assert repo state is as we expect
assert os.path.isdir(os.path.join(lb.root_dir, 'code/sillyfolder'))
assert os.path.isdir(os.path.join(lb.root_dir, 'input/newfolder'))
assert os.path.isdir(os.path.join(lb.root_dir, 'output/otherdir'))
assert lb.is_repo_clean
# Return to original branch and check proper state
bm.workon_branch(bm.workspace_branch)
assert os.path.isdir(os.path.join(lb.root_dir, 'code/sillyfolder'))
assert os.path.isdir(os.path.join(lb.root_dir, 'input/newfolder'))
assert not os.path.isdir(os.path.join(lb.root_dir, 'output/otherdir'))
assert lb.is_repo_clean
def test_delete_dir(self, mock_create_labbooks):
im = InventoryManager(mock_create_labbooks[0])
lb = im.load_labbook('default', 'default', 'labbook1')
FileOperations.makedir(lb, 'code/subdir')
test_file = os.path.join(lb.root_dir, 'code', 'subdir', 'test.txt')
with open(test_file, 'wt') as tf:
tf.write("puppers")
lb.git.add_all('code/')
lb.git.commit("blah")
dir_path = os.path.join(lb.root_dir, 'code', 'subdir')
assert os.path.exists(dir_path) is True
assert os.path.exists(test_file) is True
# Note, deleting a file should work with and without a trailing / at the end.
query = """
mutation deleteLabbookFiles {
deleteLabbookFiles(
input: {
owner: "default",
labbookName: "labbook1",
section: "code",
filePaths: ["subdir/"]
}) {
success
}
}
"""
res = mock_create_labbooks[2].execute(query)
print(res)
assert res['data']['deleteLabbookFiles']['success'] is True
assert os.path.exists(dir_path) is False
assert os.path.exists(test_file) is False
assert os.path.exists(os.path.join(lb.root_dir, 'code')) is True
def test_add_file_fail_due_to_git_ignore(self, mock_create_labbooks):
"""Test adding a new file to a labbook"""
class DummyContext(object):
def __init__(self, file_handle):
self.labbook_loader = None
self.files = {'uploadChunk': file_handle}
client = Client(mock_create_labbooks[3],
middleware=[DataloaderMiddleware()])
new_file_size = 9000000
# Create file to upload
test_file = os.path.join(tempfile.gettempdir(), ".DS_Store")
with open(test_file, 'wb') as tf:
tf.write(os.urandom(new_file_size))
# Get upload params
chunk_size = 4194000
file_info = os.stat(test_file)
file_size = int(file_info.st_size / 1000)
total_chunks = int(math.ceil(file_info.st_size / chunk_size))
target_file = os.path.join(mock_create_labbooks[1], 'default',
'default', 'labbooks', 'labbook1', 'code',
'newdir', '.DS_Store')
try:
os.remove(target_file)
except:
pass
lb = InventoryManager(mock_create_labbooks[0]).load_labbook(
'default', 'default', 'labbook1')
FileOperations.makedir(lb, 'code/newdir', create_activity_record=True)
with open(test_file, 'rb') as tf:
# Check for file to exist (shouldn't yet)
assert os.path.exists(target_file) is False
for chunk_index in range(total_chunks):
# Upload a chunk
chunk = io.BytesIO()
chunk.write(tf.read(chunk_size))
chunk.seek(0)
file = FileStorage(chunk)
query = f"""
mutation addLabbookFile{{
addLabbookFile(input:{{owner:"default",
labbookName: "labbook1",
section: "code",
filePath: "newdir/.DS_Store",
transactionId: "111-unittest-tx",
chunkUploadParams:{{
uploadId: "jfdjfdjdisdjwdoijwlkfjd",
chunkSize: {chunk_size},
totalChunks: {total_chunks},
chunkIndex: {chunk_index},
fileSizeKb: {file_size},
filename: "{os.path.basename(test_file)}"
}}
}}) {{
newLabbookFileEdge {{
node{{
id
key
isDir
size
}}
}}
}}
}}
"""
r = client.execute(query, context_value=DummyContext(file))
# This must be outside of the chunk upload loop
pprint.pprint(r)
assert 'errors' in r
assert len(r['errors']) == 1
assert 'matches ignored pattern' in r['errors'][0]['message']
# When done uploading, file should exist in the labbook
assert os.path.isfile(target_file) is False
assert os.path.exists(target_file) is False
def test_add_file(self, mock_create_labbooks):
"""Test adding a new file to a labbook"""
class DummyContext(object):
def __init__(self, file_handle):
self.labbook_loader = None
self.files = {'uploadChunk': file_handle}
client = Client(mock_create_labbooks[3],
middleware=[DataloaderMiddleware()])
# Create file to upload
test_file = os.path.join(tempfile.gettempdir(), "myValidFile.dat")
est_size = 9000000
try:
os.remove(test_file)
except:
pass
with open(test_file, 'wb') as tf:
tf.write(os.urandom(est_size))
new_file_size = os.path.getsize(tf.name)
# Get upload params
chunk_size = 4194000
file_info = os.stat(test_file)
file_size = int(file_info.st_size / 1000)
total_chunks = int(math.ceil(file_info.st_size / chunk_size))
target_file = os.path.join(mock_create_labbooks[1], 'default',
'default', 'labbooks', 'labbook1', 'code',
'newdir', "myValidFile.dat")
lb = InventoryManager(mock_create_labbooks[0]).load_labbook(
'default', 'default', 'labbook1')
FileOperations.makedir(lb, 'code/newdir', create_activity_record=True)
txid = "000-unitest-transaction"
with open(test_file, 'rb') as tf:
# Check for file to exist (shouldn't yet)
assert os.path.exists(target_file) is False
for chunk_index in range(total_chunks):
# Upload a chunk
chunk = io.BytesIO()
chunk.write(tf.read(chunk_size))
chunk.seek(0)
file = FileStorage(chunk)
query = f"""
mutation addLabbookFile{{
addLabbookFile(input:{{owner:"default",
labbookName: "labbook1",
section: "code",
filePath: "newdir/myValidFile.dat",
transactionId: "{txid}",
chunkUploadParams:{{
uploadId: "fdsfdsfdsfdfs",
chunkSize: {chunk_size},
totalChunks: {total_chunks},
chunkIndex: {chunk_index},
fileSizeKb: {file_size},
filename: "{os.path.basename(test_file)}"
}}
}}) {{
newLabbookFileEdge {{
node{{
id
key
isDir
size
modifiedAt
}}
}}
}}
}}
"""
r = client.execute(query, context_value=DummyContext(file))
assert 'errors' not in r
# So, these will only be populated once the last chunk is uploaded. Will be None otherwise.
assert r['data']['addLabbookFile']['newLabbookFileEdge']['node'][
'isDir'] is False
assert r['data']['addLabbookFile']['newLabbookFileEdge']['node'][
'key'] == 'newdir/myValidFile.dat'
assert r['data']['addLabbookFile']['newLabbookFileEdge']['node'][
'size'] == f"{new_file_size}"
assert isinstance(
r['data']['addLabbookFile']['newLabbookFileEdge']['node']
['modifiedAt'], float)
assert r['data']['addLabbookFile']['newLabbookFileEdge']['node']['modifiedAt'] != \
round(r['data']['addLabbookFile']['newLabbookFileEdge']['node']['modifiedAt'])
# When done uploading, file should exist in the labbook
assert os.path.exists(target_file)
assert os.path.isfile(target_file)
complete_query = f"""
mutation completeQuery {{
completeBatchUploadTransaction(input: {{
owner: "default",
labbookName: "labbook1",
transactionId: "{txid}"
}}) {{
success
}}
}}
"""
r = client.execute(complete_query, context_value=DummyContext(file))
assert 'errors' not in r
assert lb.is_repo_clean
assert 'Uploaded 1 new file(s)' in lb.git.log()[0]['message']
def test_success_rollback_basic(self, mock_labbook_lfs_disabled):
""" Basic test of rollback feature - making a branch from """
test_user_lb = mock_labbook_lfs_disabled[2]
# Create a directory and capture that Git revision (to be used as basis for rollback).
FileOperations.makedir(test_user_lb,
relative_path='code/folder1',
create_activity_record=True)
commit = test_user_lb.git.commit_hash
# Make follow-up changes to be reverted (sort of).
FileOperations.makedir(test_user_lb,
relative_path='code/folder2',
create_activity_record=True)
FileOperations.makedir(test_user_lb,
relative_path='code/folder3',
create_activity_record=True)
# Make rollback branch from Git revision captured above.
bm = BranchManager(test_user_lb, username=TEST_USER)
new_b = bm.create_branch('rollback-from-folder-1', revision=commit)
FileOperations.makedir(test_user_lb,
relative_path='input/branch-folder',
create_activity_record=True)
# Check state of repo is as exptected
assert os.path.exists(
os.path.join(test_user_lb.root_dir, 'code/folder1'))
assert not os.path.exists(
os.path.join(test_user_lb.root_dir, 'code/folder2'))
assert not os.path.exists(
os.path.join(test_user_lb.root_dir, 'code/folder3'))
# Now, make chagnes to rollback branch
FileOperations.makedir(test_user_lb,
relative_path='input/branch-1',
create_activity_record=True)
FileOperations.makedir(test_user_lb,
relative_path='input/branch-2',
create_activity_record=True)
FileOperations.makedir(test_user_lb,
relative_path='input/branch-3',
create_activity_record=True)
# Now, try pulling upstream changes back into the rollback branch, then demonstrate state
# is as expected.
bm.merge_from(bm.workspace_branch)
assert os.path.exists(
os.path.join(test_user_lb.root_dir, 'code/folder2'))
assert os.path.exists(
os.path.join(test_user_lb.root_dir, 'code/folder3'))
assert os.path.exists(
os.path.join(test_user_lb.root_dir, 'input/branch-1'))
assert os.path.exists(
os.path.join(test_user_lb.root_dir, 'input/branch-2'))
assert os.path.exists(
os.path.join(test_user_lb.root_dir, 'input/branch-3'))
assert test_user_lb.is_repo_clean
def test_get_recent_activity(self, fixture_working_dir, snapshot,
fixture_test_file):
"""Test paging through activity records"""
im = InventoryManager(fixture_working_dir[0])
lb = im.create_labbook("default",
"default",
"labbook11",
description="my test description",
author=GitAuthor(name="tester",
email="*****@*****.**"))
FileOperations.insert_file(lb, "code", fixture_test_file)
# fake activity
store = ActivityStore(lb)
adr1 = ActivityDetailRecord(ActivityDetailType.CODE)
adr1.show = False
adr1.importance = 100
adr1.add_value("text/plain", "first")
ar = ActivityRecord(ActivityType.CODE,
show=False,
message="ran some code",
importance=50,
linked_commit="asdf")
ar.add_detail_object(adr1)
# Create Activity Record
store.create_activity_record(ar)
store.create_activity_record(ar)
store.create_activity_record(ar)
store.create_activity_record(ar)
open('/tmp/test_file.txt', 'w').write("xxx" * 50)
FileOperations.insert_file(lb, "input", '/tmp/test_file.txt')
FileOperations.makedir(lb, "input/test")
open('/tmp/test_file.txt', 'w').write("xxx" * 50)
FileOperations.insert_file(lb, "input", '/tmp/test_file.txt', "test")
FileOperations.makedir(lb, "input/test2")
open('/tmp/test_file.txt', 'w').write("xxx" * 50)
FileOperations.insert_file(lb, "input", '/tmp/test_file.txt', "test2")
store.create_activity_record(ar)
store.create_activity_record(ar)
store.create_activity_record(ar)
store.create_activity_record(ar)
store.create_activity_record(ar)
open('/tmp/test_file.txt', 'w').write("xxx" * 50)
FileOperations.insert_file(lb, "output", '/tmp/test_file.txt')
# Get all records at once with no pagination args and verify cursors look OK directly
query = """
{
labbook(owner: "default", name: "labbook11") {
overview {
recentActivity {
message
type
show
importance
tags
}
}
}
}
"""
snapshot.assert_match(fixture_working_dir[2].execute(query))
def test_with_the_whole_suite_of_file_operations_on_an_UNTRACKED_labbook(
self, mock_labbook):
x, y, lb = mock_labbook
hash_0 = lb.git.commit_hash
FileOperations.set_untracked(labbook=lb, section='input')
hash_1 = lb.git.commit_hash
assert hash_0 != hash_1
with open('/tmp/unittestfile', 'wb') as f:
f.write('àbčdęfghįjkłmñöpqrštūvwxÿż0123456789'.encode('utf-8'))
assert not os.path.exists(
os.path.join(lb.root_dir, 'input', 'unittestfile'))
r = FileOperations.put_file(lb,
section="input",
src_file=f.name,
dst_path='')
assert os.path.exists(
os.path.join(lb.root_dir, 'input', 'unittestfile'))
hash_2 = lb.git.commit_hash
FileOperations.delete_files(lb,
section='input',
relative_paths=['unittestfile'])
hash_3 = lb.git.commit_hash
target_path = os.path.join(lb.root_dir, 'input', 'unittestfile')
assert not os.path.exists(target_path)
assert lb.is_repo_clean
# Hash_2 == hash_3 because we delete a file in an UNTRACKED section
assert hash_2 == hash_3
FileOperations.makedir(lb, 'input/sample-untracked-dir/nested-dir')
hash_4 = lb.git.commit_hash
assert hash_3 == hash_4
with open('/tmp/unittestfile', 'wb') as f:
f.write('aaaaaæ'.encode('utf-8'))
FileOperations.put_file(lb,
section='input',
src_file=f.name,
dst_path='sample-untracked-dir/nested-dir')
hash_5 = lb.git.commit_hash
assert hash_4 == hash_5
FileOperations.move_file(
lb,
section='input',
src_rel_path='sample-untracked-dir/nested-dir/unittestfile',
dst_rel_path='unittestfile')
assert not os.path.exists(
os.path.join(lb.root_dir, 'input',
'sample-untracked-dir/nested-dir/unittestfile'))
assert os.path.exists(
os.path.join(lb.root_dir, 'input', 'unittestfile'))
hash_6 = lb.git.commit_hash
assert hash_5 == hash_6
FileOperations.delete_files(
lb,
section='input',
relative_paths=['sample-untracked-dir/nested-dir'])
hash_7 = lb.git.commit_hash
assert hash_6 == hash_7
