def test_local_bytes(self, fixture_single_dataset):
"""Test getting the a Dataset's local_bytes"""
ds = fixture_single_dataset[3]
query = """
{
dataset(owner: "default", name: "test-dataset") {
overview {
localBytes
}
}
}
"""
result = fixture_single_dataset[2].execute(query)
assert 'errors' not in result
assert result['data']['dataset']['overview']['localBytes'] == '35'
# Delete all files
m = Manifest(ds, 'default')
current_revision_dir = m.cache_mgr.current_revision_dir
shutil.rmtree(current_revision_dir)
os.makedirs(current_revision_dir)
result = fixture_single_dataset[2].execute(query)
assert 'errors' not in result
assert result['data']['dataset']['overview']['localBytes'] == '0'
# Update manifest after all files have been deleted, should still be 0
m.update()
result = fixture_single_dataset[2].execute(query)
assert 'errors' not in result
assert result['data']['dataset']['overview']['localBytes'] == '0'
def test_num_files(self, fixture_single_dataset):
"""Test getting the a Dataset's file count"""
ds = fixture_single_dataset[3]
query = """
{
dataset(owner: "default", name: "test-dataset") {
overview {
numFiles
}
}
}
"""
result = fixture_single_dataset[2].execute(query)
assert 'errors' not in result
assert result['data']['dataset']['overview']['numFiles'] == 5
m = Manifest(ds, 'default')
current_revision_dir = m.cache_mgr.current_revision_dir
shutil.rmtree(current_revision_dir)
os.makedirs(current_revision_dir)
m.update()
result = fixture_single_dataset[2].execute(query)
assert 'errors' not in result
assert result['data']['dataset']['overview']['numFiles'] == 0
def fixture_single_dataset():
"""A pytest fixture that creates a temporary working directory, a config file to match, creates the schema,
and populates the environment component repository.
Class scope modifier attached
"""
# Create temp dir
config_file, temp_dir = _create_temp_work_dir()
# Create user identity
insert_cached_identity(temp_dir)
# Create test client
schema = graphene.Schema(query=LabbookQuery, mutation=LabbookMutations)
# Create a bunch of lab books
im = InventoryManager(config_file)
ds = im.create_dataset('default',
'default',
"test-dataset",
storage_type="gigantum_object_v1",
description="Cats 2")
m = Manifest(ds, 'default')
cm_class = get_cache_manager_class(ds.client_config)
cache_mgr = cm_class(ds, 'default')
revision = ds.git.repo.head.commit.hexsha
os.makedirs(os.path.join(cache_mgr.cache_root, revision, "other_dir"))
helper_append_file(cache_mgr.cache_root, revision, "test1.txt", "asdfasdf")
helper_append_file(cache_mgr.cache_root, revision, "test2.txt", "rtg")
helper_append_file(cache_mgr.cache_root, revision, "test3.txt", "wer")
helper_append_file(cache_mgr.cache_root, revision, "other_dir/test4.txt",
"dfasdfhfgjhg")
helper_append_file(cache_mgr.cache_root, revision, "other_dir/test5.txt",
"fdghdfgsa")
m.update()
with patch.object(Configuration, 'find_default_config',
lambda self: config_file):
# Load User identity into app context
app = Flask("lmsrvlabbook")
app.config["LABMGR_CONFIG"] = Configuration()
app.config["LABMGR_ID_MGR"] = get_identity_manager(Configuration())
with app.app_context():
# within this block, current_app points to app. Set current user explicitly (this is done in the middleware)
flask.g.user_obj = app.config["LABMGR_ID_MGR"].get_user_profile()
# Create a test client
client = Client(schema,
middleware=[DataloaderMiddleware()],
context_value=ContextMock())
yield config_file, temp_dir, client, ds, cache_mgr
# Remove the temp_dir
shutil.rmtree(temp_dir)
def test_file_distribution_hidden(self, fixture_single_dataset):
""""""
ds = fixture_single_dataset[3]
query = """
{
dataset(owner: "default", name: "test-dataset") {
overview {
fileTypeDistribution
}
}
}
"""
result = fixture_single_dataset[2].execute(query)
assert 'errors' not in result
assert result['data']['dataset']['overview'][
'fileTypeDistribution'] == ['1.00|.txt']
# Delete all files
m = Manifest(ds, 'default')
os.makedirs(
os.path.join(m.cache_mgr.cache_root, m.dataset_revision,
".hiddendir"))
os.makedirs(
os.path.join(m.cache_mgr.cache_root, m.dataset_revision,
".hiddendir", "subdir"))
helper_append_file(m.cache_mgr.cache_root, m.dataset_revision,
"test55.csv", "22222")
helper_append_file(m.cache_mgr.cache_root, m.dataset_revision,
"df.csv", "11")
helper_append_file(m.cache_mgr.cache_root, m.dataset_revision,
".hidden", "343")
helper_append_file(m.cache_mgr.cache_root, m.dataset_revision,
"noextension", "6t4")
helper_append_file(m.cache_mgr.cache_root, m.dataset_revision,
".hiddendir/tester.png", "8544")
helper_append_file(m.cache_mgr.cache_root, m.dataset_revision,
".hiddendir/subdir/blah.jpeg", "8544")
helper_append_file(m.cache_mgr.cache_root, m.dataset_revision,
".hiddendir/subdir/.hiddenfile", "jhg")
m.update()
result = fixture_single_dataset[2].execute(query)
assert 'errors' not in result
assert len(
result['data']['dataset']['overview']['fileTypeDistribution']) == 4
assert result['data']['dataset']['overview']['fileTypeDistribution'][
0] == '0.56|.txt'
assert result['data']['dataset']['overview']['fileTypeDistribution'][
1] == '0.22|.csv'
assert result['data']['dataset']['overview']['fileTypeDistribution'][
2] == '0.11|.jpeg'
assert result['data']['dataset']['overview']['fileTypeDistribution'][
3] == '0.11|.png'
def test_file_info_combined(self, fixture_single_dataset):
"""Test getting the a Dataset's file info"""
ds = fixture_single_dataset[3]
query = """
{
dataset(owner: "default", name: "test-dataset") {
overview {
fileTypeDistribution
localBytes
totalBytes
}
}
}
"""
result = fixture_single_dataset[2].execute(query)
assert 'errors' not in result
assert result['data']['dataset']['overview'][
'fileTypeDistribution'] == ['1.00|.txt']
assert result['data']['dataset']['overview']['localBytes'] == '35'
assert result['data']['dataset']['overview']['totalBytes'] == '35'
# Delete all files
m = Manifest(ds, 'default')
current_revision_dir = m.cache_mgr.current_revision_dir
shutil.rmtree(current_revision_dir)
os.makedirs(current_revision_dir)
result = fixture_single_dataset[2].execute(query)
assert 'errors' not in result
assert result['data']['dataset']['overview'][
'fileTypeDistribution'] == ['1.00|.txt']
assert result['data']['dataset']['overview']['localBytes'] == '0'
assert result['data']['dataset']['overview']['totalBytes'] == '35'
m.update()
result = fixture_single_dataset[2].execute(query)
assert 'errors' not in result
assert result['data']['dataset']['overview'][
'fileTypeDistribution'] == []
assert result['data']['dataset']['overview']['localBytes'] == '0'
assert result['data']['dataset']['overview']['totalBytes'] == '0'
def test_file_distribution(self, fixture_single_dataset):
"""Test getting the a Dataset's local_bytes"""
ds = fixture_single_dataset[3]
query = """
{
dataset(owner: "default", name: "test-dataset") {
overview {
fileTypeDistribution
}
}
}
"""
result = fixture_single_dataset[2].execute(query)
assert 'errors' not in result
assert len(
result['data']['dataset']['overview']['fileTypeDistribution']) == 1
assert result['data']['dataset']['overview'][
'fileTypeDistribution'] == ['1.00|.txt']
# Delete all files
m = Manifest(ds, 'default')
helper_append_file(m.cache_mgr.cache_root, m.dataset_revision,
"test55.csv", "22222")
helper_append_file(m.cache_mgr.cache_root, m.dataset_revision,
"df.csv", "33333")
helper_append_file(m.cache_mgr.cache_root, m.dataset_revision,
".hidden", "33333")
helper_append_file(m.cache_mgr.cache_root, m.dataset_revision,
"noextension", "33333")
m.update()
result = fixture_single_dataset[2].execute(query)
assert 'errors' not in result
assert len(
result['data']['dataset']['overview']['fileTypeDistribution']) == 2
assert result['data']['dataset']['overview']['fileTypeDistribution'][
0] == '0.71|.txt'
assert result['data']['dataset']['overview']['fileTypeDistribution'][
1] == '0.29|.csv'
