def test_read_traits(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
root = Path(root, 'projects')
os.makedirs(root)
self.create_files(root)
db = Database(root, hb_root, [Spec001, Spec002])
db.update()
# test file traits
db.data.file_traits = db.data.file_traits\
.apply(lambda x: {'foo': 'bar', 'illegal': set()})
data = db.read()
result = data.columns
self.assertIn('foo', result)
self.assertNotIn('illegal', result)
# test asset traits
db.update()
db.data.asset_traits = db.data.asset_traits\
.apply(lambda x: {'foo': 'bar', 'illegal': set()})
data = db.read(group_by_asset=True)
result = data.columns
self.assertIn('foo', result)
self.assertNotIn('illegal', result)
def test_read_coordinates(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
root = Path(root, 'projects')
os.makedirs(root)
self.create_files(root)
db = Database(root, hb_root, [Spec001, Spec002])
db.update()
db.data.file_traits = db.data.file_traits\
.apply(lambda x: {'coordinate': [0, 1]})
data = db.read()
# xy
result = data.columns
expected = ['coordinate_x', 'coordinate_y']
for col in expected:
self.assertIn(col, result)
self.assertNotIn('coordinate_z', result)
# xyz
db.update()
db.data.file_traits = db.data.file_traits\
.apply(lambda x: {'coordinate': [0, 1, 0]})
data = db.read()
result = data.columns
expected = ['coordinate_x', 'coordinate_y', 'coordinate_z']
for col in expected:
self.assertIn(col, result)
def test_search(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
root = Path(root, 'projects')
os.makedirs(root)
self.create_files(root)
db = Database(root, hb_root, [Spec001, Spec002])
db.update()
db.search('SELECT * FROM data WHERE version == 3')
def test_read_no_files(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
root = Path(root, 'projects')
os.makedirs(root)
db = Database(root, hb_root, [Spec001, Spec002])
db.update()
result = db.read()
self.assertEqual(len(result), 0)
def test_update_include_exclude(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
expected = self.create_files(root).filepath\
.apply(lambda x: x.as_posix()).tolist()
i_regex = r'pizza'
expected = list(filter(lambda x: re.search(i_regex, x), expected))
e_regex = r'misc\.txt|vdb'
expected = list(
filter(lambda x: not re.search(e_regex, x), expected))
expected = sorted(expected)
result = Database(
root,
hb_root,
[Spec001, Spec002],
include_regex=i_regex,
exclude_regex=e_regex,
)
result = result.update().data.filepath.tolist()
result = sorted(result)
self.assertEqual(result, expected)
def test_read_legal_types(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
root = Path(root, 'projects')
os.makedirs(root)
self.create_files(root)
db = Database(root, hb_root, [Spec001, Spec002])
# test data initiliazation error
expected = 'Data not initialized. Please call update.'
with self.assertRaisesRegexp(RuntimeError, expected):
db.read()
db.update()
data = db.read()
# test types by file
result = data.applymap(type)\
.apply(lambda x: x.unique().tolist())\
.tolist()
result = list(chain(*result))
result = set(result)
expected = set([int, float, str, bool, None])
result = result.difference(expected)
self.assertEqual(len(result), 0)
# test types by asset
data = db.read(group_by_asset=True)
result = data.applymap(type)\
.apply(lambda x: x.unique().tolist())\
.tolist()
result = list(chain(*result))
result = set(result)
expected = set([int, float, str, bool, None])
result = result.difference(expected)
self.assertEqual(len(result), 0)
def test_create_copy(self):
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
root = Path(root, 'projects').as_posix()
os.makedirs(root)
Spec001, Spec002, BadSpec = self.get_specifications()
expected = self.create_files(root).filepath\
.apply(lambda x: x.as_posix()).tolist()
expected = sorted(expected)
db = Database(root, hb_root, [Spec001, Spec002], write_mode='copy')
db.update()
db.create()
result = tools.directory_to_dataframe(root).filepath.tolist()
result = sorted(result)
self.assertEqual(result, expected)
def test_create_all_invalid(self):
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
Spec001, Spec002, BadSpec = self.get_specifications()
self.create_files(root)
db = Database(root, hb_root, [Spec001, Spec002])
db.update()
data = db.data
data['asset_valid'] = False
db.create()
result = Path(hb_root, 'data')
self.assertFalse(result.exists())
result = Path(hb_root, 'metadata', 'file')
self.assertFalse(result.exists())
result = Path(hb_root, 'metadata', 'asset')
self.assertFalse(result.exists())
def test_create_move(self):
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
root = Path(root, 'projects').as_posix()
os.makedirs(root)
self.create_files(root)
Spec001, Spec002, BadSpec = self.get_specifications()
db = Database(root, hb_root, [Spec001, Spec002], write_mode='move')
db.update()
data = db.data
db.create()
data = data[data.asset_valid]
# assert that no valid asset files are found in their original
result = data.filepath.apply(
lambda x: os.path.exists(x)).unique().tolist()
self.assertEqual(result, [False])
# ensure files are written
result = Path(hb_root, 'data')
result = tools.directory_to_dataframe(result)
result = sorted(result.filename.tolist())
self.assertGreater(len(result), 0)
expected = sorted(data.filename.tolist())
self.assertEqual(result, expected)
# ensure file metadata is written
result = len(os.listdir(Path(hb_root, 'metadata', 'file')))
self.assertGreater(result, 0)
expected = data.filepath.nunique()
self.assertEqual(result, expected)
# ensure asset metadata is written
result = len(os.listdir(Path(hb_root, 'metadata', 'asset')))
self.assertGreater(result, 0)
expected = data.asset_path.nunique()
self.assertEqual(result, expected)
def test_create(self):
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
Spec001, Spec002, BadSpec = self.get_specifications()
self.create_files(root)
db = Database(root, hb_root, [Spec001, Spec002])
# test data initiliazation error
expected = 'Data not initialized. Please call update.'
with self.assertRaisesRegexp(RuntimeError, expected):
db.create()
db.update()
data = db.data
db.create()
data = data[data.asset_valid]
# ensure files are written
result = Path(hb_root, 'data')
result = tools.directory_to_dataframe(result)
result = sorted(result.filename.tolist())
self.assertGreater(len(result), 0)
expected = sorted(data.filename.tolist())
self.assertEqual(result, expected)
# ensure file metadata is written
result = len(os.listdir(Path(hb_root, 'metadata', 'file')))
self.assertGreater(result, 0)
expected = data.filepath.nunique()
self.assertEqual(result, expected)
# ensure asset metadata is written
result = len(os.listdir(Path(hb_root, 'metadata', 'asset')))
self.assertGreater(result, 0)
expected = data.asset_path.nunique()
self.assertEqual(result, expected)
def test_export(self):
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
Spec001, Spec002, BadSpec = self.get_specifications()
self.create_files(root)
exporters = dict(girder=dict(api_key='api_key', root_id='root_id'))
db = Database(root,
hb_root, [Spec001, Spec002],
exporters=exporters)
db._Database__exporter_lut = dict(girder=MockGirderExporter)
db.update().create().export()
client = db._Database__exporter_lut['girder']._client
result = list(client.folders.keys())
asset_paths = [
'p-proj001_s-spec001_d-pizza_v001',
'p-proj001_s-spec001_d-pizza_v002',
'p-proj001_s-spec002_d-taco_v001',
]
for expected in asset_paths:
self.assertIn(expected, result)
def test_read_column_order(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
root = Path(root, 'projects')
os.makedirs(root)
self.create_files(root)
db = Database(root, hb_root, [Spec001, Spec002])
db.update()
result = db.read().columns.tolist()
expected = [
'project', 'specification', 'descriptor', 'version',
'coordinate_x', 'coordinate_y', 'coordinate_z', 'frame',
'extension', 'filename', 'filepath', 'file_error',
'asset_name', 'asset_path', 'asset_type', 'asset_error',
'asset_valid'
]
expected = list(filter(lambda x: x in result, expected))
result = result[:len(expected)]
self.assertEqual(result, expected)