def test_update_include_exclude(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
expected = self.create_files(root).filepath\
.apply(lambda x: x.as_posix()).tolist()
i_regex = r'pizza'
expected = list(filter(lambda x: re.search(i_regex, x), expected))
e_regex = r'misc\.txt|vdb'
expected = list(
filter(lambda x: not re.search(e_regex, x), expected))
expected = sorted(expected)
result = Database(
root,
hb_root,
[Spec001, Spec002],
include_regex=i_regex,
exclude_regex=e_regex,
)
result = result.update().data.filepath.tolist()
result = sorted(result)
self.assertEqual(result, expected)
def test_delete(self):
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
data_dir = Path(hb_root, 'data')
os.makedirs(data_dir)
meta_dir = Path(hb_root, 'metadata')
os.makedirs(meta_dir)
root = Path(root, 'projects')
os.makedirs(root)
self.create_files(root)
expected = tools.directory_to_dataframe(root).filepath.tolist()
expected = sorted(expected)
db = Database(root, hb_root, [])
db.delete()
self.assertFalse(data_dir.exists())
self.assertFalse(meta_dir.exists())
result = tools.directory_to_dataframe(root).filepath.tolist()
result = sorted(result)
self.assertEqual(result, expected)
def test_init_bad_hb_root(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
expected = '/hidebound is not a directory or does not exist'
with self.assertRaisesRegexp(FileNotFoundError, expected):
Database(root, hb_root)
temp = Path(root, 'Hidebound')
os.makedirs(temp)
expected = r'Hidebound directory is not named hidebound\.$'
with self.assertRaisesRegexp(NameError, expected):
Database(root, temp)
def test_init(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
root = Path(root, 'projects')
os.makedirs(root)
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
self.create_files(root)
Database(root, hb_root)
Database(root, hb_root, [Spec001])
Database(root, hb_root, [Spec001, Spec002])
def test_read_traits(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
root = Path(root, 'projects')
os.makedirs(root)
self.create_files(root)
db = Database(root, hb_root, [Spec001, Spec002])
db.update()
# test file traits
db.data.file_traits = db.data.file_traits\
.apply(lambda x: {'foo': 'bar', 'illegal': set()})
data = db.read()
result = data.columns
self.assertIn('foo', result)
self.assertNotIn('illegal', result)
# test asset traits
db.update()
db.data.asset_traits = db.data.asset_traits\
.apply(lambda x: {'foo': 'bar', 'illegal': set()})
data = db.read(group_by_asset=True)
result = data.columns
self.assertIn('foo', result)
self.assertNotIn('illegal', result)
def test_read_no_files(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
root = Path(root, 'projects')
os.makedirs(root)
db = Database(root, hb_root, [Spec001, Spec002])
db.update()
result = db.read()
self.assertEqual(len(result), 0)
def test_read_coordinates(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
root = Path(root, 'projects')
os.makedirs(root)
self.create_files(root)
db = Database(root, hb_root, [Spec001, Spec002])
db.update()
db.data.file_traits = db.data.file_traits\
.apply(lambda x: {'coordinate': [0, 1]})
data = db.read()
# xy
result = data.columns
expected = ['coordinate_x', 'coordinate_y']
for col in expected:
self.assertIn(col, result)
self.assertNotIn('coordinate_z', result)
# xyz
db.update()
db.data.file_traits = db.data.file_traits\
.apply(lambda x: {'coordinate': [0, 1, 0]})
data = db.read()
result = data.columns
expected = ['coordinate_x', 'coordinate_y', 'coordinate_z']
for col in expected:
self.assertIn(col, result)
def test_init_bad_specifications(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
self.create_files(root)
expected = 'SpecificationBase may only contain subclasses of'
expected += ' SpecificationBase. Found: .*.'
with self.assertRaisesRegexp(TypeError, expected):
Database(root, hb_root, [BadSpec])
with self.assertRaisesRegexp(TypeError, expected):
Database(root, hb_root, [Spec001, BadSpec])
def test_update_no_files(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
result = Database(root, hb_root, [Spec001]).update().data
self.assertEqual(len(result), 0)
self.assertEqual(result.columns.tolist(), self.columns)
def test_update(self):
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
Spec001, Spec002, BadSpec = self.get_specifications()
expected = self.create_files(root).filepath\
.apply(lambda x: x.as_posix()).tolist()
expected = sorted(expected)
data = Database(root, hb_root, [Spec001, Spec002]).update().data
result = data.filepath.tolist()
result = sorted(result)
self.assertEqual(result, expected)
result = data.groupby('asset_path').asset_valid.first().tolist()
expected = [True, True, False, True, False]
self.assertEqual(result, expected)
def test_init_bad_write_mode(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
expected = r"Invalid write mode: foo not in \['copy', 'move'\]\."
with self.assertRaisesRegexp(ValueError, expected):
Database(root, hb_root, [Spec001], write_mode='foo')
def test_from_json(self):
with TemporaryDirectory() as root:
# create hb root dir
hb_root = Path(root, 'hidebound').as_posix()
os.makedirs(hb_root)
spec_file = self.write_spec_file(root)
config = dict(
root_directory=root,
hidebound_directory=hb_root,
specification_files=[spec_file],
include_regex='foo',
exclude_regex='bar',
write_mode='copy',
)
config_file = Path(root, 'config.json')
with open(config_file, 'w') as f:
json.dump(config, f)
Database.from_json(config_file)
def test_from_config(self):
with TemporaryDirectory() as root:
# create hb root dir
hb_root = Path(root, 'hidebound').as_posix()
os.makedirs(hb_root)
spec_file = self.write_spec_file(root)
config = dict(
root_directory=root,
hidebound_directory=hb_root,
specification_files=[spec_file],
include_regex='foo',
exclude_regex='bar',
write_mode='copy',
)
expected = copy(config)
Database.from_config(config)
self.assertEqual(config, expected)
config['specification_files'] = ['/foo/bar.py']
with self.assertRaises(DataError):
Database.from_config(config)
def test_read_column_order(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
root = Path(root, 'projects')
os.makedirs(root)
self.create_files(root)
db = Database(root, hb_root, [Spec001, Spec002])
db.update()
result = db.read().columns.tolist()
expected = [
'project', 'specification', 'descriptor', 'version',
'coordinate_x', 'coordinate_y', 'coordinate_z', 'frame',
'extension', 'filename', 'filepath', 'file_error',
'asset_name', 'asset_path', 'asset_type', 'asset_error',
'asset_valid'
]
expected = list(filter(lambda x: x in result, expected))
result = result[:len(expected)]
self.assertEqual(result, expected)
def test_export(self):
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
Spec001, Spec002, BadSpec = self.get_specifications()
self.create_files(root)
exporters = dict(girder=dict(api_key='api_key', root_id='root_id'))
db = Database(root,
hb_root, [Spec001, Spec002],
exporters=exporters)
db._Database__exporter_lut = dict(girder=MockGirderExporter)
db.update().create().export()
client = db._Database__exporter_lut['girder']._client
result = list(client.folders.keys())
asset_paths = [
'p-proj001_s-spec001_d-pizza_v001',
'p-proj001_s-spec001_d-pizza_v002',
'p-proj001_s-spec002_d-taco_v001',
]
for expected in asset_paths:
self.assertIn(expected, result)
def test_update_error(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
files = self.create_files(root)
data = Database(root, hb_root, [Spec001, Spec002]).update().data
keys = files.filepath.tolist()
lut = dict(zip(keys, files.file_error.tolist()))
data = data[data.filepath.apply(lambda x: x in keys)]
regexes = data.filepath.apply(lambda x: lut[x.as_posix()]).tolist()
results = data.file_error.apply(lambda x: x[0]).tolist()
for result, regex in zip(results, regexes):
self.assertRegex(result, regex)
def test_read_legal_types(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
root = Path(root, 'projects')
os.makedirs(root)
self.create_files(root)
db = Database(root, hb_root, [Spec001, Spec002])
# test data initiliazation error
expected = 'Data not initialized. Please call update.'
with self.assertRaisesRegexp(RuntimeError, expected):
db.read()
db.update()
data = db.read()
# test types by file
result = data.applymap(type)\
.apply(lambda x: x.unique().tolist())\
.tolist()
result = list(chain(*result))
result = set(result)
expected = set([int, float, str, bool, None])
result = result.difference(expected)
self.assertEqual(len(result), 0)
# test types by asset
data = db.read(group_by_asset=True)
result = data.applymap(type)\
.apply(lambda x: x.unique().tolist())\
.tolist()
result = list(chain(*result))
result = set(result)
expected = set([int, float, str, bool, None])
result = result.difference(expected)
self.assertEqual(len(result), 0)
def test_search(self):
Spec001, Spec002, BadSpec = self.get_specifications()
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
root = Path(root, 'projects')
os.makedirs(root)
self.create_files(root)
db = Database(root, hb_root, [Spec001, Spec002])
db.update()
db.search('SELECT * FROM data WHERE version == 3')
def test_create(self):
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
Spec001, Spec002, BadSpec = self.get_specifications()
self.create_files(root)
db = Database(root, hb_root, [Spec001, Spec002])
# test data initiliazation error
expected = 'Data not initialized. Please call update.'
with self.assertRaisesRegexp(RuntimeError, expected):
db.create()
db.update()
data = db.data
db.create()
data = data[data.asset_valid]
# ensure files are written
result = Path(hb_root, 'data')
result = tools.directory_to_dataframe(result)
result = sorted(result.filename.tolist())
self.assertGreater(len(result), 0)
expected = sorted(data.filename.tolist())
self.assertEqual(result, expected)
# ensure file metadata is written
result = len(os.listdir(Path(hb_root, 'metadata', 'file')))
self.assertGreater(result, 0)
expected = data.filepath.nunique()
self.assertEqual(result, expected)
# ensure asset metadata is written
result = len(os.listdir(Path(hb_root, 'metadata', 'asset')))
self.assertGreater(result, 0)
expected = data.asset_path.nunique()
self.assertEqual(result, expected)
def initialize():
# type: () -> flask.Response
'''
Initialize database with given config.
Returns:
Response: Flask Response instance.
'''
global DATABASE
global CONFIG
config = dict(
specification_files=[],
include_regex='',
exclude_regex=r'\.DS_Store',
write_mode='copy',
)
temp = flask.request.get_json()
try:
temp = json.loads(temp)
except (JSONDecodeError, TypeError):
return server_tools.get_config_error()
if not isinstance(temp, dict):
return server_tools.get_config_error()
config.update(temp)
DATABASE = Database.from_config(config)
CONFIG = config
return flask.Response(
response=json.dumps(dict(
message='Database initialized.',
config=config,
)),
mimetype='application/json'
)
def test_create_copy(self):
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
root = Path(root, 'projects').as_posix()
os.makedirs(root)
Spec001, Spec002, BadSpec = self.get_specifications()
expected = self.create_files(root).filepath\
.apply(lambda x: x.as_posix()).tolist()
expected = sorted(expected)
db = Database(root, hb_root, [Spec001, Spec002], write_mode='copy')
db.update()
db.create()
result = tools.directory_to_dataframe(root).filepath.tolist()
result = sorted(result)
self.assertEqual(result, expected)
def test_create_all_invalid(self):
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
Spec001, Spec002, BadSpec = self.get_specifications()
self.create_files(root)
db = Database(root, hb_root, [Spec001, Spec002])
db.update()
data = db.data
data['asset_valid'] = False
db.create()
result = Path(hb_root, 'data')
self.assertFalse(result.exists())
result = Path(hb_root, 'metadata', 'file')
self.assertFalse(result.exists())
result = Path(hb_root, 'metadata', 'asset')
self.assertFalse(result.exists())
def test_create_move(self):
with TemporaryDirectory() as root:
hb_root = Path(root, 'hidebound')
os.makedirs(hb_root)
root = Path(root, 'projects').as_posix()
os.makedirs(root)
self.create_files(root)
Spec001, Spec002, BadSpec = self.get_specifications()
db = Database(root, hb_root, [Spec001, Spec002], write_mode='move')
db.update()
data = db.data
db.create()
data = data[data.asset_valid]
# assert that no valid asset files are found in their original
result = data.filepath.apply(
lambda x: os.path.exists(x)).unique().tolist()
self.assertEqual(result, [False])
# ensure files are written
result = Path(hb_root, 'data')
result = tools.directory_to_dataframe(result)
result = sorted(result.filename.tolist())
self.assertGreater(len(result), 0)
expected = sorted(data.filename.tolist())
self.assertEqual(result, expected)
# ensure file metadata is written
result = len(os.listdir(Path(hb_root, 'metadata', 'file')))
self.assertGreater(result, 0)
expected = data.filepath.nunique()
self.assertEqual(result, expected)
# ensure asset metadata is written
result = len(os.listdir(Path(hb_root, 'metadata', 'asset')))
self.assertGreater(result, 0)
expected = data.asset_path.nunique()
self.assertEqual(result, expected)
def test_init_bad_root(self):
Spec001, Spec002, BadSpec = self.get_specifications()
expected = '/foo is not a directory or does not exist.'
with self.assertRaisesRegexp(FileNotFoundError, expected):
Database('/foo', '/bar', [Spec001])
