def test_deduplicate_annotations(self):
"""Test removing duplicated annotations."""
store = FileSystemDatastore(STORE_DIR)
ds = store.create_dataset(
columns=[
DatasetColumn(identifier=0, name='A'),
DatasetColumn(identifier=1, name='B')
],
rows=[DatasetRow(identifier=0, values=['a', 'b'])],
annotations=DatasetMetadata(
cells=[
DatasetAnnotation(column_id=0, row_id=0, key='X', value=1),
DatasetAnnotation(column_id=0, row_id=0, key='X', value=2),
DatasetAnnotation(column_id=1, row_id=0, key='X', value=3),
DatasetAnnotation(column_id=1, row_id=1, key='X', value=3),
DatasetAnnotation(column_id=0, row_id=0, key='Y', value=1),
DatasetAnnotation(column_id=0, row_id=0, key='X', value=1),
DatasetAnnotation(column_id=0, row_id=0, key='X', value=2),
DatasetAnnotation(column_id=1, row_id=0, key='X', value=3),
DatasetAnnotation(column_id=1, row_id=1, key='X', value=3),
],
columns=[
DatasetAnnotation(column_id=0, key='A', value='x'),
DatasetAnnotation(column_id=1, key='A', value='x'),
DatasetAnnotation(column_id=0, key='A', value='x'),
DatasetAnnotation(column_id=1, key='A', value='x'),
DatasetAnnotation(column_id=0, key='A', value='x'),
DatasetAnnotation(column_id=1, key='A', value='x'),
DatasetAnnotation(column_id=0, key='A', value='x'),
DatasetAnnotation(column_id=1, key='A', value='x')
],
rows=[
DatasetAnnotation(row_id=0, key='E', value=100),
DatasetAnnotation(row_id=0, key='E', value=100)
]
)
)
ds = store.get_dataset(ds.identifier)
self.assertEqual(len(ds.annotations.cells), 4)
self.assertEqual(len(ds.annotations.columns), 2)
self.assertEqual(len(ds.annotations.rows), 1)
annos = ds.annotations.for_cell(column_id=0, row_id=0)
self.assertEqual(len(annos), 3)
self.assertTrue(1 in [a.value for a in annos])
self.assertTrue(2 in [a.value for a in annos])
self.assertFalse(3 in [a.value for a in annos])
self.assertEqual(len(ds.annotations.find_all(values=annos, key='X')), 2)
with self.assertRaises(ValueError):
ds.annotations.find_one(values=annos, key='X')
self.assertEqual(len(ds.annotations.for_column(column_id=0)), 1)
self.assertEqual(len(ds.annotations.for_row(row_id=0)), 1)
annotations = ds.annotations.filter(columns=[1])
self.assertEqual(len(annotations.cells), 1)
self.assertEqual(len(annotations.columns), 1)
self.assertEqual(len(annotations.rows), 1)
def test_query_annotations(self):
"""Test retrieving annotations via the datastore."""
store = FileSystemDatastore(STORE_DIR)
ds = store.create_dataset(
columns=[
DatasetColumn(identifier=0, name='A'),
DatasetColumn(identifier=1, name='B')
],
rows=[DatasetRow(identifier=0, values=['a', 'b'])],
properties=EXAMPLE_PROPERTIES
)
properties = store.get_properties(ds.identifier)
self.assertEqual(len(properties["columns"]), 2)
def test_query_annotations(self):
"""Test retrieving annotations via the datastore."""
store = FileSystemDatastore(STORE_DIR)
ds = store.create_dataset(
columns=[
DatasetColumn(identifier=0, name='A'),
DatasetColumn(identifier=1, name='B')
],
rows=[DatasetRow(identifier=0, values=['a', 'b'])],
annotations=DatasetMetadata(
cells=[
DatasetAnnotation(column_id=0, row_id=0, key='X', value=1),
DatasetAnnotation(column_id=0, row_id=0, key='X', value=2),
DatasetAnnotation(column_id=1, row_id=0, key='X', value=3),
DatasetAnnotation(column_id=0, row_id=0, key='Y', value=1)
],
columns=[
DatasetAnnotation(column_id=0, key='A', value='x'),
DatasetAnnotation(column_id=1, key='A', value='x')
],
rows=[
DatasetAnnotation(row_id=0, key='E', value=100)
]
)
)
annos = store.get_annotations(ds.identifier, column_id=1)
self.assertEqual(len(annos.columns), 1)
self.assertEqual(len(annos.rows), 0)
self.assertEqual(len(annos.cells), 0)
annos = store.get_annotations(ds.identifier, column_id=0)
self.assertEqual(len(annos.columns), 1)
self.assertEqual(len(annos.rows), 0)
self.assertEqual(len(annos.cells), 0)
annos = store.get_annotations(ds.identifier, row_id=0)
self.assertEqual(len(annos.columns), 0)
self.assertEqual(len(annos.rows), 1)
self.assertEqual(len(annos.cells), 0)
annos = store.get_annotations(ds.identifier, column_id=1, row_id=0)
self.assertEqual(len(annos.columns), 0)
self.assertEqual(len(annos.rows), 0)
self.assertEqual(len(annos.cells), 1)
annos = store.get_annotations(ds.identifier, column_id=0, row_id=0)
self.assertEqual(len(annos.columns), 0)
self.assertEqual(len(annos.rows), 0)
self.assertEqual(len(annos.cells), 3)
def test_properties(self):
"""Test loading a dataset from file."""
store = FileSystemDatastore(STORE_DIR)
ds = store.create_dataset(
columns=[
DatasetColumn(identifier=0, name='A'),
DatasetColumn(identifier=1, name='B')
],
rows=[DatasetRow(identifier=0, values=[1, 2])],
properties=EXAMPLE_PROPERTIES
)
ds = store.get_dataset(ds.identifier)
column_props = ds.properties['columns']
self.assertEqual(len(column_props), 2)
self.assertTrue('A' in [prop['name'] for prop in column_props])
# Reload datastore
store = FileSystemDatastore(STORE_DIR)
ds = store.get_dataset(ds.identifier)
column_props = ds.properties['columns']
self.assertEqual(len(column_props), 2)
def test_create_dataset(self):
"""Test loading a dataset from file."""
store = FileSystemDatastore(STORE_DIR)
ds = store.create_dataset(
columns=[
DatasetColumn(identifier=0, name='A'),
DatasetColumn(identifier=1, name='B')
],
rows=[DatasetRow(identifier=0, values=['a', 'b'])]
)
ds = store.get_dataset(ds.identifier)
column_ids = [col.identifier for col in ds.columns]
self.assertEqual(len(ds.columns), 2)
for id in [0, 1]:
self.assertTrue(id in column_ids)
column_names = [col.name for col in ds.columns]
for name in ['A', 'B']:
self.assertTrue(name in column_names)
rows = ds.fetch_rows()
self.assertEqual(len(rows), 1)
self.assertEqual(rows[0].values, ['a', 'b'])
self.assertEqual(len(ds.annotations.cells), 0)
self.assertEqual(len(ds.annotations.columns), 0)
self.assertEqual(len(ds.annotations.rows), 0)
# Reload the datastore
store = FileSystemDatastore(STORE_DIR)
ds = store.get_dataset(ds.identifier)
column_ids = [col.identifier for col in ds.columns]
self.assertEqual(len(ds.columns), 2)
for id in [0, 1]:
self.assertTrue(id in column_ids)
column_names = [col.name for col in ds.columns]
for name in ['A', 'B']:
self.assertTrue(name in column_names)
rows = ds.fetch_rows()
self.assertEqual(len(rows), 1)
self.assertEqual(rows[0].values, ['a', 'b'])
self.assertEqual(len(ds.annotations.cells), 0)
self.assertEqual(len(ds.annotations.columns), 0)
self.assertEqual(len(ds.annotations.rows), 0)
def test_update_annotations(self):
"""Test updating annotations via the datastore."""
store = FileSystemDatastore(STORE_DIR)
ds = store.create_dataset(
columns=[
DatasetColumn(identifier=0, name='A'),
DatasetColumn(identifier=1, name='B')
],
rows=[DatasetRow(identifier=0, values=['a', 'b'])],
annotations=DatasetMetadata(
cells=[
DatasetAnnotation(column_id=0, row_id=0, key='X', value=1),
DatasetAnnotation(column_id=0, row_id=0, key='X', value=2),
DatasetAnnotation(column_id=1, row_id=0, key='X', value=3),
DatasetAnnotation(column_id=0, row_id=0, key='Y', value=1)
],
columns=[
DatasetAnnotation(column_id=0, key='A', value='x'),
DatasetAnnotation(column_id=1, key='A', value='x')
],
rows=[
DatasetAnnotation(row_id=0, key='E', value=100)
]
)
)
# INSERT row annotatins
store.update_annotation(
ds.identifier,
key='D',
row_id=0,
new_value=200
)
annos = store.get_annotations(ds.identifier, row_id=0)
self.assertEqual(len(annos.rows), 2)
for key in ['D', 'E']:
self.assertTrue(key in [a.key for a in annos.rows])
for val in [100, 200]:
self.assertTrue(val in [a.value for a in annos.rows])
# UPDATE column annotation
store.update_annotation(
ds.identifier,
key='A',
column_id=1,
old_value='x',
new_value='y'
)
annos = store.get_annotations(ds.identifier, column_id=1)
self.assertEqual(annos.columns[0].key, 'A')
self.assertEqual(annos.columns[0].value, 'y')
# DELETE cell annotation
store.update_annotation(
ds.identifier,
key='X',
column_id=0,
row_id=0,
old_value=2,
)
annos = store.get_annotations(ds.identifier, column_id=0, row_id=0)
self.assertEqual(len(annos.cells), 2)
for a in annos.cells:
self.assertNotEqual(a.value, 2)
result = store.update_annotation(
ds.identifier,
key='X',
column_id=1,
row_id=0,
old_value=3,
)
self.assertTrue(result)
annos = store.get_annotations(ds.identifier, column_id=1, row_id=0)
self.assertEqual(len(annos.cells), 0)