def test_upsert_from_link(self) -> None:
with self.db_adapter:
test1 = ExampleStore(a=["a", "b", "c"],
b=[1, 2, 3],
c=[True, False, True])
self.db_adapter.create_group(ExampleStore.data_token.data_group)
self.db_adapter.create_group_table(ExampleStore.data_token,
ExampleStore)
self.db_adapter.create_index(ExampleStore.data_token,
ExampleStore.a_index)
self.db_adapter.insert(ExampleStore.data_token, test1)
token2 = DataToken("test2", RAW_GROUP)
test2 = ExampleStore(a=["b", "e"], b=[4, 5], c=[True, False])
self.db_adapter.create_group_table(token2, ExampleStore)
self.db_adapter.create_index(token2, ExampleStore.a_index)
self.db_adapter.insert(token2, test2)
linked_store = ExampleStore.from_backend(MockBackend(
token2, test2.columns),
validate=False)
self.db_adapter.upsert(ExampleStore.data_token, linked_store,
[ExampleStore.a])
raw1 = self.db_adapter.query(ExampleStore.data_token)
queried2 = ExampleStore.from_rows(raw1)
test2 = ExampleStore(a=["a", "b", "c", "e"],
b=[1, 4, 3, 5],
c=[True, True, True, False])
assert_that(queried2.equals(test2), equal_to(True))
class IndexReference(DataStore, version=1):
data_token = DataToken(f"index_reference", PROTECTED_GROUP)
store_type: Column[str]
store_version: Column[int]
index_name: Column[str]
column_name: Column[str]
complete_index: Index[store_type, store_version, index_name, column_name]
@staticmethod
def from_type(store_type: Type[T]) -> IndexReference:
builder = IndexReference.builder()
for index in store_type.indices:
for col in index.columns:
builder.append_row(
store_type=store_type.__name__,
store_version=store_type.version,
index_name=index.name,
column_name=col.name,
)
return builder.build()
def index_columns(self) -> dict[str, list[str]]:
if self.store_type.nunique() > 1 or self.store_version.nunique() > 1:
raise RuntimeError("Cannot request index columns on multiple data stores")
index_columns: dict[str, list[str]] = {}
for _, _, index, column in self.itertuples(ignore_index=True):
if index not in index_columns:
index_columns[index] = []
index_columns[index].append(column)
return index_columns
class StoreReference(DataStore, version=1):
data_token = DataToken(f"store_reference", PROTECTED_GROUP)
store_type: Column[str]
store_version: Column[int]
definition_reference: Column[str]
definition_version: Column[int]
type_version_index: Index[store_type, store_version]
def store_versions(self: StoreReference) -> dict[str, set[int]]:
store_versions: dict[str, set[int]] = {}
for _, type, version, _, _ in self.itertuples():
if type not in store_versions:
store_versions[type] = set()
store_versions[type].add(version)
return store_versions
@staticmethod
def create_row(store_type: Type[T]) -> StoreReference:
reference_token = StoreDefinition.data_token(store_type)
return StoreReference(
store_type=store_type.__name__,
store_version=store_type.version,
definition_reference=reference_token.table_name,
definition_version=StoreDefinition.version,
)
class MetadataReference(DataStore, version=1):
data_token = DataToken(f"metadata_reference", PROTECTED_GROUP)
metadata_type: Column[str]
metadata_version: Column[int]
definition_reference: Column[str]
definition_version: Column[int]
type_version_index: Index[metadata_type, metadata_version]
def metadata_versions(self: MetadataReference) -> dict[str, set[int]]:
metadata_versions: dict[str, set[int]] = {}
for _, type, version, _, _ in self.itertuples():
if type not in metadata_versions:
metadata_versions[type] = set()
metadata_versions[type].add(version)
return metadata_versions
@staticmethod
def create_row(metadata_type: Type[M]) -> MetadataReference:
reference_token = MetadataDefinition.data_token(metadata_type)
return MetadataReference(
metadata_type=metadata_type.__name__,
metadata_version=metadata_type.version,
definition_reference=reference_token.table_name,
definition_version=MetadataDefinition.version,
)
def test_create_insert_query_data(self) -> None:
metadata = ExampleMetadata(
test_str="test",
test_int=123,
test_float=0.123,
test_bool=True,
test_timestamp=datetime.now(),
)
now = datetime.now()
insert_store = ExampleStore(
metadata=metadata,
a=["a", "b", "c"],
b=[1, 2, 3],
c=[True, False, True],
d=[now, now, now],
)
conn_conf = self.sql_db.connection_config()
data_token = DataToken("test_table", "raw")
with Sqlite3Database(conn_conf) as db:
db.insert(data_token, insert_store)
with Sqlite3Database(conn_conf) as db:
query_mask = ExampleStore.b >= 2
actual = db.query(ExampleStore, data_token, query_mask)
expected = ExampleStore(
metadata=metadata,
a=["b", "c"],
b=[2, 3],
c=[False, True],
d=[now, now]
)
assert_that(actual.equals(expected), equal_to(True))
def _store_definition_reference_version(
self, store_type: str,
store_version: int) -> tuple[DataToken, int]:
with self._db_adapter:
if not self._db_adapter.has_group_table(StoreReference.data_token):
raise DatabaseCorruptionError(
"StoreReference table is missing")
else:
columns = [
str(StoreReference.definition_reference),
str(StoreReference.definition_version),
]
table_rows = self._db_adapter.query(
StoreReference.data_token,
(StoreReference.store_type == store_type)
& (StoreReference.store_version == store_version),
columns,
)
if len(table_rows) == 0:
raise DatabaseCorruptionError(
f"Data store definition {store_type} V{store_version} missing from StoreReference"
)
if len(table_rows) > 1:
raise DatabaseCorruptionError(
f"Duplicate data store references found for {store_type} V{store_version}"
)
return (
DataToken(table_rows[0][0], PROTECTED_GROUP),
table_rows[0][1],
)
def test_drop_table(self) -> None:
assert_that(self.registrar.has_group(RAW_GROUP), equal_to(False))
assert_that(self.registrar.has_table(ExampleStore.data_token), equal_to(False))
assert_that(self.registrar._has_store_type("ExampleStore", 1), equal_to(False))
def_token = DataToken("ExampleStore_v1_definition", PROTECTED_GROUP)
assert_that(self.registrar.has_table(def_token), equal_to(False))
self.registrar.create_table(ExampleStore.data_token, ExampleStore)
assert_that(self.registrar.has_group(RAW_GROUP), equal_to(True))
assert_that(self.registrar.has_table(ExampleStore.data_token), equal_to(True))
assert_that(self.registrar._has_store_type("ExampleStore", 1), equal_to(True))
assert_that(self.registrar.has_table(def_token), equal_to(True))
self.registrar.drop_table(ExampleStore.data_token)
assert_that(self.registrar.has_group(RAW_GROUP), equal_to(False))
assert_that(self.registrar.has_table(ExampleStore.data_token), equal_to(False))
assert_that(self.registrar._has_store_type("ExampleStore", 1), equal_to(False))
def_token = DataToken("ExampleStore_v1_definition", PROTECTED_GROUP)
assert_that(self.registrar.has_table(def_token), equal_to(False))
def test_reference_table_internal_setup(self) -> None:
table_refs = self.registrar._table_references()
assert_that(table_refs.equals(TABLE_REFERENCE), is_(True))
store_refs = self.registrar._store_references()
assert_that(store_refs.equals(STORE_REFERENCE))
table_ref_store_def = self.registrar._store_definition(
DataToken("TableReference_v1_definition", PROTECTED_GROUP), 1
)
assert_that(table_ref_store_def.equals(TABLE_REFERENCE_STORE_DEFINITION))
store_ref_store_def = self.registrar._store_definition(
DataToken("StoreReference_v1_definition", PROTECTED_GROUP), 1
)
assert_that(store_ref_store_def.equals(STORE_REFERENCE_STORE_DEFINITION))
store_def_store_def = self.registrar._store_definition(
DataToken("StoreDefinition_v1_definition", PROTECTED_GROUP), 1
)
assert_that(store_def_store_def.equals(STORE_DEFINITION_STORE_DEFINITION))
def test_store_definition(self) -> None:
self.registrar.create_table(ExampleStore.data_token, ExampleStore)
actual = self.registrar._store_definition(
DataToken("ExampleStore_v1_definition", PROTECTED_GROUP), 1
)
expected = StoreDefinition(
column_name=["a", "b", "c"],
column_type=[
pickle.dumps(String),
pickle.dumps(Int64),
pickle.dumps(Boolean),
],
)
assert_that(actual.equals(expected), is_(True))
def test_definition_reference_version(self) -> None:
self.registrar.create_table(ExampleStore.data_token, ExampleStore)
store_name_def_tokens = {
"ExampleStore": DataToken("ExampleStore_v1_definition", PROTECTED_GROUP),
"TableReference": DataToken(
"TableReference_v1_definition", PROTECTED_GROUP
),
"StoreReference": DataToken(
"StoreReference_v1_definition", PROTECTED_GROUP
),
"StoreDefinition": DataToken(
"StoreDefinition_v1_definition", PROTECTED_GROUP
),
"IndexReference": DataToken(
"IndexReference_v1_definition", PROTECTED_GROUP
),
}
for store_name, def_token in store_name_def_tokens.items():
store_token, def_version = self.registrar._store_definition_reference_version(
store_name, 1
)
assert_that(store_token, equal_to(def_token))
assert_that(def_version, equal_to(1))
def test_delete_group_metadata(self) -> None:
with self.db_adapter:
test_metadata = ExampleMetadata(
test_str="test",
test_int=123,
test_float=0.123,
test_bool=True,
test_timestamp=datetime.now(),
)
token1 = DataToken("test1", RAW_GROUP)
token2 = DataToken("test2", RAW_GROUP)
self.db_adapter._update_group_table_metadata(token1, test_metadata)
self.db_adapter._update_group_table_metadata(token2, test_metadata)
assert_that(
self.db_adapter.get_group_table_metadata(
token1, ExampleMetadata),
not_(equal_to(None)),
)
assert_that(
self.db_adapter.get_group_table_metadata(
token2, ExampleMetadata),
not_(equal_to(None)),
)
self.db_adapter._delete_group_metadata(RAW_GROUP)
assert_that(
self.db_adapter.get_group_table_metadata(
token1, ExampleMetadata),
equal_to(None),
)
assert_that(
self.db_adapter.get_group_table_metadata(
token2, ExampleMetadata),
equal_to(None),
)
class ExampleStore(DataStore):
data_token = DataToken("test", RAW_GROUP)
a: Column[str]
b: Column[int]
c: Column[bool]
d: Column[datetime]
a_index: Index[a]
ab_index: Index[a, b]
metadata: ExampleMetadata
def test_method(self: "ExampleStore") -> str:
return "test_result"
def test_table_protected(self) -> None:
self.registrar.create_table(ExampleStore.data_token, ExampleStore)
assert_that(
self.registrar._is_table_protected(ExampleStore.data_token), is_(False)
)
protected_tokens = [
DataToken("table_reference", PROTECTED_GROUP),
DataToken("store_reference", PROTECTED_GROUP),
DataToken("index_reference", PROTECTED_GROUP),
DataToken("StoreDefinition_v1_definition", PROTECTED_GROUP),
DataToken("TableReference_v1_definition", PROTECTED_GROUP),
DataToken("StoreReference_v1_definition", PROTECTED_GROUP),
DataToken("IndexReference_v1_definition", PROTECTED_GROUP),
]
for token in protected_tokens:
assert_that(self.registrar._is_table_protected(token), is_(True))
def test_list_group_tables(self) -> None:
protected_tokens = [
DataToken("table_reference", PROTECTED_GROUP),
DataToken("store_reference", PROTECTED_GROUP),
DataToken("index_reference", PROTECTED_GROUP),
DataToken("StoreDefinition_v1_definition", PROTECTED_GROUP),
DataToken("TableReference_v1_definition", PROTECTED_GROUP),
DataToken("StoreReference_v1_definition", PROTECTED_GROUP),
DataToken("IndexReference_v1_definition", PROTECTED_GROUP),
DataToken("ExampleStore_v1_definition", PROTECTED_GROUP),
]
self.registrar.create_table(ExampleStore.data_token, ExampleStore)
assert_that(
self.registrar.list_group_tables(RAW_GROUP),
equal_to([ExampleStore.data_token]),
)
assert_that(
self.registrar.list_group_tables(PROTECTED_GROUP),
equal_to(protected_tokens),
)
def test_list_group_tables(self) -> None:
protected_tokens = [
DataToken("table_reference", PROTECTED_GROUP),
DataToken("store_reference", PROTECTED_GROUP),
DataToken("index_reference", PROTECTED_GROUP),
DataToken("StoreDefinition_v1_definition", PROTECTED_GROUP),
DataToken("TableReference_v1_definition", PROTECTED_GROUP),
DataToken("StoreReference_v1_definition", PROTECTED_GROUP),
DataToken("IndexReference_v1_definition", PROTECTED_GROUP),
DataToken("ExampleStore_v1_definition", PROTECTED_GROUP),
]
test = ExampleStore(a=["1", "2", "3"], b=[1, 2, 3], c=[True, False, True])
self.database.insert(ExampleStore.data_token, test)
assert_that(
self.database.list_group_tables(RAW_GROUP),
equal_to([ExampleStore.data_token]),
)
assert_that(
self.database.list_group_tables(PROTECTED_GROUP),
equal_to(protected_tokens),
)
def test_insert_from_link(self) -> None:
with self.db_adapter:
test1 = ExampleStore(a=["a", "b", "c"],
b=[1, 2, 3],
c=[True, False, True])
self.db_adapter.create_group(ExampleStore.data_token.data_group)
self.db_adapter.create_group_table(ExampleStore.data_token,
ExampleStore)
self.db_adapter.insert(ExampleStore.data_token, test1)
token2 = DataToken("test2", RAW_GROUP)
self.db_adapter.create_group_table(token2, ExampleStore)
linked_store = ExampleStore.from_backend(MockBackend(
ExampleStore.data_token, test1.columns),
validate=False)
self.db_adapter.insert(token2, linked_store)
raw1 = self.db_adapter.query(token2)
queried1 = ExampleStore.from_rows(raw1)
assert_that(test1.equals(queried1), equal_to(True))
def test_has_table(self) -> None:
self.registrar.create_table(ExampleStore.data_token, ExampleStore)
all_tokens = [
ExampleStore.data_token,
DataToken("table_reference", PROTECTED_GROUP),
DataToken("store_reference", PROTECTED_GROUP),
DataToken("TableReference_v1_definition", PROTECTED_GROUP),
DataToken("StoreDefinition_v1_definition", PROTECTED_GROUP),
DataToken("StoreReference_v1_definition", PROTECTED_GROUP),
]
for token in all_tokens:
assert_that(self.registrar.has_table(token), is_(True))
assert_that(
self.registrar.has_table(DataToken("table_reference", "NOT_GROUP")),
is_(False),
)
assert_that(
self.registrar.has_table(DataToken("not_table", "public")), is_(False)
)
def test_has_table(self) -> None:
test = ExampleStore(a=["1", "2", "3"], b=[1, 2, 3], c=[True, False, True])
self.database.insert(ExampleStore.data_token, test)
all_tokens = [
ExampleStore.data_token,
DataToken("table_reference", PROTECTED_GROUP),
DataToken("store_reference", PROTECTED_GROUP),
DataToken("TableReference_v1_definition", PROTECTED_GROUP),
DataToken("StoreDefinition_v1_definition", PROTECTED_GROUP),
DataToken("StoreReference_v1_definition", PROTECTED_GROUP),
]
for token in all_tokens:
assert_that(self.database.has_table(token), is_(True))
assert_that(
self.database.has_table(DataToken("table_reference", "NOT_GROUP")),
is_(False),
)
assert_that(
self.database.has_table(DataToken("not_table", "public")), is_(False)
)
class TableReference(DataStore, version=1):
data_token = DataToken(f"table_reference", PROTECTED_GROUP)
table_name: Column[str]
data_group: Column[str]
metadata_type: Column[str]
metadata_version: Column[int]
store_type: Column[str]
store_version: Column[int]
protected: Column[bool]
table_group_index: Index[table_name, data_group]
def data_tokens(self: TableReference) -> list[DataToken]:
return [
DataToken(row.table_name.item(), row.data_group.item())
for _, row in self.iterrows()
]
@staticmethod
def create_row(
data_token: DataToken, store_type: Type[T], protected: bool = False
) -> TableReference:
metadata_type = None
metadata_version = None
if store_type.metadata is not None:
metadata_type = store_type.metadata.__name__
metadata_version = store_type.metadata.version
return TableReference(
table_name=data_token.table_name,
data_group=data_token.data_group,
metadata_type=metadata_type,
metadata_version=metadata_version,
store_type=store_type.__name__,
store_version=store_type.version,
protected=protected,
)
def test_create_index(self) -> None:
with self.db_adapter:
test1 = ExampleStore(a=["a", "b", "c"],
b=[1, 2, 3],
c=[True, False, True])
self.db_adapter.create_group(ExampleStore.data_token.data_group)
self.db_adapter.create_group_table(ExampleStore.data_token,
ExampleStore)
self.db_adapter.create_index(ExampleStore.data_token,
ExampleStore.a_index)
token2 = DataToken("test2", RAW_GROUP)
self.db_adapter.create_group_table(token2, ExampleStore)
self.db_adapter.create_index(token2, ExampleStore.a_index)
assert_that(
self.db_adapter.has_index(ExampleStore.data_token,
ExampleStore.a_index),
equal_to(True),
)
assert_that(
self.db_adapter.has_index(token2, ExampleStore.a_index),
equal_to(True))
self.db_adapter.insert(ExampleStore.data_token, test1)
def test_reference_table_external_setup(self) -> None:
protected_tokens = [
DataToken("table_reference", PROTECTED_GROUP),
DataToken("store_reference", PROTECTED_GROUP),
DataToken("index_reference", PROTECTED_GROUP),
DataToken("StoreDefinition_v1_definition", PROTECTED_GROUP),
DataToken("TableReference_v1_definition", PROTECTED_GROUP),
DataToken("StoreReference_v1_definition", PROTECTED_GROUP),
DataToken("IndexReference_v1_definition", PROTECTED_GROUP),
]
assert_that(self.registrar.list_groups(), equal_to([]))
assert_that(
self.registrar.list_group_tables(PROTECTED_GROUP),
equal_to(protected_tokens),
)
assert_that(
self.registrar.list_tables(),
equal_to([]),
)
assert_that(self.registrar.has_group(PROTECTED_GROUP), is_(True))
for token in protected_tokens:
assert_that(self.registrar.has_table(token), is_(True))
assert_that(self.registrar._is_table_protected(token), is_(True))
assert_that(self.registrar._has_reference_tables(), is_(True))
def data_tokens(self: TableReference) -> list[DataToken]:
return [
DataToken(row.table_name.item(), row.data_group.item())
for _, row in self.iterrows()
]
def data_token(store_type: Type[T]) -> DataToken:
return DataToken(
f"{store_type.__name__}_v{store_type.version}_store_definition",
PROTECTED_GROUP,
)
def data_token(metadata_type: Type[M]) -> DataToken:
return DataToken(
f"{metadata_type.__name__}_v{metadata_type.version}_metadata_definition",
PROTECTED_GROUP,
)