def update_data_model(self, descriptor: Descriptor):
try:
descriptor_file_name = descriptor.file_name
table_name = descriptor.table_name
table_schema = descriptor.table_schema
api_schema = descriptor.api_schema
model_checksum = descriptor.get_checksum()
self.logger.info(f"{descriptor_file_name}: Found changed.")
# Get the index for this descriptor within our local metadata
data_model_index = self.get_data_model_index(descriptor_file_name)
# Create the sql alchemy orm
self.orm_factory.create_orm_from_dict(table_schema, table_name,
api_schema)
# Something needs to be modified
self.db.revision(table_name, create_table=False)
self.db.upgrade()
self.db.update_model_checksum(table_name, model_checksum,
descriptor.descriptor)
# store metadata for descriptor locally
self.data_store[data_model_index].model_checksum = model_checksum
except Exception:
self.logger.exception("Error checking data model")
def load_models_from_db(self) -> None:
# Getting all remote json
remote_descriptors = self.db.get_stored_descriptors()
for remote_descriptor_json in remote_descriptors:
remote_descriptor = Descriptor(remote_descriptor_json)
self.logger.info(
f"Loading descriptor '{remote_descriptor.table_name}' from db."
)
self.load_model_to_data_store(remote_descriptor)
self.logger.info("Loaded remote descriptors.")
def data_model_does_exist(self, descriptor: Descriptor):
try:
descriptor_file_name = descriptor.file_name
model_checksum = descriptor.get_checksum()
self.logger.info(f"{descriptor_file_name}: {model_checksum}")
self.logger.info(f"{descriptor_file_name}: Found existing.")
# check if the cached db checksum has changed from the new file
# checksum
if not self.data_model_changed(descriptor_file_name,
model_checksum):
self.logger.info(f"{descriptor_file_name}: Unchanged.")
return
self.update_data_model(descriptor)
except Exception:
self.logger.exception("Error checking data model")
def data_model_does_exist(self, descriptor: Descriptor):
try:
descriptor_file_name = descriptor.file_name
self.logger.debug(f"Looking at {descriptor_file_name}")
# Extract data for easier use
data_resource_name = descriptor.data_resource_name
table_name = descriptor.table_name
table_schema = descriptor.table_schema
api_schema = descriptor.api_schema
# calculate the checksum for this json
data_resource_checksum = descriptor.get_checksum()
restricted_fields = descriptor.restricted_fields
# determine if api changed
# model = self.db.get_model_checksum(table_name) # TODO what
# did this used to call?
data_resource_index = self.get_data_resource_index(
data_resource_name)
if self.data_resource_changed(data_resource_name,
data_resource_checksum):
data_resource = self.data_store[data_resource_index]
data_resource.checksum = data_resource_checksum
data_resource.data_resource_methods = api_schema
data_resource.data_model_name = table_name
data_resource.data_model_schema = table_schema
data_resource.data_model_object = self.orm_factory.create_orm_from_dict(
table_schema, table_name, api_schema)
data_resource.model_checksum = self.db.get_model_checksum(
table_name)
data_resource.data_resource_object.data_model = (
data_resource.data_model_object)
data_resource.data_resource_object.table_schema = table_schema
data_resource.data_resource_object.api_schema = api_schema
data_resource.data_resource_object.restricted_fields = restricted_fields
self.data_store[data_resource_index] = data_resource
except Exception:
self.logger.exception("Error checking data resource")
def data_model_does_not_exist(self, descriptor: Descriptor):
try:
descriptor_file_name = descriptor.file_name
self.logger.debug(f"Looking at {descriptor_file_name}")
# Extract data for easier use
data_resource_name = descriptor.data_resource_name
table_name = descriptor.table_name
table_schema = descriptor.table_schema
api_schema = descriptor.api_schema
# calculate the checksum for this json
data_resource_checksum = descriptor.get_checksum()
restricted_fields = descriptor.restricted_fields
data_resource = DataResource()
data_resource.checksum = data_resource_checksum
data_resource.data_resource_name = data_resource_name
data_resource.data_resource_methods = api_schema
data_resource.data_model_name = table_name
data_resource.data_model_schema = table_schema
data_resource.data_model_object = self.orm_factory.create_orm_from_dict(
table_schema, table_name, api_schema)
data_resource.model_checksum = self.db.get_model_checksum(
table_name)
data_resource.data_resource_object = self.data_resource_factory.create_api_from_dict(
api_schema,
data_resource_name,
table_name,
self.api,
data_resource.data_model_object,
table_schema,
restricted_fields,
)
self.data_store.append(data_resource)
except Exception:
self.logger.exception("Error checking data resource")
def data_model_does_not_exist(self, descriptor: Descriptor):
try:
descriptor_file_name = descriptor.file_name
table_name = descriptor.table_name
model_checksum = descriptor.get_checksum()
self.logger.info(f"{descriptor_file_name}: Unseen before now.")
self.load_model_to_data_store(descriptor)
# get the databases checksum value
stored_checksum = self.db.get_model_checksum(table_name)
# if there is no checksum in the data base
# or the database checksum does not equal this files checksum
if (stored_checksum is None
or stored_checksum.model_checksum != model_checksum):
# perform a revision
self.db.revision(table_name)
self.db.upgrade()
self.db.add_model_checksum(table_name, model_checksum,
descriptor.descriptor)
except Exception:
self.logger.exception("Error checking data resource")
def test_load():
# does not raise an error
_ = Descriptor(frameworks_descriptor)
def test_descriptor_with_file_name():
desc = Descriptor(frameworks_descriptor, "asdf.json")
file_name = desc.file_name
expect(file_name).to(equal("asdf.json"))
def test_get_api_schema():
desc = Descriptor(frameworks_descriptor)
api_schema = desc.api_schema
expect(api_schema).to(equal(frameworks_descriptor["api"]["methods"][0]))
def test_get_table_schema():
desc = Descriptor(frameworks_descriptor)
table_schema = desc.table_schema
expect(table_schema).to(equal(
frameworks_descriptor["datastore"]["schema"]))
def test_get_table_name():
desc = Descriptor(frameworks_descriptor)
table_name = desc.table_name
expect(table_name).to(
equal(frameworks_descriptor["datastore"]["tablename"]))