def test_query_entities_when_partition_key_invalid(self):
# Expecting result to be empty when partition key is invalid
result = storage_helpers.query_entities(
self.table_service,
self.table_status,
"abcd")
assert len(result) == 0
def test_query_entities_when_table_invalid(self):
# Expecting failure when table is invalid
result = storage_helpers.query_entities(
self.table_service,
"abcd",
self.label)
assert result == None
def test_query_entities_when_valid(self):
# Expecting success when all parameters are valid
result = storage_helpers.query_entities(
self.table_service,
self.table_status,
self.label)
assert result != None
def test_create_table_service_when_key_invalid(self):
# Expecting failure when storage account key is invalid
table_service = storage_helpers.create_table_service(
self.storage_account,
"abcd")
result = storage_helpers.query_entities(table_service, self.table, self.label)
assert result == None
def test_create_table_service_when_account_invalid(self):
# Expecting failure when storage account name is invalid
table_service = storage_helpers.create_table_service(
"aezaehzajebvheahez",
self.storage_key)
result = storage_helpers.query_entities(table_service, self.table, self.label)
assert result == None
def test_create_table_service_when_valid(self):
# Expecting success when all parameters are valid
table_service = storage_helpers.create_table_service(
self.storage_account,
self.storage_key)
result = storage_helpers.query_entities(table_service, self.table, self.label)
assert result != None
def run(self, doctype, train_supervised, train_unsupervised):
logging.info(
f"Training will start. Training supervised: {train_supervised}, Training unsupervised: {train_unsupervised}"
)
training_data_path = self.get_training_data_path()
result_supervised = None
result_unsupervised = None
if train_unsupervised == 'True':
# Training unsupervised
result_unsupervised = self.train_and_save(doctype,
training_data_path,
False)
entities_status = storage_helpers.query_entities(
self.table_service, self.app_settings.status_table, doctype)
# If all files are processed, we can train the model
if self.files_processed(entities_status):
logging.info(
f"All files processed for doc type {doctype}, supervised training will start..."
)
if train_supervised == 'True':
# Training supervised
result_supervised = self.train_and_save(
doctype, training_data_path, True)
if result_supervised != None and result_unsupervised != None:
return True, {
"text": "Training finished.",
"modelId_supervised": result_supervised['model_id'],
"status_supervised": result_supervised['status'],
"avgAccuracy": result_supervised['accuracy'],
"fieldsAccuracy": result_supervised['fields_accuracy'],
"modelId_unsupervised": result_unsupervised['model_id'],
"status_unsupervised": result_unsupervised['status']
}
elif result_supervised != None and result_unsupervised == None:
return True, {
"text":
"Training supervised finished, training unsupervised failed.",
"modelId_supervised": result_supervised['model_id'],
"status": result_supervised['status'],
"avgAccuracy": result_supervised['accuracy'],
"fieldsAccuracy": result_supervised['fields_accuracy']
}
elif result_supervised == None and result_unsupervised != None:
return True, {
"text":
"Training unsupervised finished, training supervised failed.",
"modelId_unsupervised": result_unsupervised['model_id'],
"status_unsupervised": result_unsupervised['status']
}
else:
raise EnvironmentError("Error during training.")
else:
logging.info(
f"Not all files are done processing for doc type {doctype}.")
raise Warning(
f"Processing not finished ({str(entities_status.count('done'))}/{str(len(entities_status))} files processed). \n Please retry later.",
)