def _log_end_of_run_report(self):
missing_models = missing_model_hashes(self.experiment_hash,
self.db_engine)
if len(missing_models) > 0:
logging.info(
"Found %s missing model hashes."
"This means that they were supposed to either be trained or reused"
"by this experiment but are not present in the models table."
"Inspect the logs for any training errors. Full list: %s",
len(missing_models), missing_models)
else:
logging.info(
"All models that were supposed to be trained were trained. Awesome!"
)
missing_matrices = missing_matrix_uuids(self.experiment_hash,
self.db_engine)
if len(missing_matrices) > 0:
logging.info(
"Found %s missing matrix uuids."
"This means that they were supposed to either be build or reused"
"by this experiment but are not present in the matrices table."
"Inspect the logs for any matrix building errors. Full list: %s",
len(missing_matrices), missing_matrices)
else:
logging.info(
"All matrices that were supposed to be build were built. Awesome!"
)
def test_missing_model_hashes():
with testing.postgresql.Postgresql() as postgresql:
db_engine = create_engine(postgresql.url())
ensure_db(db_engine)
experiment_hash = save_experiment_and_get_hash({}, 1234, db_engine)
model_hashes = ['abcd', 'bcde', 'cdef']
# if we associate model hashes with an experiment but don't actually train the models
# they should show up as missing
associate_models_with_experiment(experiment_hash, model_hashes, db_engine)
assert missing_model_hashes(experiment_hash, db_engine) == model_hashes
# if we insert a model row they should no longer be considered missing
db_engine.execute(
f"insert into {Model.__table__.fullname} (model_hash) values (%s)",
model_hashes[0]
)
assert missing_model_hashes(experiment_hash, db_engine) == model_hashes[1:]