def test_get_most_recent_model_id(self):
model_id_1 = "a36d6fc9-49d0-4789-a7dd-a2b72369ca45"
model_metadata_dict_1 = self.get_model_metadata(model_id_1)
df_1 = utils.create_dataframe(schema.convert_model_metadata_to_row(model_metadata_dict_1),
schema.model_metadata_schema)
model_id_2 = "bbbd6fc9-49d0-4789-a7dd-a2b72369ca45"
model_metadata_dict_2 = self.get_model_metadata(model_id_2)
df_2 = utils.create_dataframe(schema.convert_model_metadata_to_row(model_metadata_dict_2),
schema.model_metadata_schema)
model_metadata = df_1.union(df_2)
utils.save_parquet(model_metadata, path.RECOMMENDATION_RECORDING_MODEL_METADATA)
expected_model_id = recommend.get_most_recent_model_id()
self.assertEqual(expected_model_id, model_id_2)
def save_dataframe_metadata_to_HDFS(metadata):
""" Save dataframe metadata to model_metadata dataframe.
"""
# Convert metadata to row object.
metadata_row = schema.convert_model_metadata_to_row(metadata)
try:
# Create dataframe from the row object.
dataframe_metadata = utils.create_dataframe(
metadata_row, schema.model_metadata_schema)
except DataFrameNotCreatedException as err:
current_app.logger.error(str(err), exc_info=True)
sys.exit(-1)
try:
# Append the dataframe to existing dataframe if already exist or create a new one.
utils.append(dataframe_metadata, path.MODEL_METADATA)
except DataFrameNotAppendedException as err:
current_app.logger.error(str(err), exc_info=True)
sys.exit(-1)
def save_model_metadata_to_hdfs(metadata):
""" Save model metadata.
Args:
metadata: dict containing model metadata.
"""
metadata_row = schema.convert_model_metadata_to_row(metadata)
try:
# Create dataframe from the row object.
model_metadata_df = utils.create_dataframe(metadata_row, schema.model_metadata_schema)
except DataFrameNotCreatedException as err:
current_app.logger.error(str(err), exc_info=True)
raise
try:
current_app.logger.info('Saving model metadata...')
# Append the dataframe to existing dataframe if already exist or create a new one.
utils.append(model_metadata_df, path.MODEL_METADATA)
current_app.logger.info('Model metadata saved...')
except DataFrameNotAppendedException as err:
current_app.logger.error(str(err), exc_info=True)
raise