def test_get_most_recent_model_id(self): model_id_1 = "a36d6fc9-49d0-4789-a7dd-a2b72369ca45" model_metadata_dict_1 = self.get_model_metadata(model_id_1) df_1 = utils.create_dataframe(schema.convert_model_metadata_to_row(model_metadata_dict_1), schema.model_metadata_schema) model_id_2 = "bbbd6fc9-49d0-4789-a7dd-a2b72369ca45" model_metadata_dict_2 = self.get_model_metadata(model_id_2) df_2 = utils.create_dataframe(schema.convert_model_metadata_to_row(model_metadata_dict_2), schema.model_metadata_schema) model_metadata = df_1.union(df_2) utils.save_parquet(model_metadata, path.RECOMMENDATION_RECORDING_MODEL_METADATA) expected_model_id = recommend.get_most_recent_model_id() self.assertEqual(expected_model_id, model_id_2)
def save_dataframe_metadata_to_HDFS(metadata): """ Save dataframe metadata to model_metadata dataframe. """ # Convert metadata to row object. metadata_row = schema.convert_model_metadata_to_row(metadata) try: # Create dataframe from the row object. dataframe_metadata = utils.create_dataframe( metadata_row, schema.model_metadata_schema) except DataFrameNotCreatedException as err: current_app.logger.error(str(err), exc_info=True) sys.exit(-1) try: # Append the dataframe to existing dataframe if already exist or create a new one. utils.append(dataframe_metadata, path.MODEL_METADATA) except DataFrameNotAppendedException as err: current_app.logger.error(str(err), exc_info=True) sys.exit(-1)
def save_model_metadata_to_hdfs(metadata): """ Save model metadata. Args: metadata: dict containing model metadata. """ metadata_row = schema.convert_model_metadata_to_row(metadata) try: # Create dataframe from the row object. model_metadata_df = utils.create_dataframe(metadata_row, schema.model_metadata_schema) except DataFrameNotCreatedException as err: current_app.logger.error(str(err), exc_info=True) raise try: current_app.logger.info('Saving model metadata...') # Append the dataframe to existing dataframe if already exist or create a new one. utils.append(model_metadata_df, path.MODEL_METADATA) current_app.logger.info('Model metadata saved...') except DataFrameNotAppendedException as err: current_app.logger.error(str(err), exc_info=True) raise