def test_get_latest_dataframe_id(self):
df_id_1 = "a36d6fc9-49d0-4789-a7dd-a2b72369ca45"
df_metadata_dict_1 = self.get_dataframe_metadata(df_id_1)
df_1 = utils.create_dataframe(schema.convert_dataframe_metadata_to_row(df_metadata_dict_1),
schema.dataframe_metadata_schema)
df_id_2 = "bbbd6fc9-49d0-4789-a7dd-a2b72369ca45"
df_metadata_dict_2 = self.get_dataframe_metadata(df_id_2)
df_2 = utils.create_dataframe(schema.convert_dataframe_metadata_to_row(df_metadata_dict_2),
schema.dataframe_metadata_schema)
df_metadata = df_1.union(df_2)
expected_dataframe_id = train_models.get_latest_dataframe_id(df_metadata)
self.assertEqual(expected_dataframe_id, df_id_2)
def save_dataframe_metadata_to_hdfs(metadata):
""" Save dataframe metadata.
"""
# Convert metadata to row object.
metadata_row = schema.convert_dataframe_metadata_to_row(metadata)
try:
# Create dataframe from the row object.
dataframe_metadata = utils.create_dataframe(metadata_row, schema.dataframe_metadata_schema)
except DataFrameNotCreatedException as err:
current_app.logger.error(str(err), exc_info=True)
raise
try:
# Append the dataframe to existing dataframe if already exists or create a new one.
utils.append(dataframe_metadata, path.DATAFRAME_METADATA)
except DataFrameNotAppendedException as err:
current_app.logger.error(str(err), exc_info=True)
raise
def save_dataframe_metadata_to_hdfs(metadata: dict, df_metadata_path: str):
""" Save dataframe metadata.
Args:
metadata (dict): metadata dataframe to append.
df_metadata_path (str): path where metadata dataframe should be saved.
"""
# Convert metadata to row object.
metadata_row = schema.convert_dataframe_metadata_to_row(metadata)
try:
# Create dataframe from the row object.
dataframe_metadata = utils.create_dataframe(metadata_row, schema.dataframe_metadata_schema)
except DataFrameNotCreatedException as err:
current_app.logger.error(str(err), exc_info=True)
raise
try:
# Append the dataframe to existing dataframe if already exists or create a new one.
utils.append(dataframe_metadata, df_metadata_path)
except DataFrameNotAppendedException as err:
current_app.logger.error(str(err), exc_info=True)
raise
