def read_metrics_from_db(self,
metric_namespace: str,
metric_name: str,
data_type,
job_level=False):
metrics = []
try:
tracking_metric_model = self.get_dynamic_db_model(
TrackingMetric, self.job_id)
tracking_metrics = tracking_metric_model.select(
tracking_metric_model.f_key,
tracking_metric_model.f_value).where(
tracking_metric_model.f_job_id == self.job_id,
tracking_metric_model.f_component_name == (
self.component_name if not job_level else
job_utils.job_virtual_component_name()),
tracking_metric_model.f_role == self.role,
tracking_metric_model.f_party_id == self.party_id,
tracking_metric_model.f_metric_namespace ==
metric_namespace,
tracking_metric_model.f_metric_name == metric_name,
tracking_metric_model.f_type == data_type)
for tracking_metric in tracking_metrics:
yield deserialize_b64(tracking_metric.f_key), deserialize_b64(
tracking_metric.f_value)
except Exception as e:
schedule_logger(self.job_id).exception(e)
raise e
return metrics
def save_table_meta(self, meta):
schedule_logger(self.job_id).info(f'start save table meta:{meta}')
address = storage.StorageTableMeta.create_address(storage_engine=meta.get("engine"),
address_dict=meta.get("address"))
table_meta = storage.StorageTableMeta(name=meta.get("name"), namespace=meta.get("namespace"), new=True)
table_meta.set_metas(**meta)
meta["address"] = address
meta["part_of_data"] = deserialize_b64(meta["part_of_data"])
meta["schema"] = deserialize_b64(meta["schema"])
table_meta.create()
schedule_logger(self.job_id).info(f'save table meta success')
def python_value(self, value):
if self._serialized_type == SerializedType.PICKLE:
return deserialize_b64(value)
elif self._serialized_type == SerializedType.JSON:
if value is None:
return {}
return json_loads(value, object_hook=self._object_hook, object_pairs_hook=self._object_pairs_hook)
else:
raise ValueError(f"the serialized type {self._serialized_type} is not supported")
def get_table_meta(self, table_name, table_namespace):
request_body = {"table_name": table_name, "namespace": table_namespace}
response = api_utils.local_api(job_id=self.job_id,
method='POST',
endpoint='/tracker/{}/{}/{}/{}/{}/{}/table_meta/get'.format(
self.job_id,
self.component_name,
self.task_id,
self.task_version,
self.role,
self.party_id),
json_body=request_body)
if response['retcode'] != RetCode.SUCCESS:
raise Exception(f"create table meta failed:{response['retmsg']}")
else:
data_table_meta = storage.StorageTableMeta(name=table_name,
namespace=table_namespace, new=True)
data_table_meta.set_metas(**response["data"])
data_table_meta.address = storage.StorageTableMeta.create_address(storage_engine=response["data"].get("engine"),
address_dict=response["data"].get("address"))
data_table_meta.part_of_data = deserialize_b64(data_table_meta.part_of_data)
data_table_meta.schema = deserialize_b64(data_table_meta.schema)
return data_table_meta
def save_metric_meta(job_id, component_name, task_version, task_id, role,
party_id):
request_data = request.json
tracker = Tracker(job_id=job_id,
component_name=component_name,
task_id=task_id,
task_version=task_version,
role=role,
party_id=party_id)
metric_meta = deserialize_b64(request_data['metric_meta'])
tracker.save_metric_meta(metric_namespace=request_data['metric_namespace'],
metric_name=request_data['metric_name'],
metric_meta=metric_meta,
job_level=request_data['job_level'])
return get_json_result()
def read_summary_from_db(self):
try:
summary_model = self.get_dynamic_db_model(ComponentSummary,
self.job_id)
summary = summary_model.get_or_none(
summary_model.f_job_id == self.job_id,
summary_model.f_component_name == self.component_name,
summary_model.f_role == self.role,
summary_model.f_party_id == self.party_id)
if summary:
cpn_summary = deserialize_b64(summary.f_summary)
else:
cpn_summary = ""
except Exception as e:
schedule_logger(self.job_id).exception(e)
raise e
return cpn_summary
def restore(self, model_id: str, model_version: str, store_address: dict):
"""
Restore model from mysql to local cache
:param model_id:
:param model_version:
:param store_address:
:return:
"""
model = PipelinedModel(model_id, model_version)
self.get_connection(store_address)
try:
with DB.connection_context():
models_in_tables = MachineLearningModel.select().where(
MachineLearningModel.f_model_id == model_id,
MachineLearningModel.f_model_version == model_version,
).order_by(MachineLearningModel.f_slice_index)
if not models_in_tables:
raise ValueError(f"Cannot found model in table.")
model_archive_data = b''.join(
deserialize_b64(models_in_table.f_content)
for models_in_table in models_in_tables)
if not model_archive_data:
raise ValueError(f"Cannot get model archive data.")
with open(model.archive_model_file_path, "wb") as fw:
fw.write(model_archive_data)
model.unpack_model(model.archive_model_file_path)
except Exception as e:
LOGGER.exception(e)
raise Exception(
f"Restore model {model_id} {model_version} from mysql failed.")
else:
LOGGER.info(
f"Restore model to {model.archive_model_file_path} from mysql successfully."
)
finally:
self.close_connection()
def restore(self, model_id: str, model_version: str, store_address: dict):
"""
Restore model from mysql to local cache
:param model_id:
:param model_version:
:param store_address:
:return:
"""
try:
self.get_connection(config=store_address)
model = PipelinedModel(model_id=model_id, model_version=model_version)
with DB.connection_context():
models_in_tables = MachineLearningModel.select().where(MachineLearningModel.f_model_id == model_id,
MachineLearningModel.f_model_version == model_version).\
order_by(MachineLearningModel.f_slice_index)
if not models_in_tables:
raise Exception("Restore model {} {} from mysql failed: {}".format(
model_id, model_version, "can not found model in table"))
f_content = ''
for models_in_table in models_in_tables:
if not f_content:
f_content = models_in_table.f_content
else:
f_content += models_in_table.f_content
model_archive_data = deserialize_b64(f_content)
if not model_archive_data:
raise Exception("Restore model {} {} from mysql failed: {}".format(
model_id, model_version, "can not get model archive data"))
with open(model.archive_model_file_path, "wb") as fw:
fw.write(model_archive_data)
model.unpack_model(model.archive_model_file_path)
LOGGER.info("Restore model to {} from mysql successfully".format(model.archive_model_file_path))
self.close_connection()
except Exception as e:
LOGGER.exception(e)
raise Exception("Restore model {} {} from mysql failed".format(model_id, model_version))
def python_value(self, value):
return deserialize_b64(value)
