def _monitor_thread(report_server):
while report_server and report_server._thread_runing:
watched_vars = deepcopy(ReportServer.__variables__)
saved_records = report_server.all_records
for var in watched_vars:
step_name, worker_id = var.split(".")
if step_name != General.step_name:
continue
record_dict = None
try:
record_dict = ShareMemory(var).get()
except:
logging.warn(
"Failed to get record, step name: {}, worker id: {}.".
format(step_name, worker_id))
if record_dict:
record = ReportRecord().from_dict(record_dict)
code = record.code
saved_record = list(
filter(
lambda x: x.step_name == step_name and str(
x.worker_id) == str(worker_id), saved_records))
if not saved_record or record.code != saved_record[0].code:
report_server.add(record)
ReportServer().dump()
ShareMemory(var).close()
time.sleep(0.2)
def update_report(self, worker_info):
"""Get finished worker's info, and use it to update target `generator`.
Will get the finished worker's working dir, and then call the function
`report.update(worker_info)`.
:param worker_info: `worker_info` is the finished worker's info, usually
a dict or list of dict include `step_name` and `worker_id`.
:type worker_info: dict or list of dict.
"""
if worker_info is None:
return
if not isinstance(worker_info, list):
worker_info = [worker_info]
for one_info in worker_info:
step_name = one_info["step_name"]
worker_id = one_info["worker_id"]
logging.info("update report, step name: {}, worker id: {}".format(
step_name, worker_id))
try:
result = ShareMemory("{}.{}".format(step_name,
worker_id)).get()
record = ReportRecord().from_dict(result)
self.add(record)
self.dump_report(step_name, record)
except Exception:
logging.error(
"Failed to upgrade report, step_name={}, worker_id={}.".
format(step_name, worker_id))
logging.error(traceback.format_exc())
def broadcast(cls, record):
"""Broadcast one record to Shared Memory."""
if not record:
logging.warning("Broadcast Record is None.")
return
ShareMemory("{}.{}".format(record.step_name, record.worker_id)).put(record.serialize())
cls._save_worker_record(record.serialize())
def get_record(cls, step_name, worker_id):
"""Get value from Shared Memory."""
value = ShareMemory("{}.{}".format(step_name, worker_id)).get()
if value:
record = ReportRecord().from_dict(value)
else:
record = ReportRecord(step_name, worker_id)
return record
def close(cls, step_name, worker_id):
"""Clear Shared Memory."""
ShareMemory("{}.{}".format(step_name, worker_id)).close()