def dump(self):
"""Dump report to file."""
try:
_file = FileOps.join_path(TaskOps().step_path, "reports.csv")
FileOps.make_base_dir(_file)
data = self.all_records
data_dict = {}
for step in data:
step_data = step.serialize().items()
for k, v in step_data:
if k in data_dict:
data_dict[k].append(v)
else:
data_dict[k] = [v]
data = pd.DataFrame(data_dict)
data.to_csv(_file, index=False)
_file = os.path.join(TaskOps().step_path, ".reports")
_dump_data = [
ReportServer._hist_records, ReportServer.__instances__
]
with open(_file, "wb") as f:
pickle.dump(_dump_data, f, protocol=pickle.HIGHEST_PROTOCOL)
self.backup_output_path()
except Exception:
logging.warning(traceback.format_exc())
def _append_record_to_csv(self,
record_name=None,
step_name=None,
record=None,
mode='a'):
"""Transfer record to csv file."""
local_output_path = os.path.join(TaskOps().local_output_path,
step_name)
logging.debug(
"recode to csv, local_output_path={}".format(local_output_path))
if not record_name and os.path.exists(local_output_path):
return
file_path = os.path.join(local_output_path,
"{}.csv".format(record_name))
FileOps.make_base_dir(file_path)
try:
for key in record:
if isinstance(record[key], dict) or isinstance(
record[key], list):
record[key] = str(record[key])
data = pd.DataFrame([record])
if not os.path.exists(file_path):
data.to_csv(file_path, index=False)
elif os.path.exists(file_path) and os.path.getsize(
file_path) and mode == 'a':
data.to_csv(file_path, index=False, mode=mode, header=0)
else:
data.to_csv(file_path, index=False, mode=mode)
except Exception as ex:
logging.info(
'Can not transfer record to csv file Error: {}'.format(ex))
def _copy_needed_file(self):
if self.config.pareto_front_file is None:
raise FileNotFoundError(
"Config item paretor_front_file not found in config file.")
init_pareto_front_file = self.config.pareto_front_file.replace(
"{local_base_path}", self.local_base_path)
self.pareto_front_file = FileOps.join_path(
self.local_output_path, self.step_name, "pareto_front.csv")
FileOps.make_base_dir(self.pareto_front_file)
FileOps.copy_file(init_pareto_front_file, self.pareto_front_file)
if self.config.random_file is None:
raise FileNotFoundError(
"Config item random_file not found in config file.")
init_random_file = self.config.random_file.replace(
"{local_base_path}", self.local_base_path)
self.random_file = FileOps.join_path(
self.local_output_path, self.step_name, "random.csv")
FileOps.copy_file(init_random_file, self.random_file)
def dump(self):
"""Dump report to file."""
try:
_file = FileOps.join_path(TaskOps().step_path, "reports.json")
FileOps.make_base_dir(_file)
data = {}
for record in self.all_records:
if record.step_name in data:
data[record.step_name].append(record.to_dict())
else:
data[record.step_name] = [record.to_dict()]
with open(_file, "w") as f:
json.dump(data, f, indent=4)
_file = os.path.join(TaskOps().step_path, ".reports")
_dump_data = [
ReportServer._hist_records, ReportServer.__instances__
]
with open(_file, "wb") as f:
pickle.dump(_dump_data, f, protocol=pickle.HIGHEST_PROTOCOL)
self.backup_output_path()
except Exception:
logging.warning(traceback.format_exc())