def dump(self): """Dump report to file.""" try: _file = FileOps.join_path(TaskOps().step_path, "reports.csv") FileOps.make_base_dir(_file) data = self.all_records data_dict = {} for step in data: step_data = step.serialize().items() for k, v in step_data: if k in data_dict: data_dict[k].append(v) else: data_dict[k] = [v] data = pd.DataFrame(data_dict) data.to_csv(_file, index=False) _file = os.path.join(TaskOps().step_path, ".reports") _dump_data = [ ReportServer._hist_records, ReportServer.__instances__ ] with open(_file, "wb") as f: pickle.dump(_dump_data, f, protocol=pickle.HIGHEST_PROTOCOL) self.backup_output_path() except Exception: logging.warning(traceback.format_exc())
def _append_record_to_csv(self, record_name=None, step_name=None, record=None, mode='a'): """Transfer record to csv file.""" local_output_path = os.path.join(TaskOps().local_output_path, step_name) logging.debug( "recode to csv, local_output_path={}".format(local_output_path)) if not record_name and os.path.exists(local_output_path): return file_path = os.path.join(local_output_path, "{}.csv".format(record_name)) FileOps.make_base_dir(file_path) try: for key in record: if isinstance(record[key], dict) or isinstance( record[key], list): record[key] = str(record[key]) data = pd.DataFrame([record]) if not os.path.exists(file_path): data.to_csv(file_path, index=False) elif os.path.exists(file_path) and os.path.getsize( file_path) and mode == 'a': data.to_csv(file_path, index=False, mode=mode, header=0) else: data.to_csv(file_path, index=False, mode=mode) except Exception as ex: logging.info( 'Can not transfer record to csv file Error: {}'.format(ex))
def _copy_needed_file(self): if self.config.pareto_front_file is None: raise FileNotFoundError( "Config item paretor_front_file not found in config file.") init_pareto_front_file = self.config.pareto_front_file.replace( "{local_base_path}", self.local_base_path) self.pareto_front_file = FileOps.join_path( self.local_output_path, self.step_name, "pareto_front.csv") FileOps.make_base_dir(self.pareto_front_file) FileOps.copy_file(init_pareto_front_file, self.pareto_front_file) if self.config.random_file is None: raise FileNotFoundError( "Config item random_file not found in config file.") init_random_file = self.config.random_file.replace( "{local_base_path}", self.local_base_path) self.random_file = FileOps.join_path( self.local_output_path, self.step_name, "random.csv") FileOps.copy_file(init_random_file, self.random_file)
def dump(self): """Dump report to file.""" try: _file = FileOps.join_path(TaskOps().step_path, "reports.json") FileOps.make_base_dir(_file) data = {} for record in self.all_records: if record.step_name in data: data[record.step_name].append(record.to_dict()) else: data[record.step_name] = [record.to_dict()] with open(_file, "w") as f: json.dump(data, f, indent=4) _file = os.path.join(TaskOps().step_path, ".reports") _dump_data = [ ReportServer._hist_records, ReportServer.__instances__ ] with open(_file, "wb") as f: pickle.dump(_dump_data, f, protocol=pickle.HIGHEST_PROTOCOL) self.backup_output_path() except Exception: logging.warning(traceback.format_exc())