def output_hps(self, id=None, hps=None):
"""Save model desc and performance.
:param id: model desc id, usually worker id.
:type id: int or str.
:param hps: hyper parameters.
:type hps: json.
"""
if id is None:
id = self.worker_id
if hps is None:
if not hasattr(self, "hps"):
logger.error(
"Failed to save hyperparameters, param 'hps' is not assigned."
)
return
hps = self.hps
_file = FileOps.join_path(self.local_output_path, self.step_name,
"hyperparameters.json")
FileOps.make_base_dir(_file)
try:
with open(_file, "w") as f:
json.dump({str(id): hps}, f)
except Exception as ex:
logger.error(
"Failed to save hyperparameters, file={}, hps={}, msg={}".
format(_file, hps, str(ex)))
return
def output_evaluate_result(self,
id=None,
performance=None,
evaluate_type="gpu"):
"""Save model performance.
:param id: model desc id, usally worker id instead.
:type id: int or str.
:param performance: performance value, eg. {"accuracy": 98.23}.
:type performance: json.
:param evaluate_type: evaluate type, eg. "gpu", "davinci", "arm".
:type evaluate_type: str.
"""
if performance is None:
return
if id is None:
id = self.worker_id
_file = FileOps.join_path(
self.local_output_path, self.step_name,
"performance_{}_{}.txt".format(evaluate_type, str(id)))
FileOps.make_base_dir(_file)
try:
performance = str(performance)
with open(_file, "w") as f:
f.write(performance)
except Exception as ex:
logger.error(
"Failed to save performance, file={}, pfm={}, msg={}".format(
_file, performance, str(ex)))
return
def output_model_desc(self, id=None, model_desc=None, performance=None):
"""Save model desc and performance.
:param id: model desc id, usally worker id instead.
:type id: int or str.
:param model_desc: model description.
:type model_desc: json.
:param performance: performance value, eg. {"accuracy": 98.23}.
:type performance: json.
"""
if id is None:
id = self.worker_id
if model_desc is None:
if not hasattr(self, "model_desc"):
logger.error(
"Failed to save model desc, param 'model_desc' is not assigned."
)
return
model_desc = self.model_desc
_file = FileOps.join_path(self.local_output_path, self.step_name,
"model_desc_{}.json".format(str(id)))
FileOps.make_base_dir(_file)
try:
with open(_file, "w") as f:
json.dump(model_desc, f)
except Exception as ex:
logger.error(
"Failed to save model desc, file={}, desc={}, msg={}".format(
_file, model_desc, str(ex)))
return
if performance is not None:
self.output_evaluate_result(id, performance)
def _append_record_to_csv(self,
record_name=None,
step_name=None,
record=None,
mode='a'):
"""Transfer record to csv file."""
local_output_path = os.path.join(TaskOps().local_output_path,
step_name)
logging.debug(
"recode to csv, local_output_path={}".format(local_output_path))
if not record_name and os.path.exists(local_output_path):
return
file_path = os.path.join(local_output_path,
"{}.csv".format(record_name))
FileOps.make_base_dir(file_path)
try:
for key in record:
if isinstance(record[key], dict) or isinstance(
record[key], list):
record[key] = str(record[key])
data = pd.DataFrame([record])
if not os.path.exists(file_path):
data.to_csv(file_path, index=False)
elif os.path.exists(file_path) and os.path.getsize(
file_path) and mode == 'a':
data.to_csv(file_path, index=False, mode=mode, header=0)
else:
data.to_csv(file_path, index=False, mode=mode)
except Exception as ex:
logging.info(
'Can not transfer record to csv file Error: {}'.format(ex))
def _save_model_desc_file(self, id, desc):
output_path = TaskOps(UserConfig().data.general).local_output_path
desc_file = os.path.join(output_path, "nas",
"model_desc_{}.json".format(id))
FileOps.make_base_dir(desc_file)
output = {}
for key in desc:
if key in ["type", "modules", "custom"]:
output[key] = desc[key]
with open(desc_file, "w") as f:
json.dump(output, f)
def output_model(self,
id=None,
model=None,
model_desc=None,
performance=None):
"""Save model, model description, performance.
:param id: model desc id, usually worker id.
:type id: int or str.
:param model: hyper parameters.
:type hps: json.
"""
if id is None:
id = self.worker_id
if model is None:
if not hasattr(self, "model"):
logger.error(
"Failed to save model, param 'model' is not assigned.")
return
model = self.model
if model_desc is None:
if not hasattr(self, "model_desc"):
logger.error(
"Failed to save model, param 'model_desc' is not assigned."
)
return
model_desc = self.model_desc
_pth_file = FileOps.join_path(self.local_output_path, self.step_name,
"model_{}.pth".format(id))
FileOps.make_base_dir(_pth_file)
try:
torch.save(model.state_dict(), _pth_file)
except Exception as ex:
logger.error("Failed to save model pth, file={}, msg={}".format(
_pth_file, str(ex)))
self.output_model_desc(id, model_desc, performance)
