def _split_sample(sample):
"""Split sample to (id, model_desc, hps)."""
if len(sample) not in [2, 3]:
raise Exception("Incorrect sample length, sample: {}".format(sample))
if len(sample) == 3:
return sample[0], remove_np_value(sample[1]), remove_np_value(
sample[2])
if len(sample) == 2:
mixed = deepcopy(sample[1])
hps = {}
for key in ["trainer", "dataset"]:
if key in mixed:
hps[key] = mixed[key]
mixed.pop(key)
return sample[0], remove_np_value(mixed), remove_np_value(hps)
def from_dict(cls, src_dic):
"""Create report class from dict."""
src_cls = cls()
if src_dic:
for key, value in src_dic.items():
setattr(src_cls, key, remove_np_value(value))
return src_cls
def performance(self, value):
"""Set performance and parse value into dict."""
if isinstance(value, str):
value = json.loads(value)
value = remove_np_value(value)
self._performance = value
self._cal_rewards()
def _save_worker_record(cls, record):
step_name = record.get('step_name')
worker_id = record.get('worker_id')
_path = TaskOps().get_local_worker_path(step_name, worker_id)
for record_name in ["desc", "hps", "performance"]:
_file_name = None
_file = None
record_value = remove_np_value(record.get(record_name))
if not record_value:
continue
_file = None
try:
# for cars/darts save multi-desc
if isinstance(record_value, list) and record_name == "desc":
for idx, value in enumerate(record_value):
_file_name = "desc_{}.json".format(idx)
_file = FileOps.join_path(_path, _file_name)
with open(_file, "w") as f:
json.dump(value, f)
else:
_file_name = None
if record_name == "desc":
_file_name = "desc_{}.json".format(worker_id)
if record_name == "hps":
_file_name = "hps_{}.json".format(worker_id)
if record_name == "performance":
_file_name = "performance_{}.json".format(worker_id)
_file = FileOps.join_path(_path, _file_name)
with open(_file, "w") as f:
json.dump(record_value, f)
except Exception as ex:
logging.error("Failed to save {}, file={}, desc={}, msg={}".format(
record_name, _file, record_value, str(ex)))
def init(self, step_name, worker_id, desc=None, hps=None, **kwargs):
"""Set reord initial values."""
self.step_name = step_name
self.worker_id = worker_id
self.desc = desc
self.hps = hps
for key in kwargs:
setattr(self, key, remove_np_value(kwargs[key]))
return self
def to_dict(self):
"""Convert to dictionary."""
all_attr = {}
for _name in dir(self):
if _name.startswith("_"):
continue
if _name in self.__dict__:
all_attr[_name] = self.__dict__[_name]
elif "_" + _name in self.__dict__:
all_attr[_name] = self.__dict__["_" + _name]
all_attr["original_rewards"] = self._original_rewards
all_attr = remove_np_value(all_attr)
return all_attr
def load_dict(self, src_dic):
"""Load values from dict."""
if src_dic:
for key, value in src_dic.items():
setattr(self, key, remove_np_value(value))
return self
def hps(self, value):
"""Set hps and parse value into dict."""
if isinstance(value, str):
value = json.loads(value)
value = remove_np_value(value)
self._hps = value
def desc(self, value):
"""Set desc and parse value into dict."""
if isinstance(value, str):
value = json.loads(value)
value = remove_np_value(value)
self._desc = value