def _init_model(self, model=None):
"""Load model desc from save path and parse to model."""
if model is not None:
return model
model_cfg = ClassFactory.__configs__.get('model')
if 'model_desc_file' in model_cfg and model_cfg.model_desc_file is not None:
desc_file = model_cfg.model_desc_file.replace(
"{model_zoo}", self.model_zoo_path)
desc_file = desc_file.replace("{local_base_path}",
self.local_base_path)
if ":" not in desc_file:
desc_file = os.path.abspath(desc_file)
if ":" in desc_file:
local_desc_file = FileOps.join_path(
self.local_output_path, os.path.basename(desc_file))
FileOps.copy_file(desc_file, local_desc_file)
desc_file = local_desc_file
if self.horovod:
hvd.join()
model_desc = Config(desc_file)
logging.info("net_desc:{}".format(model_desc))
elif 'model_desc' in model_cfg and model_cfg.model_desc is not None:
model_desc = model_cfg.model_desc
else:
return None
if model_desc is not None:
self.model_desc = model_desc
net_desc = NetworkDesc(model_desc)
model = net_desc.to_model()
return model
else:
return None
def get_model(cls, model_desc=None, model_checkpoint=None):
"""Get model from model zoo.
:param network_name: the name of network, eg. ResNetVariant.
:type network_name: str or None.
:param network_desc: the description of network.
:type network_desc: str or None.
:param model_checkpoint: path of model.
:type model_checkpoint: str.
:return: model.
:rtype: model.
"""
try:
network = NetworkDesc(model_desc)
model = network.to_model()
except Exception as e:
logging.error("Failed to get model, model_desc={}, msg={}".format(
model_desc, str(e)))
raise e
logging.info("Model was created.")
logging.debug("model_desc={}".format(model_desc))
if model_checkpoint is not None:
logging.info("Load model with weight.")
model = cls._load_pretrained_model(network, model, model_checkpoint)
logging.info("Model was loaded.")
return model
def get_model(cls, network_desc=None, pretrained=False):
"""Get model from model zoo.
:param network_name: the name of network, eg. ResNetVariant.
:type network_name: str or None.
:param network_desc: the description of network.
:type network_desc: str or None.
:param pretrained: pre-trained model or not.
:type pretrained: bool.
:return: model.
:rtype: model.
"""
try:
network = NetworkDesc(network_desc)
model = network.to_model()
except Exception as e:
logging.error(
"Failed to get model, network_desc={}, msg={}".format(
network_desc, str(e)))
raise e
logging.info("Model was created, model_desc={}".format(network_desc))
if pretrained is True:
logging.info("Load pretrained model.")
model = cls._load_pretrained_model(network, model)
logging.info("Pretrained model was loaded.")
return model
def new_model(self):
"""Build new model."""
net_desc = NetworkDesc(self.search_space)
model_new = net_desc.to_model().cuda()
for x, y in zip(model_new.arch_parameters(),
self.model.arch_parameters()):
x.detach().copy_(y.detach())
return model_new
def _init_model(self):
"""Initialize the model architecture for full train step.
:return: train model
:rtype: class
"""
search_space = Config({"search_space": self.model_desc})
self.codec = Codec(self.cfg.codec, search_space)
self.get_selected_arch()
indiv_cfg = self.codec.decode(self.elitism)
self.trainer.model_desc = self.elitism.active_net_list()
# self.output_model_desc()
net_desc = NetworkDesc(indiv_cfg)
model = net_desc.to_model()
return model
def search(self):
"""Search one mutated model.
:return: current number of samples, and the model
"""
search_desc = self.search_space.search_space.custom
pareto_front_folder = FileOps.join_path(self.local_base_path, "result")
if 'pareto_folder' in self.search_space.cfg and self.search_space.cfg.pareto_folder is not None:
pareto_front_folder = self.search_space.cfg.pareto_folder.replace(
"{local_base_path}", self.local_base_path)
pareto_front_df = pd.read_csv(
FileOps.join_path(pareto_front_folder, "pareto_front.csv"))
code_to_mutate = random.choice(pareto_front_df['Code'])
current_mutate, code_mutated = 0, code_to_mutate
num_candidates = len(search_desc["candidates"])
while current_mutate < self.num_mutate:
code_new = self.mutate_once(code_mutated, num_candidates)
if code_new != code_mutated:
current_mutate += 1
code_mutated = code_new
logging.info("Mutate from {} to {}".format(code_to_mutate,
code_mutated))
search_desc['code'] = code_mutated
search_desc['method'] = "mutate"
search_desc = self.codec.decode(search_desc)
self.sample_count += 1
return self.sample_count, NetworkDesc(self.search_space.search_space)
def search(self):
"""Search one random model.
:return: current number of samples, and the model
"""
search_desc = self.search_space.search_space.custom
num_ops = len(search_desc.op_names)
ops = [random.randrange(num_ops) for _ in range(7)]
inputs = list()
for inputs_index in range(3):
for i in range(2):
inputs.append(random.randint(0, (inputs_index + 2) * 3 - 5))
conns = list()
for conns_index in range(3):
for i in range(2):
conns.append(random.randint(0, conns_index + 3))
decoder_cell_str = list()
decoder_cell_str.append(ops[0])
decoder_cell_str.append([inputs[0], inputs[1], ops[1], ops[2]])
decoder_cell_str.append([inputs[2], inputs[3], ops[3], ops[4]])
decoder_cell_str.append([inputs[4], inputs[5], ops[5], ops[6]])
decoder_conn_str = [[conns[0], conns[1]], [conns[2], conns[3]],
[conns[4], conns[5]]]
decoder_arch_str = [decoder_cell_str, decoder_conn_str]
search_desc['config'] = decoder_arch_str
search_desc['method'] = "random"
search_desc = self.codec.encode(search_desc)
self.sample_count += 1
return self.sample_count, NetworkDesc(self.search_space.search_space)
def _init_model(self):
"""Initialize the model architecture for full train step.
:return: train model
:rtype: class
"""
logging.info('Initializing model')
if 'model_desc' in self.cfg and self.cfg.model_desc is not None:
if self.horovod:
hvd.join()
model_desc = self.cfg.model_desc
self.model_desc = self.cfg.model_desc
net_desc = NetworkDesc(model_desc)
model = net_desc.to_model()
return model
else:
return None
def search(self):
idx = self.points[self.sample_count]
logging.info('Checking architecture %d' % idx)
desc = self._desc_from_choices(self.choices[idx])
self.sample_count += 1
self._save_model_desc_file(self.sample_count, desc)
return self.sample_count, NetworkDesc(desc)
def _init_model(self):
"""Initialize the model architecture for full train step.
:return: train model
:rtype: class
"""
model_cfg = ClassFactory.__configs__.get('model')
if 'model_desc' in model_cfg and model_cfg.model_desc is not None:
model_desc = model_cfg.model_desc
else:
raise ValueError('Model_desc is None for evaluator')
search_space = Config({"search_space": model_desc})
self.codec = Codec(self.cfg.codec, search_space)
self._get_selected_arch()
indiv_cfg = self.codec.decode(self.elitism)
logger.info('Model arch:{}'.format(self.elitism.active_net_list()))
self.model_desc = self.elitism.active_net_list()
net_desc = NetworkDesc(indiv_cfg)
model = net_desc.to_model()
return model
def _init_model(self):
"""Initialize the model architecture for full train step.
:return: train model
:rtype: class
"""
logging.info('Initializing model')
if self.cfg.model_desc:
logging.debug("model_desc: {}".format(self.cfg.model_desc))
_file = FileOps.join_path(
self.worker_path, "model_desc_{}.json".format(self._worker_id))
with open(_file, "w") as f:
json.dump(self.cfg.model_desc, f)
if self.cfg.distributed:
hvd.join()
model_desc = self.cfg.model_desc
net_desc = NetworkDesc(model_desc)
model = net_desc.to_model()
return model
else:
return None
def decode(self, code):
"""Decode the code to Network Desc.
:param code: input code
:type code: 2D array of float
:return: network desc
:rtype: NetworkDesc
"""
genotype = self.calc_genotype(code)
cfg_result = copy.deepcopy(self.darts_cfg)
cfg_result.super_network.normal.genotype = genotype[0]
cfg_result.super_network.reduce.genotype = genotype[1]
return NetworkDesc(cfg_result)
def _init_model(self, model=None):
"""Load model desc from save path and parse to model."""
if model is not None:
if vega.is_torch_backend() and self.use_cuda:
model = model.cuda()
return model
model_cfg = Config(ClassFactory.__configs__.get('model'))
if "model_desc_file" in model_cfg and model_cfg.model_desc_file is not None:
desc_file = model_cfg.model_desc_file
desc_file = desc_file.replace("{local_base_path}",
self.local_base_path)
if ":" not in desc_file:
desc_file = os.path.abspath(desc_file)
if ":" in desc_file:
local_desc_file = FileOps.join_path(
self.local_output_path, os.path.basename(desc_file))
FileOps.copy_file(desc_file, local_desc_file)
desc_file = local_desc_file
model_desc = Config(desc_file)
logging.info("net_desc:{}".format(model_desc))
elif "model_desc" in model_cfg and model_cfg.model_desc is not None:
model_desc = model_cfg.model_desc
elif "models_folder" in model_cfg and model_cfg.models_folder is not None:
folder = model_cfg.models_folder.replace("{local_base_path}",
self.local_base_path)
pattern = FileOps.join_path(folder, "desc_*.json")
desc_file = glob.glob(pattern)[0]
model_desc = Config(desc_file)
else:
return None
if model_desc is not None:
self.model_desc = model_desc
net_desc = NetworkDesc(model_desc)
model = net_desc.to_model()
if vega.is_torch_backend() and self.use_cuda:
model = model.cuda()
return model
else:
return None
def decode(self, code):
"""Decode the code to Network Desc.
:param code: input code
:type code: list of int
:return: network desc
:rtype: NetworkDesc
"""
chn_info = self._code_to_chninfo(code)
desc = {
"backbone": chn_info
}
desc = update_dict(desc, copy.deepcopy(self.search_space))
return NetworkDesc(desc)
def search(self):
"""Search one NetworkDesc from search space.
:return: search id, network desc
:rtype: int, NetworkDesc
"""
desc = copy.deepcopy(self.search_space.search_space)
desc.pop('type')
blocks = random.choice(desc.backbone.blocks)
channels = random.choice(desc.backbone.channels)
desc.backbone.blocks = blocks
desc.backbone.channels = channels
self.count += 1
logging.info('Search No.{} sample'.format(self.count))
return self.count, NetworkDesc(desc)
def decode(self, code):
"""Decode the code.
:param code: code of network
:type code: list
:return: network desc
:rtype: NetworkDesc
"""
length = len(code)
desc = {
"backbone": {
"nbit_w_list": code[:length // 2],
"nbit_a_list": code[length // 2:]
}
}
desc = update_dict(desc, copy.deepcopy(self.search_space))
return NetworkDesc(desc)
def search(self):
"""Search one random model.
:return: current number of samples, and the model
"""
search_desc = self.search_space.search_space.custom
num_blocks = random.randint(*search_desc.block_range)
num_cibs = random.randint(*search_desc.cib_range)
candidates = search_desc.candidates
blocks = [random.choice(candidates) for _ in range(num_blocks)]
for _ in range(num_cibs):
cib = [random.choice(candidates) for _ in range(2)]
blocks.insert(random.randint(0, len(blocks)), cib)
search_desc['blocks'] = blocks
search_desc['method'] = "random"
search_desc = self.codec.encode(search_desc)
self.sample_count += 1
return self.sample_count, NetworkDesc(self.search_space.search_space)
def _new_model_init(self, model_prune):
"""Init new model.
:param model_prune: searched pruned model
:type model_prune: torch.nn.Module
:return: initial model after loading pretrained model
:rtype: torch.nn.Module
"""
init_model_file = self.config.init_model_file
if ":" in init_model_file:
local_path = FileOps.join_path(
self.trainer.get_local_worker_path(),
os.path.basename(init_model_file))
FileOps.copy_file(init_model_file, local_path)
self.config.init_model_file = local_path
network_desc = copy.deepcopy(self.base_net_desc)
network_desc.backbone.chn = network_desc.backbone.base_chn
network_desc.backbone.chn_node = network_desc.backbone.base_chn_node
network_desc.backbone.encoding = model_prune.encoding
model_init = NetworkDesc(network_desc).to_model()
return model_init
def search(self):
"""Search one random model.
:return: current number of samples, and the model
"""
search_desc = self.search_space.search_space.custom
pareto_front_df = pd.read_csv(self.pareto_front_file)
num_ops = len(search_desc.op_names)
upper_bounds = [num_ops, 2, 2, num_ops, num_ops, 5, 5, num_ops, num_ops,
8, 8, num_ops, num_ops, 4, 4, 5, 5, 6, 6]
code_to_mutate = random.choice(pareto_front_df['Code'])
index = random.randrange(len(upper_bounds))
choices = list(range(upper_bounds[index]))
choices.pop(int(code_to_mutate[index + 1], 36))
choice = random.choice(choices)
code_mutated = code_to_mutate[:index + 1] + str(choice) + code_to_mutate[index + 2:]
search_desc['code'] = code_mutated
search_desc['method'] = "mutate"
logging.info("Mutate from {} to {}".format(code_to_mutate, code_mutated))
search_desc = self.codec.decode(search_desc)
self.sample_count += 1
return self.sample_count, NetworkDesc(self.search_space.search_space)
def search(self):
"""Search one random model.
:return: current number of samples, and the model
:rtype: int and class
"""
if self.indiv_count > 0 and self.indiv_count % self.individual_num == 0:
if np.sum(np.asarray(self.fit_state)) < self.individual_num:
return None, None
else:
self.update_fitness(self.fitness_pop)
self.update_elitism(self.fitness_pop)
self.save_results()
self.reproduction()
self.evolution_count += 1
self.fitness_pop = [0] * self.individual_num
self.fit_state = [0] * self.individual_num
current_indiv = self.pop[self.indiv_count % self.individual_num]
indiv_cfg = self.codec.decode(current_indiv)
self.indiv_count += 1
logging.info('model parameters:{}, model flops:{}'.format(current_indiv.parameter, current_indiv.flops))
logging.info('model arch:{}'.format(current_indiv.active_net_list()))
return self.indiv_count, NetworkDesc(indiv_cfg)
def search(self):
"""Search in search_space and return a sample."""
sample = {}
while sample is None or 'code' not in sample:
pareto_dict = self.pareto_front.get_pareto_front()
pareto_list = list(pareto_dict.values())
if self.pareto_front.size < self.min_sample or random.random(
) < self.random_ratio or len(pareto_list) == 0:
sample_desc = self.random_search.search()
sample = self.codec.encode(sample_desc)
else:
sample = pareto_list[0]
if sample is not None and 'code' in sample:
code = sample['code']
code = self.ea_sample(code)
sample['code'] = code
if not self.pareto_front._add_to_board(id=self.sample_count + 1,
config=sample):
sample = None
self.sample_count += 1
logging.info(sample)
sample_desc = self.codec.decode(sample)
return self.sample_count, NetworkDesc(sample_desc)
def search(self):
"""Search function."""
logging.info('====> {}.search()'.format(__name__))
self.completed = True
return self.sample_idx, NetworkDesc(self.ss.search_space)
def _init_model(self):
"""Load model desc from save path and parse to model."""
model_desc = self.cfg.model_desc
model = NetworkDesc(model_desc).to_model()
return model
def search(self):
"""Search function."""
return self.sample_idx, NetworkDesc(self.ss.search_space)