def genotype(self): PRIMITIVES = get_primitives() def get_op(w): if 'none' in PRIMITIVES: i = max([ k for k in range(len(PRIMITIVES)) if k != PRIMITIVES.index('none') ], key=lambda k: w[k]) else: i = max(range(len(PRIMITIVES)), key=lambda k: w[k]) return w[i], PRIMITIVES[i] def _parse(alphas): gene = [] start = 0 for i in range(self._steps): end = start + i + 2 W = alphas[start:end] edges = sorted(range(i + 2), key=lambda x: -get_op(W[x])[0])[:2] for j in edges: gene.append((get_op(W[j])[1], j)) start = end return gene alphas_normal = tf.nn.softmax(self.alphas_normal, axis=-1).numpy() betas_normal = beta_softmax(self.betas_normal, self._steps).numpy() alphas_normal = alphas_normal * betas_normal[:, None] alphas_reduce = tf.nn.softmax(self.alphas_reduce, axis=-1).numpy() betas_reduce = beta_softmax(self.betas_reduce, self._steps).numpy() alphas_reduce = alphas_reduce * betas_reduce[:, None] gene_normal = _parse(alphas_normal) gene_reduce = _parse(alphas_reduce) concat = range(2 + self._steps - self._multiplier, self._steps + 2) genotype = Genotype(normal=gene_normal, normal_concat=concat, reduce=gene_reduce, reduce_concat=concat) return genotype
def standardize(genotype: Genotype): if len(genotype.normal[0]) == 2: assert all(len(c) == 2 for c in genotype.normal) n = len(genotype.normal) // 2 op_indices = concat([(i, i) for i in range(2, 2 + n)]) op_names, indices = zip(*genotype.normal) normal = list(zip(op_names, op_indices, indices)) assert all(len(c) == 2 for c in genotype.reduce) n = len(genotype.reduce) // 2 op_indices = concat([(i, i) for i in range(2, 2 + n)]) op_names, indices = zip(*genotype.reduce) reduce = list(zip(op_names, op_indices, indices)) else: normal = genotype.normal reduce = genotype.reduce normal = sorted(normal, key=lambda c: (c[1], c[2])) reduce = sorted(reduce, key=lambda c: (c[1], c[2])) return Genotype(normal=normal, normal_concat=genotype.normal_concat, reduce=reduce, reduce_concat=genotype.reduce_concat)
from hanser.models.nas.genotypes import Genotype mul = 1 PDARTS = Genotype(normal=[('skip_connect', 0), ('dil_conv_3x3', 1), ('skip_connect', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 1), ('sep_conv_3x3', 3), ('sep_conv_3x3', 0), ('dil_conv_5x5', 4)], normal_concat=[2, 3, 4, 5], reduce=[('avg_pool_3x3', 0), ('sep_conv_5x5', 1), ('sep_conv_3x3', 0), ('dil_conv_5x5', 2), ('max_pool_3x3', 0), ('dil_conv_3x3', 1), ('dil_conv_3x3', 1), ('dil_conv_5x5', 3)], reduce_concat=[2, 3, 4, 5]) genotype = PDARTS batch_size = 96 weight_decay = 3e-4 drop_path = 0.3 aux_weight = 0.4 base_lr = 0.025 warmup_epoch = 0 epochs = 600 + warmup_epoch grad_clip_norm = 5.0 sync_bn = False val_freq = 1 valid_after = 500
from hanser.models.profile.fvcore import profile, count_mac from hanser.models.imagenet.gen_resnet.res2net.resnet import resnet50 from hanser.models.imagenet.gen_resnet.resnet import resnet50 from hanser.models.imagenet.efficientnet import efficientnet_b2 model = resnet50() model.build((None, 224, 224, 3)) n, t = profile(model) from hanser.models.cifar.nasnet import NASNet from hanser.models.nas.genotypes import Genotype input_shape = (64, 64, 5) my_genotype = Genotype(normal=[('avg_pool_3x3', 1), ('skip_connect', 0), ('skip_connect', 0), ('dil_conv_3x3', 1), ('skip_connect', 0), ('dil_conv_5x5', 2), ('skip_connect', 0), ('dil_conv_3x3', 1)], normal_concat=range(2, 6), reduce=[('max_pool_3x3', 1), ('max_pool_3x3', 0), ('skip_connect', 2), ('avg_pool_3x3', 0), ('avg_pool_3x3', 1), ('avg_pool_3x3', 3), ('max_pool_3x3', 2), ('max_pool_3x3', 3)], reduce_concat=range(2, 6)) model = NASNet(36, 23, True, 0.2, 4, my_genotype) # model = NASNet(8, 8, True, 0.2, 4, my_genotype) model.build((None, *input_shape)) profile(model)
seed = 42 mul = 8 # 2.69 SETN = Genotype( normal=[ ('skip_connect', 0), ('sep_conv_5x5', 1), ('sep_conv_5x5', 0), ('sep_conv_3x3', 1), ('sep_conv_5x5', 1), ('sep_conv_5x5', 3), ('max_pool_3x3', 1), ('conv_3x1_1x3', 4), ], normal_concat=[2, 3, 4, 5], reduce=[ ('sep_conv_3x3', 0), ('sep_conv_5x5', 1), ('avg_pool_3x3', 0), ('sep_conv_5x5', 1), ('avg_pool_3x3', 0), ('sep_conv_5x5', 1), ('avg_pool_3x3', 0), ('skip_connect', 1), ], reduce_concat=[2, 3, 4, 5], ) genotype = SETN batch_size = 96
def derive(alphas_normal, alphas_reduce, betas_normal, betas_reduce, primitives, steps=4, multiplier=4, max_sk=2): def get_op(w): if 'none' in primitives: i = max([ k for k in range(len(primitives)) if k != primitives.index('none') ], key=lambda k: w[k]) else: i = max(range(len(primitives)), key=lambda k: w[k]) return w[i], primitives[i] def _parse(alphas): gene = [] start = 0 for i in range(steps): end = start + i + 2 W = alphas[start:end] edges = sorted(range(i + 2), key=lambda x: -get_op(W[x])[0])[:2] for j in edges: gene.append((get_op(W[j])[1], j)) start = end return gene def _parse_max_sk(alphas): gene = _parse(alphas) if max_sk: sk_idx = primitives.index('skip_connect') while len([op for op in gene if op[0] == 'skip_connect']) > max_sk: print(alphas) print(len([op for op in gene if op[0] == 'skip_connect'])) sk_probs = alphas[:, sk_idx].copy() sk_probs[sk_probs == 0] = 1 alphas[sk_probs.argmin(), sk_idx] = 0. gene = _parse(alphas) return gene alphas_normal = tf.nn.softmax(alphas_normal, axis=-1).numpy() betas_normal = beta_softmax(betas_normal, steps, scale=True).numpy() alphas_normal = alphas_normal * betas_normal[:, None] alphas_reduce = tf.nn.softmax(alphas_reduce, axis=-1).numpy() betas_reduce = beta_softmax(betas_reduce, steps, scale=True).numpy() alphas_reduce = alphas_reduce * betas_reduce[:, None] gene_normal = _parse_max_sk(alphas_normal) gene_reduce = _parse_max_sk(alphas_reduce) concat = range(2 + steps - multiplier, steps + 2) genotype = Genotype(normal=gene_normal, normal_concat=concat, reduce=gene_reduce, reduce_concat=concat) return genotype
from hanser.models.nas.genotypes import Genotype mul = 8 # 2.55 CDARTS_cifar_2 = Genotype( normal=[ ('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 0), ('sep_conv_3x3', 2), ('skip_connect', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ], normal_concat=[2, 3, 4, 5], reduce=[('sep_conv_3x3', 0), ('skip_connect', 1), ('sep_conv_3x3', 0), ('max_pool_3x3', 1), ('skip_connect', 0), ('skip_connect', 2), ('skip_connect', 2), ('skip_connect', 4)], reduce_concat=[2, 3, 4, 5], ) genotype = CDARTS_cifar_2 batch_size = 128 weight_decay = 5e-4 drop_path = 0.3 aux_weight = 0.4 base_lr = 0.025
from hanser.models.nas.operations import ReLUConvBN from hanser.models.cifar.nasnet import NASNet from hanser.models.nas.genotypes import Genotype from horch.models.nas.cifar.darts import DARTS from horch.models.utils import summary FTSO = Genotype( normal=[ ('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ], normal_concat=[2, 3, 4, 5], reduce=[ ('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 0), ('sep_conv_3x3', 1) ], reduce_concat=[2, 3, 4, 5], ) def copy_conv(src, dst): if isinstance(src, tf.keras.Sequential): src = src.layers[1] with torch.no_grad(): if "Depth" in type(src).__name__: weight = src.depthwise_kernel