def genotype(self):
PRIMITIVES = get_primitives()
def get_op(w):
if 'none' in PRIMITIVES:
i = max([
k for k in range(len(PRIMITIVES))
if k != PRIMITIVES.index('none')
],
key=lambda k: w[k])
else:
i = max(range(len(PRIMITIVES)), key=lambda k: w[k])
return w[i], PRIMITIVES[i]
def _parse(alphas):
gene = []
start = 0
for i in range(self._steps):
end = start + i + 2
W = alphas[start:end]
edges = sorted(range(i + 2),
key=lambda x: -get_op(W[x])[0])[:2]
for j in edges:
gene.append((get_op(W[j])[1], j))
start = end
return gene
alphas_normal = tf.nn.softmax(self.alphas_normal, axis=-1).numpy()
betas_normal = beta_softmax(self.betas_normal, self._steps).numpy()
alphas_normal = alphas_normal * betas_normal[:, None]
alphas_reduce = tf.nn.softmax(self.alphas_reduce, axis=-1).numpy()
betas_reduce = beta_softmax(self.betas_reduce, self._steps).numpy()
alphas_reduce = alphas_reduce * betas_reduce[:, None]
gene_normal = _parse(alphas_normal)
gene_reduce = _parse(alphas_reduce)
concat = range(2 + self._steps - self._multiplier, self._steps + 2)
genotype = Genotype(normal=gene_normal,
normal_concat=concat,
reduce=gene_reduce,
reduce_concat=concat)
return genotype
def standardize(genotype: Genotype):
if len(genotype.normal[0]) == 2:
assert all(len(c) == 2 for c in genotype.normal)
n = len(genotype.normal) // 2
op_indices = concat([(i, i) for i in range(2, 2 + n)])
op_names, indices = zip(*genotype.normal)
normal = list(zip(op_names, op_indices, indices))
assert all(len(c) == 2 for c in genotype.reduce)
n = len(genotype.reduce) // 2
op_indices = concat([(i, i) for i in range(2, 2 + n)])
op_names, indices = zip(*genotype.reduce)
reduce = list(zip(op_names, op_indices, indices))
else:
normal = genotype.normal
reduce = genotype.reduce
normal = sorted(normal, key=lambda c: (c[1], c[2]))
reduce = sorted(reduce, key=lambda c: (c[1], c[2]))
return Genotype(normal=normal,
normal_concat=genotype.normal_concat,
reduce=reduce,
reduce_concat=genotype.reduce_concat)
from hanser.models.nas.genotypes import Genotype
mul = 1
PDARTS = Genotype(normal=[('skip_connect', 0), ('dil_conv_3x3', 1),
('skip_connect', 0), ('sep_conv_3x3', 1),
('sep_conv_3x3', 1), ('sep_conv_3x3', 3),
('sep_conv_3x3', 0), ('dil_conv_5x5', 4)],
normal_concat=[2, 3, 4, 5],
reduce=[('avg_pool_3x3', 0), ('sep_conv_5x5', 1),
('sep_conv_3x3', 0), ('dil_conv_5x5', 2),
('max_pool_3x3', 0), ('dil_conv_3x3', 1),
('dil_conv_3x3', 1), ('dil_conv_5x5', 3)],
reduce_concat=[2, 3, 4, 5])
genotype = PDARTS
batch_size = 96
weight_decay = 3e-4
drop_path = 0.3
aux_weight = 0.4
base_lr = 0.025
warmup_epoch = 0
epochs = 600 + warmup_epoch
grad_clip_norm = 5.0
sync_bn = False
val_freq = 1
valid_after = 500
from hanser.models.profile.fvcore import profile, count_mac
from hanser.models.imagenet.gen_resnet.res2net.resnet import resnet50
from hanser.models.imagenet.gen_resnet.resnet import resnet50
from hanser.models.imagenet.efficientnet import efficientnet_b2
model = resnet50()
model.build((None, 224, 224, 3))
n, t = profile(model)
from hanser.models.cifar.nasnet import NASNet
from hanser.models.nas.genotypes import Genotype
input_shape = (64, 64, 5)
my_genotype = Genotype(normal=[('avg_pool_3x3', 1), ('skip_connect', 0),
('skip_connect', 0), ('dil_conv_3x3', 1),
('skip_connect', 0), ('dil_conv_5x5', 2),
('skip_connect', 0), ('dil_conv_3x3', 1)],
normal_concat=range(2, 6),
reduce=[('max_pool_3x3', 1), ('max_pool_3x3', 0),
('skip_connect', 2), ('avg_pool_3x3', 0),
('avg_pool_3x3', 1), ('avg_pool_3x3', 3),
('max_pool_3x3', 2), ('max_pool_3x3', 3)],
reduce_concat=range(2, 6))
model = NASNet(36, 23, True, 0.2, 4, my_genotype)
# model = NASNet(8, 8, True, 0.2, 4, my_genotype)
model.build((None, *input_shape))
profile(model)
seed = 42
mul = 8
# 2.69
SETN = Genotype(
normal=[
('skip_connect', 0),
('sep_conv_5x5', 1),
('sep_conv_5x5', 0),
('sep_conv_3x3', 1),
('sep_conv_5x5', 1),
('sep_conv_5x5', 3),
('max_pool_3x3', 1),
('conv_3x1_1x3', 4),
],
normal_concat=[2, 3, 4, 5],
reduce=[
('sep_conv_3x3', 0),
('sep_conv_5x5', 1),
('avg_pool_3x3', 0),
('sep_conv_5x5', 1),
('avg_pool_3x3', 0),
('sep_conv_5x5', 1),
('avg_pool_3x3', 0),
('skip_connect', 1),
],
reduce_concat=[2, 3, 4, 5],
)
genotype = SETN
batch_size = 96
def derive(alphas_normal,
alphas_reduce,
betas_normal,
betas_reduce,
primitives,
steps=4,
multiplier=4,
max_sk=2):
def get_op(w):
if 'none' in primitives:
i = max([
k for k in range(len(primitives))
if k != primitives.index('none')
],
key=lambda k: w[k])
else:
i = max(range(len(primitives)), key=lambda k: w[k])
return w[i], primitives[i]
def _parse(alphas):
gene = []
start = 0
for i in range(steps):
end = start + i + 2
W = alphas[start:end]
edges = sorted(range(i + 2), key=lambda x: -get_op(W[x])[0])[:2]
for j in edges:
gene.append((get_op(W[j])[1], j))
start = end
return gene
def _parse_max_sk(alphas):
gene = _parse(alphas)
if max_sk:
sk_idx = primitives.index('skip_connect')
while len([op for op in gene if op[0] == 'skip_connect']) > max_sk:
print(alphas)
print(len([op for op in gene if op[0] == 'skip_connect']))
sk_probs = alphas[:, sk_idx].copy()
sk_probs[sk_probs == 0] = 1
alphas[sk_probs.argmin(), sk_idx] = 0.
gene = _parse(alphas)
return gene
alphas_normal = tf.nn.softmax(alphas_normal, axis=-1).numpy()
betas_normal = beta_softmax(betas_normal, steps, scale=True).numpy()
alphas_normal = alphas_normal * betas_normal[:, None]
alphas_reduce = tf.nn.softmax(alphas_reduce, axis=-1).numpy()
betas_reduce = beta_softmax(betas_reduce, steps, scale=True).numpy()
alphas_reduce = alphas_reduce * betas_reduce[:, None]
gene_normal = _parse_max_sk(alphas_normal)
gene_reduce = _parse_max_sk(alphas_reduce)
concat = range(2 + steps - multiplier, steps + 2)
genotype = Genotype(normal=gene_normal,
normal_concat=concat,
reduce=gene_reduce,
reduce_concat=concat)
return genotype
from hanser.models.nas.genotypes import Genotype
mul = 8
# 2.55
CDARTS_cifar_2 = Genotype(
normal=[
('sep_conv_3x3', 0),
('sep_conv_3x3', 1),
('sep_conv_3x3', 0),
('sep_conv_3x3', 2),
('skip_connect', 0),
('sep_conv_3x3', 1),
('sep_conv_3x3', 0),
('sep_conv_3x3', 1),
],
normal_concat=[2, 3, 4, 5],
reduce=[('sep_conv_3x3', 0), ('skip_connect', 1), ('sep_conv_3x3', 0),
('max_pool_3x3', 1), ('skip_connect', 0), ('skip_connect', 2),
('skip_connect', 2), ('skip_connect', 4)],
reduce_concat=[2, 3, 4, 5],
)
genotype = CDARTS_cifar_2
batch_size = 128
weight_decay = 5e-4
drop_path = 0.3
aux_weight = 0.4
base_lr = 0.025
from hanser.models.nas.operations import ReLUConvBN
from hanser.models.cifar.nasnet import NASNet
from hanser.models.nas.genotypes import Genotype
from horch.models.nas.cifar.darts import DARTS
from horch.models.utils import summary
FTSO = Genotype(
normal=[
('sep_conv_3x3', 0), ('sep_conv_3x3', 1),
('sep_conv_3x3', 0), ('sep_conv_3x3', 1),
('sep_conv_3x3', 0), ('sep_conv_3x3', 1),
('sep_conv_3x3', 0), ('sep_conv_3x3', 1),
],
normal_concat=[2, 3, 4, 5],
reduce=[
('sep_conv_3x3', 0), ('sep_conv_3x3', 1),
('sep_conv_3x3', 0), ('sep_conv_3x3', 1),
('sep_conv_3x3', 0), ('sep_conv_3x3', 1),
('sep_conv_3x3', 0), ('sep_conv_3x3', 1)
],
reduce_concat=[2, 3, 4, 5],
)
def copy_conv(src, dst):
if isinstance(src, tf.keras.Sequential):
src = src.layers[1]
with torch.no_grad():
if "Depth" in type(src).__name__:
weight = src.depthwise_kernel