def __init__(self):
parser = self.build_parser()
args = parser.parse_args()
super().__init__(**vars(args))
self.data_path = './data/'
# self.data_path = './data/imagenet'
self.path = os.path.join('results/augment_Stage/cifar/', self.name)
self.genotype = gt.from_str(self.genotype)
self.DAG = gt.from_str(self.DAG)
self.gpus = parse_gpus(self.gpus)
self.amp_sync_bn = True
self.amp_opt_level = "O0"
def __init__(self):
parser = self.build_parser()
args = parser.parse_args()
super().__init__(**vars(args))
time_str = time.asctime(time.localtime()).replace(' ', '_')
random_str = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(8))
if self.save_path:
self.path = os.path.join(self.save_dir, self.name, self.save_path)
else:
self.path = os.path.join(self.save_dir, self.name,
time_str + random_str)
if os.path.exists(self.path):
exit('The path is exists!!')
if self.file:
file_ = open(self.file)
lines = file_.readlines()
for i, line in enumerate(lines):
self.path = os.path.join(
'/userhome/project/pt.darts/experiment/', self.name,
str(i))
if os.path.isdir(self.path):
continue
else:
self.genotype = line
print(line)
break
self.genotype = gt.from_str(self.genotype)
self.gpus = parse_gpus(self.gpus)
def __init__(self):
parser = self.build_parser()
args = parser.parse_args()
super().__init__(**vars(args))
time_str = time.asctime(time.localtime()).replace(' ', '_')
random_str = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(8))
if self.save_path:
self.path = os.path.join(self.save_dir, self.name, self.save_path)
else:
self.path = os.path.join(self.save_dir, self.name,
time_str + random_str)
if args.genotype:
print('Using single genotype')
self.genotype = gt.from_str(self.genotype)
self.path = os.path.join(self.save_dir, 'rl', self.name)
if self.file:
print('Using multi genotypes from file')
file_ = open(self.file)
lines = file_.readlines()
for i, line in enumerate(lines):
self.path = os.path.join(self.save_dir, self.name, str(i))
if i < self.i:
continue
if os.path.isdir(self.path):
continue
else:
self.genotype = line
print(line)
break
self.gpus = parse_gpus(self.gpus)
def __init__(self):
parser = self.build_parser()
args = parser.parse_args()
super().__init__(**vars(args))
self.path = os.path.join(self.path, self.name)
self.genotype = gt.from_str(self.genotype)
self.gpus = parse_gpus(self.gpus)
def __init__(self):
parser = self.build_parser()
args = parser.parse_args()
super().__init__(**vars(args))
self.data_path = './data/'
self.path = os.path.join('augments', self.name)
self.genotype = gt.from_str(self.genotype)
def __init__(self):
parser = self.build_parser()
args = parser.parse_args()
super().__init__(**vars(args))
self.data_path = '/userhome/temp_data/cifar10'
self.path = os.path.join('project/pt.darts/experiment/', self.name)
self.genotype = gt.from_str(self.genotype)
self.gpus = parse_gpus(self.gpus)
def get_genotype(config, ovr_genotype):
g_str = config.genotype
if not ovr_genotype is None:
genotype = gt.from_file(ovr_genotype)
elif g_str == '':
genotype = gt.from_file(config.gt_file)
else:
genotype = gt.from_str(g_str)
return genotype
def __init__(self):
parser = self.build_parser()
args = parser.parse_args()
super().__init__(**vars(args))
self.gpus = parse_gpus(self.gpus)
self.data_path = './data/'
self.path = os.path.join(
'searchs_%d_%s' % (len(self.gpus), self.noise), self.name)
self.genotype = gt.from_str(self.genotype)
def __init__(self):
parser = self.build_parser()
args = parser.parse_args()
super().__init__(**vars(args))
self.data_path = './data/'
self.path = os.path.join('results/search_Stage/ImageNet/', self.name)
self.genotype = gt.from_str(self.genotype)
self.DAG_path = os.path.join(self.path, 'DAG')
self.gpus = parse_gpus(self.gpus)
def __init__(self):
parser = self.build_parser()
args = parser.parse_args()
super().__init__(**vars(args))
self.data_path = os.path.expanduser('~/dataroot')
self.path = os.path.join(os.path.expanduser('~/logdir'), self.name)
os.makedirs(self.path, exist_ok=True)
self.genotype = gt.from_str(self.genotype)
self.gpus = parse_gpus(self.gpus)
def __init__(self):
parser = self.build_parser()
args = parser.parse_args()
super().__init__(**vars(args))
# first get the grand-parent directory of the current main-script (search.py) '
self.data_path = os.path.dirname(
os.path.dirname(os.path.abspath(__file__)))
# then join the datapath
self.data_path = os.path.join(self.data_path, args.datapath)
self.path = os.path.join(self.data_path, self.name, 'augments')
self.genotype = gt.from_str(self.genotype)
def __init__(self):
parser = self.build_parser()
args = parser.parse_args()
super().__init__(**vars(args))
self.data_path = '/home/LAB/gaoch/asdf/data'
self.path = os.path.join('augments', self.name)
self.path = os.path.join(self.path, os.environ["SLURM_JOBID"])
if not os.path.exists(self.path):
os.makedirs(self.path)
self.genotype = gt.from_str(self.genotype)
self.gpus = parse_gpus(self.gpus)
def __init__(self):
parser = self.build_parser()
args = parser.parse_args()
super().__init__(**vars(args))
time_str = time.asctime(time.localtime()).replace(' ', '_')
random_str = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(8))
if self.save_path:
self.path = os.path.join(self.save_dir, self.name, self.save_path)
else:
self.path = os.path.join(self.save_dir, self.name,
time_str + random_str)
if args.genotype:
print('Using single genotype')
self.genotype = gt.from_str(self.genotype)
self.path = os.path.join(self.save_dir, 'rl', self.name)
if self.file:
print('Using multi genotypes from file')
# file_ = open(self.file)
file_ = open(
'/home/ubuntu/2_workspace/nni_hpo/3rdparty/rethinking_performance_estimation_in_NAS/random_darts_architecture.txt'
)
lines = file_.readlines()
for i, line in enumerate(lines):
self.path = os.path.join(self.save_dir, self.name, str(i))
if i < self.i:
continue
if os.path.isdir(self.path):
continue
else:
self.genotype = line
print(line)
break
self.gpus = parse_gpus(self.gpus)
def run(args, myargs):
my_config = getattr(myargs.config, args.command)
config = AugmentConfig()
for k, v in args.items():
assert not hasattr(config, k)
setattr(config, k, v)
for k, v in my_config.items():
if not hasattr(config, k):
print('* config does not have %s'%k)
setattr(config, k, v)
device = torch.device("cuda")
writer = myargs.writer
writer.add_text('all_config', config.as_markdown(), 0)
logger = myargs.logger
config.print_params(logger.info_msg)
config.genotype = gt.from_str(config.genotype)
config.data_path = os.path.expanduser(config.data_path)
config.plot_path = os.path.join(args.outdir, 'plot')
config.path = args.outdir
main(config=config, logger=logger, device=device,
myargs=myargs)
v = str(i)
g.edge(u, v, label=op, fillcolor="gray")
# output node
g.node("c_{k}", fillcolor='palegoldenrod')
for i in range(n_nodes):
g.edge(str(i), "c_{k}", fillcolor="gray")
# add image caption
if caption:
g.attr(label=caption, overlap='false', fontsize='20', fontname='times')
g.render(file_path, view=False)
if __name__ == '__main__':
if len(sys.argv) not in [2, 3]:
raise ValueError("usage:\n python {} GENOTYPE [N_EDGES]".format(
sys.argv[0]))
genotype_str = sys.argv[1]
n_edges_per_node = int(sys.argv[2]) if len(sys.argv) == 3 else None
try:
genotype = gt.from_str(genotype_str, n_edges_per_node)
except AttributeError:
raise ValueError("Cannot parse {}".format(genotype_str))
plot(genotype.normal, "normal")
plot(genotype.reduce, "reduction")
def main():
# Breakdown the Composition of DARTS
# childnet = torch.load('./results/02-random-beta-childnet/best.pth.tar', map_location=torch.device(device))
# childnet.module.cells[0].dag[0][0]
# supernet = torch.load('./results/02-random-beta-supernet/best.pth.tar', map_location=torch.device(device))
# supernet.net.cells[0].dag[0][0]._ops[5].net[1]
input_size, input_channels, n_classes, train_data = utils.get_data('CIFAR10', './data/', cutout_length=0, validation=False)
net_crit = nn.CrossEntropyLoss().to(device)
supernet = SearchCNNControllerCKA(C_in=input_channels, C=16, n_classes=n_classes, n_layers=8, criterion=net_crit, device_ids=device_ids)
childnet = AugmentCNNCKA(input_size=input_size, C_in=input_channels, C=16, n_classes=10, n_layers=16, auxiliary=True, genotype=gt.from_str("Genotype(normal=[[('sep_conv_3x3', 1), ('sep_conv_3x3', 0)], [('sep_conv_3x3', 1), ('sep_conv_3x3', 0)], [('sep_conv_3x3', 1), ('sep_conv_5x5', 0)], [('sep_conv_3x3', 0), ('sep_conv_5x5', 1)]], normal_concat=range(2, 6), reduce=[[('sep_conv_3x3', 1), ('max_pool_3x3', 0)], [('max_pool_3x3', 2), ('sep_conv_5x5', 0)], [('dil_conv_5x5', 3), ('sep_conv_5x5', 1)], [('max_pool_3x3', 4), ('sep_conv_5x5', 0)]], reduce_concat=range(2, 6))"))
childnet = nn.DataParallel(childnet, device_ids=device_ids).to(device)
utils.load(supernet, './results/02-random-beta-supernet/weights.pt')
utils.load(childnet, './results/02-random-beta-l16-init16-childnet/weights.pt')
supernet.to(device)
childnet.to(device)
supernet.eval()
childnet.eval()
n_cells = len(supernet.net.cells)
n_ops = 8
for i in range(n_cells): # 8 Cells
# Normal Cell
if i not in [n_cells//3, 2*n_cells//3]:
# DAG 0
for k in range(2):
for j in range(n_ops):
if j != 3: # sep_conv_3x3
supernet.net.cells[i].dag[0][k]._ops[j] = ops.Zero(stride=1)
# DAG 1
for k in range(2):
for j in range(n_ops):
if j != 3: # sep_conv_3x3
supernet.net.cells[i].dag[1][k]._ops[j] = ops.Zero(stride=1)
for j in range(n_ops):
supernet.net.cells[i].dag[1][2]._ops[j] = ops.Zero(stride=1)
# DAG 2
for j in range(n_ops):
if j != 4: # sep_conv_5x5
supernet.net.cells[i].dag[2][0]._ops[j] = ops.Zero(stride=1)
for j in range(n_ops):
if j != 3: # sep_conv_3x3
supernet.net.cells[i].dag[2][1]._ops[j] = ops.Zero(stride=1)
for k in range(2,4):
for j in range(n_ops):
supernet.net.cells[i].dag[2][k]._ops[j] = ops.Zero(stride=1)
# DAG 3
for j in range(n_ops):
if j != 3: # sep_conv_3x3
supernet.net.cells[i].dag[3][0]._ops[j] = ops.Zero(stride=1)
for j in range(n_ops):
if j != 4: # sep_conv_5x5
supernet.net.cells[i].dag[3][1]._ops[j] = ops.Zero(stride=1)
for k in range(2,5):
for j in range(n_ops):
supernet.net.cells[i].dag[3][k]._ops[j] = ops.Zero(stride=1)
# Reduction Cell
else:
# DAG 0
for j in range(n_ops):
if j != 1: # max_pool_3x3
supernet.net.cells[i].dag[0][0]._ops[j] = ops.Zero(stride=2)
for j in range(n_ops):
if j != 3: # sep_conv_3x3
supernet.net.cells[i].dag[0][1]._ops[j] = ops.Zero(stride=2)
# DAG 1
for j in range(n_ops):
if j != 4: # sep_conv_5x5
supernet.net.cells[i].dag[1][0]._ops[j] = ops.Zero(stride=2)
for j in range(n_ops):
supernet.net.cells[i].dag[1][1]._ops[j] = ops.Zero(stride=2)
for j in range(n_ops):
if j != 1: # max_pool_3x3
supernet.net.cells[i].dag[1][2]._ops[j] = ops.Zero(stride=1)
# DAG 2
for j in range (n_ops):
if j != 4: # sep_conv_5x5
supernet.net.cells[i].dag[2][1]._ops[j] = ops.Zero(stride=2)
for j in range(n_ops):
if j != 6: # dil_conv_5x5
supernet.net.cells[i].dag[2][3]._ops[j] = ops.Zero(stride=1)
w = [0, 2]
for k in w:
if k == 0:
stride = 2
else:
stride = 1
for j in range(n_ops):
supernet.net.cells[i].dag[2][k]._ops[j] = ops.Zero(stride=stride)
# DAG 3
for j in range(n_ops):
if j != 4: # sep_conv_5x5
supernet.net.cells[i].dag[3][0]._ops[j] = ops.Zero(stride=2)
for j in range(n_ops):
if j != 1: # max_pool_3x3
supernet.net.cells[i].dag[3][4]._ops[j] = ops.Zero(stride=1)
for k in range(1,4):
if k == 1:
stride = 2
else:
stride = 1
for j in range(n_ops):
supernet.net.cells[i].dag[3][k]._ops[j] = ops.Zero(stride=stride)
# Test Image Batch
img = torch.cat([train_data[i][0].view(-1, 3, input_size, input_size) for i in range(64)], dim=0)
img = img.to(device, non_blocking=True)
# Evaluate Forward Pass
with torch.no_grad():
supernet(img) # len(supernet.net.outpts)
childnet(img) # len(childnet.module.outputs)
sout = [x.view(64,-1).cpu().numpy() for x in supernet.net.outputs]
cout = [x.view(64,-1).cpu().numpy() for x in childnet.module.outputs]
lcka_arr = {}
rbfcka_arr = {}
lcka_arr_debiased = {}
cca_arr = {}
for i, s in enumerate(sout):
print('Itr:', i)
lcka_tmp = {}
rbfcka_tmp = {}
lcka_tmp_debiased = {}
cca_tmp = {}
for j, c in enumerate(cout):
if s.shape == c.shape:
# Linear CKA
lcka = cka.feature_space_linear_cka(s, c)
lcka_tmp[(i,j)] = round(lcka, 4)
# Non-Linear CKA
rbfcka = cka.cka(cka.gram_rbf(s, 0.5), cka.gram_rbf(c, 0.5))
rbfcka_tmp[(i,j)] = round(rbfcka, 4)
# Linear CKA Debiased
lcka_debiased = cka.feature_space_linear_cka(s, c, debiased=True)
lcka_tmp_debiased[(i,j)] = round(lcka_debiased, 4)
# CCA
rcca = cca.cca(s, c)
cca_tmp[(i,j)] = round(rcca, 4)
lcka_arr[i] = lcka_tmp
rbfcka_arr[i] = rbfcka_tmp
lcka_arr_debiased[i] = lcka_tmp_debiased
cca_arr[i] = cca_tmp
print('Linear CKA: ', lcka_arr)
print()
print('RBF CKA: ', rbfcka_arr)
print()
print('LinearCKAD: ', lcka_arr_debiased)
print()
print('CCA: ', cca_arr)
else:
u = str(i - 1)
v = str(i)
g.edge(u, v, label=op, fillcolor="gray")
# output node
g.node("c_{k}", fillcolor="palegoldenrod")
# for i in range(n_nodes):
g.edge(str(len(genotype) - 1), "c_{k}", fillcolor="gray")
# add image caption
if caption:
g.attr(label=caption, overlap="false", fontsize="20", fontname="times")
g.render(file_path, view=False)
if __name__ == "__main__":
if len(sys.argv) != 2:
raise ValueError("usage:\n python {} GENOTYPE".format(sys.argv[0]))
genotype_str = sys.argv[1]
try:
genotype = gt.from_str(genotype_str)
except AttributeError:
raise ValueError("Cannot parse {}".format(genotype_str))
plot(genotype.normal, "normal")
plot(genotype.reduce, "reduction")
y = data[0]["label"].squeeze().long().cuda(async=True)
val_iter(X, y)
valid_loader.reset()
else:
for step, (X, y) in enumerate(valid_loader):
X, y = X.to(device, non_blocking=True), y.to(device,
non_blocking=True)
val_iter(X, y)
writer.add_scalar('val/loss', losses.avg, cur_step)
writer.add_scalar('val/top1', top1.avg, cur_step)
writer.add_scalar('val/top5', top5.avg, cur_step)
logger.info("Valid: [{:3d}/{}] Final Prec@1 {:.4%}".format(
epoch + 1, config.epochs, top1.avg))
return top1.avg
if __name__ == "__main__":
import genotypes as gt
file = open(config.file)
lines = file.readlines()
best = 0
best_ge = None
for line in lines:
logger.info('Now the genotype is:{0}'.format(line))
ge = gt.from_str(line)
top1 = main(ge)
if top1 > best:
best_ge = ge
logger.info('Current best ge is:{0}'.format(str(best_ge)))
logger.info('Final best ge is:{0}'.format(format(str(best_ge))))
def __init__(self, dict):
parser = self.build_parser()
args = parser.parse_args()
super().__init__(**vars(args))
if args.bpe:
super().__init__(**(dict))
time_str = time.asctime(time.localtime()).replace(' ', '_')
random_str = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(8))
if self.save_path:
self.path = os.path.join(self.save_dir, self.name, self.save_path)
else:
self.path = os.path.join(self.save_dir, self.name,
time_str + random_str)
if args.genotype:
print('Using single genotype')
self.genotype = gt.from_str(self.genotype)
self.path = os.path.join(self.save_dir, 'rl', self.name)
# if self.file:
# print('Using multi genotypes from file')
# file_ = open(self.file)
# lines = file_.readlines()
# for i, line in enumerate(lines):
# self.path = os.path.join(self.save_dir, self.name, str(i))
# if i < self.i:
# continue
# if os.path.isdir(self.path):
# continue
# else:
# self.genotype = line
# print(line)
# break
if self.file and self.dataset == 'CIFAR10':
print("automate run cifar10 epochs".format(self.epochs))
file_ = open(self.file)
lines = file_.readlines()
for i, line in enumerate(lines):
self.path = os.path.join(self.save_dir, self.name, str(i))
if i < self.i:
continue
if os.path.isdir(self.path):
continue
else:
print('run for {}th model genotype{}'.format(
i, self.genotype))
self.i = i
self.genotype = gt.from_str(line)
print(line)
break
if self.file and self.dataset == 'cifar100':
print("automate run cifar100")
file_ = open(self.file)
lines = file_.readlines()
for i, line in enumerate(lines):
self.path = os.path.join(self.save_dir, self.name, str(i))
if i < self.i:
continue
if os.path.isdir(self.path):
continue
else:
print('run for {}th model genotype{}'.format(
i, self.genotype))
self.i = i
self.genotype = gt.from_str(line)
print(line)
break
self.gpus = parse_gpus(self.gpus)
