def __init__(self): parser = self.build_parser() args = parser.parse_args() super().__init__(**vars(args)) self.data_path = './data/' # self.data_path = './data/imagenet' self.path = os.path.join('results/augment_Stage/cifar/', self.name) self.genotype = gt.from_str(self.genotype) self.DAG = gt.from_str(self.DAG) self.gpus = parse_gpus(self.gpus) self.amp_sync_bn = True self.amp_opt_level = "O0"
def __init__(self): parser = self.build_parser() args = parser.parse_args() super().__init__(**vars(args)) time_str = time.asctime(time.localtime()).replace(' ', '_') random_str = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) if self.save_path: self.path = os.path.join(self.save_dir, self.name, self.save_path) else: self.path = os.path.join(self.save_dir, self.name, time_str + random_str) if os.path.exists(self.path): exit('The path is exists!!') if self.file: file_ = open(self.file) lines = file_.readlines() for i, line in enumerate(lines): self.path = os.path.join( '/userhome/project/pt.darts/experiment/', self.name, str(i)) if os.path.isdir(self.path): continue else: self.genotype = line print(line) break self.genotype = gt.from_str(self.genotype) self.gpus = parse_gpus(self.gpus)
def __init__(self): parser = self.build_parser() args = parser.parse_args() super().__init__(**vars(args)) time_str = time.asctime(time.localtime()).replace(' ', '_') random_str = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) if self.save_path: self.path = os.path.join(self.save_dir, self.name, self.save_path) else: self.path = os.path.join(self.save_dir, self.name, time_str + random_str) if args.genotype: print('Using single genotype') self.genotype = gt.from_str(self.genotype) self.path = os.path.join(self.save_dir, 'rl', self.name) if self.file: print('Using multi genotypes from file') file_ = open(self.file) lines = file_.readlines() for i, line in enumerate(lines): self.path = os.path.join(self.save_dir, self.name, str(i)) if i < self.i: continue if os.path.isdir(self.path): continue else: self.genotype = line print(line) break self.gpus = parse_gpus(self.gpus)
def __init__(self): parser = self.build_parser() args = parser.parse_args() super().__init__(**vars(args)) self.path = os.path.join(self.path, self.name) self.genotype = gt.from_str(self.genotype) self.gpus = parse_gpus(self.gpus)
def __init__(self): parser = self.build_parser() args = parser.parse_args() super().__init__(**vars(args)) self.data_path = './data/' self.path = os.path.join('augments', self.name) self.genotype = gt.from_str(self.genotype)
def __init__(self): parser = self.build_parser() args = parser.parse_args() super().__init__(**vars(args)) self.data_path = '/userhome/temp_data/cifar10' self.path = os.path.join('project/pt.darts/experiment/', self.name) self.genotype = gt.from_str(self.genotype) self.gpus = parse_gpus(self.gpus)
def get_genotype(config, ovr_genotype): g_str = config.genotype if not ovr_genotype is None: genotype = gt.from_file(ovr_genotype) elif g_str == '': genotype = gt.from_file(config.gt_file) else: genotype = gt.from_str(g_str) return genotype
def __init__(self): parser = self.build_parser() args = parser.parse_args() super().__init__(**vars(args)) self.gpus = parse_gpus(self.gpus) self.data_path = './data/' self.path = os.path.join( 'searchs_%d_%s' % (len(self.gpus), self.noise), self.name) self.genotype = gt.from_str(self.genotype)
def __init__(self): parser = self.build_parser() args = parser.parse_args() super().__init__(**vars(args)) self.data_path = './data/' self.path = os.path.join('results/search_Stage/ImageNet/', self.name) self.genotype = gt.from_str(self.genotype) self.DAG_path = os.path.join(self.path, 'DAG') self.gpus = parse_gpus(self.gpus)
def __init__(self): parser = self.build_parser() args = parser.parse_args() super().__init__(**vars(args)) self.data_path = os.path.expanduser('~/dataroot') self.path = os.path.join(os.path.expanduser('~/logdir'), self.name) os.makedirs(self.path, exist_ok=True) self.genotype = gt.from_str(self.genotype) self.gpus = parse_gpus(self.gpus)
def __init__(self): parser = self.build_parser() args = parser.parse_args() super().__init__(**vars(args)) # first get the grand-parent directory of the current main-script (search.py) ' self.data_path = os.path.dirname( os.path.dirname(os.path.abspath(__file__))) # then join the datapath self.data_path = os.path.join(self.data_path, args.datapath) self.path = os.path.join(self.data_path, self.name, 'augments') self.genotype = gt.from_str(self.genotype)
def __init__(self): parser = self.build_parser() args = parser.parse_args() super().__init__(**vars(args)) self.data_path = '/home/LAB/gaoch/asdf/data' self.path = os.path.join('augments', self.name) self.path = os.path.join(self.path, os.environ["SLURM_JOBID"]) if not os.path.exists(self.path): os.makedirs(self.path) self.genotype = gt.from_str(self.genotype) self.gpus = parse_gpus(self.gpus)
def __init__(self): parser = self.build_parser() args = parser.parse_args() super().__init__(**vars(args)) time_str = time.asctime(time.localtime()).replace(' ', '_') random_str = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) if self.save_path: self.path = os.path.join(self.save_dir, self.name, self.save_path) else: self.path = os.path.join(self.save_dir, self.name, time_str + random_str) if args.genotype: print('Using single genotype') self.genotype = gt.from_str(self.genotype) self.path = os.path.join(self.save_dir, 'rl', self.name) if self.file: print('Using multi genotypes from file') # file_ = open(self.file) file_ = open( '/home/ubuntu/2_workspace/nni_hpo/3rdparty/rethinking_performance_estimation_in_NAS/random_darts_architecture.txt' ) lines = file_.readlines() for i, line in enumerate(lines): self.path = os.path.join(self.save_dir, self.name, str(i)) if i < self.i: continue if os.path.isdir(self.path): continue else: self.genotype = line print(line) break self.gpus = parse_gpus(self.gpus)
def run(args, myargs): my_config = getattr(myargs.config, args.command) config = AugmentConfig() for k, v in args.items(): assert not hasattr(config, k) setattr(config, k, v) for k, v in my_config.items(): if not hasattr(config, k): print('* config does not have %s'%k) setattr(config, k, v) device = torch.device("cuda") writer = myargs.writer writer.add_text('all_config', config.as_markdown(), 0) logger = myargs.logger config.print_params(logger.info_msg) config.genotype = gt.from_str(config.genotype) config.data_path = os.path.expanduser(config.data_path) config.plot_path = os.path.join(args.outdir, 'plot') config.path = args.outdir main(config=config, logger=logger, device=device, myargs=myargs)
v = str(i) g.edge(u, v, label=op, fillcolor="gray") # output node g.node("c_{k}", fillcolor='palegoldenrod') for i in range(n_nodes): g.edge(str(i), "c_{k}", fillcolor="gray") # add image caption if caption: g.attr(label=caption, overlap='false', fontsize='20', fontname='times') g.render(file_path, view=False) if __name__ == '__main__': if len(sys.argv) not in [2, 3]: raise ValueError("usage:\n python {} GENOTYPE [N_EDGES]".format( sys.argv[0])) genotype_str = sys.argv[1] n_edges_per_node = int(sys.argv[2]) if len(sys.argv) == 3 else None try: genotype = gt.from_str(genotype_str, n_edges_per_node) except AttributeError: raise ValueError("Cannot parse {}".format(genotype_str)) plot(genotype.normal, "normal") plot(genotype.reduce, "reduction")
def main(): # Breakdown the Composition of DARTS # childnet = torch.load('./results/02-random-beta-childnet/best.pth.tar', map_location=torch.device(device)) # childnet.module.cells[0].dag[0][0] # supernet = torch.load('./results/02-random-beta-supernet/best.pth.tar', map_location=torch.device(device)) # supernet.net.cells[0].dag[0][0]._ops[5].net[1] input_size, input_channels, n_classes, train_data = utils.get_data('CIFAR10', './data/', cutout_length=0, validation=False) net_crit = nn.CrossEntropyLoss().to(device) supernet = SearchCNNControllerCKA(C_in=input_channels, C=16, n_classes=n_classes, n_layers=8, criterion=net_crit, device_ids=device_ids) childnet = AugmentCNNCKA(input_size=input_size, C_in=input_channels, C=16, n_classes=10, n_layers=16, auxiliary=True, genotype=gt.from_str("Genotype(normal=[[('sep_conv_3x3', 1), ('sep_conv_3x3', 0)], [('sep_conv_3x3', 1), ('sep_conv_3x3', 0)], [('sep_conv_3x3', 1), ('sep_conv_5x5', 0)], [('sep_conv_3x3', 0), ('sep_conv_5x5', 1)]], normal_concat=range(2, 6), reduce=[[('sep_conv_3x3', 1), ('max_pool_3x3', 0)], [('max_pool_3x3', 2), ('sep_conv_5x5', 0)], [('dil_conv_5x5', 3), ('sep_conv_5x5', 1)], [('max_pool_3x3', 4), ('sep_conv_5x5', 0)]], reduce_concat=range(2, 6))")) childnet = nn.DataParallel(childnet, device_ids=device_ids).to(device) utils.load(supernet, './results/02-random-beta-supernet/weights.pt') utils.load(childnet, './results/02-random-beta-l16-init16-childnet/weights.pt') supernet.to(device) childnet.to(device) supernet.eval() childnet.eval() n_cells = len(supernet.net.cells) n_ops = 8 for i in range(n_cells): # 8 Cells # Normal Cell if i not in [n_cells//3, 2*n_cells//3]: # DAG 0 for k in range(2): for j in range(n_ops): if j != 3: # sep_conv_3x3 supernet.net.cells[i].dag[0][k]._ops[j] = ops.Zero(stride=1) # DAG 1 for k in range(2): for j in range(n_ops): if j != 3: # sep_conv_3x3 supernet.net.cells[i].dag[1][k]._ops[j] = ops.Zero(stride=1) for j in range(n_ops): supernet.net.cells[i].dag[1][2]._ops[j] = ops.Zero(stride=1) # DAG 2 for j in range(n_ops): if j != 4: # sep_conv_5x5 supernet.net.cells[i].dag[2][0]._ops[j] = ops.Zero(stride=1) for j in range(n_ops): if j != 3: # sep_conv_3x3 supernet.net.cells[i].dag[2][1]._ops[j] = ops.Zero(stride=1) for k in range(2,4): for j in range(n_ops): supernet.net.cells[i].dag[2][k]._ops[j] = ops.Zero(stride=1) # DAG 3 for j in range(n_ops): if j != 3: # sep_conv_3x3 supernet.net.cells[i].dag[3][0]._ops[j] = ops.Zero(stride=1) for j in range(n_ops): if j != 4: # sep_conv_5x5 supernet.net.cells[i].dag[3][1]._ops[j] = ops.Zero(stride=1) for k in range(2,5): for j in range(n_ops): supernet.net.cells[i].dag[3][k]._ops[j] = ops.Zero(stride=1) # Reduction Cell else: # DAG 0 for j in range(n_ops): if j != 1: # max_pool_3x3 supernet.net.cells[i].dag[0][0]._ops[j] = ops.Zero(stride=2) for j in range(n_ops): if j != 3: # sep_conv_3x3 supernet.net.cells[i].dag[0][1]._ops[j] = ops.Zero(stride=2) # DAG 1 for j in range(n_ops): if j != 4: # sep_conv_5x5 supernet.net.cells[i].dag[1][0]._ops[j] = ops.Zero(stride=2) for j in range(n_ops): supernet.net.cells[i].dag[1][1]._ops[j] = ops.Zero(stride=2) for j in range(n_ops): if j != 1: # max_pool_3x3 supernet.net.cells[i].dag[1][2]._ops[j] = ops.Zero(stride=1) # DAG 2 for j in range (n_ops): if j != 4: # sep_conv_5x5 supernet.net.cells[i].dag[2][1]._ops[j] = ops.Zero(stride=2) for j in range(n_ops): if j != 6: # dil_conv_5x5 supernet.net.cells[i].dag[2][3]._ops[j] = ops.Zero(stride=1) w = [0, 2] for k in w: if k == 0: stride = 2 else: stride = 1 for j in range(n_ops): supernet.net.cells[i].dag[2][k]._ops[j] = ops.Zero(stride=stride) # DAG 3 for j in range(n_ops): if j != 4: # sep_conv_5x5 supernet.net.cells[i].dag[3][0]._ops[j] = ops.Zero(stride=2) for j in range(n_ops): if j != 1: # max_pool_3x3 supernet.net.cells[i].dag[3][4]._ops[j] = ops.Zero(stride=1) for k in range(1,4): if k == 1: stride = 2 else: stride = 1 for j in range(n_ops): supernet.net.cells[i].dag[3][k]._ops[j] = ops.Zero(stride=stride) # Test Image Batch img = torch.cat([train_data[i][0].view(-1, 3, input_size, input_size) for i in range(64)], dim=0) img = img.to(device, non_blocking=True) # Evaluate Forward Pass with torch.no_grad(): supernet(img) # len(supernet.net.outpts) childnet(img) # len(childnet.module.outputs) sout = [x.view(64,-1).cpu().numpy() for x in supernet.net.outputs] cout = [x.view(64,-1).cpu().numpy() for x in childnet.module.outputs] lcka_arr = {} rbfcka_arr = {} lcka_arr_debiased = {} cca_arr = {} for i, s in enumerate(sout): print('Itr:', i) lcka_tmp = {} rbfcka_tmp = {} lcka_tmp_debiased = {} cca_tmp = {} for j, c in enumerate(cout): if s.shape == c.shape: # Linear CKA lcka = cka.feature_space_linear_cka(s, c) lcka_tmp[(i,j)] = round(lcka, 4) # Non-Linear CKA rbfcka = cka.cka(cka.gram_rbf(s, 0.5), cka.gram_rbf(c, 0.5)) rbfcka_tmp[(i,j)] = round(rbfcka, 4) # Linear CKA Debiased lcka_debiased = cka.feature_space_linear_cka(s, c, debiased=True) lcka_tmp_debiased[(i,j)] = round(lcka_debiased, 4) # CCA rcca = cca.cca(s, c) cca_tmp[(i,j)] = round(rcca, 4) lcka_arr[i] = lcka_tmp rbfcka_arr[i] = rbfcka_tmp lcka_arr_debiased[i] = lcka_tmp_debiased cca_arr[i] = cca_tmp print('Linear CKA: ', lcka_arr) print() print('RBF CKA: ', rbfcka_arr) print() print('LinearCKAD: ', lcka_arr_debiased) print() print('CCA: ', cca_arr)
else: u = str(i - 1) v = str(i) g.edge(u, v, label=op, fillcolor="gray") # output node g.node("c_{k}", fillcolor="palegoldenrod") # for i in range(n_nodes): g.edge(str(len(genotype) - 1), "c_{k}", fillcolor="gray") # add image caption if caption: g.attr(label=caption, overlap="false", fontsize="20", fontname="times") g.render(file_path, view=False) if __name__ == "__main__": if len(sys.argv) != 2: raise ValueError("usage:\n python {} GENOTYPE".format(sys.argv[0])) genotype_str = sys.argv[1] try: genotype = gt.from_str(genotype_str) except AttributeError: raise ValueError("Cannot parse {}".format(genotype_str)) plot(genotype.normal, "normal") plot(genotype.reduce, "reduction")
y = data[0]["label"].squeeze().long().cuda(async=True) val_iter(X, y) valid_loader.reset() else: for step, (X, y) in enumerate(valid_loader): X, y = X.to(device, non_blocking=True), y.to(device, non_blocking=True) val_iter(X, y) writer.add_scalar('val/loss', losses.avg, cur_step) writer.add_scalar('val/top1', top1.avg, cur_step) writer.add_scalar('val/top5', top5.avg, cur_step) logger.info("Valid: [{:3d}/{}] Final Prec@1 {:.4%}".format( epoch + 1, config.epochs, top1.avg)) return top1.avg if __name__ == "__main__": import genotypes as gt file = open(config.file) lines = file.readlines() best = 0 best_ge = None for line in lines: logger.info('Now the genotype is:{0}'.format(line)) ge = gt.from_str(line) top1 = main(ge) if top1 > best: best_ge = ge logger.info('Current best ge is:{0}'.format(str(best_ge))) logger.info('Final best ge is:{0}'.format(format(str(best_ge))))
def __init__(self, dict): parser = self.build_parser() args = parser.parse_args() super().__init__(**vars(args)) if args.bpe: super().__init__(**(dict)) time_str = time.asctime(time.localtime()).replace(' ', '_') random_str = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) if self.save_path: self.path = os.path.join(self.save_dir, self.name, self.save_path) else: self.path = os.path.join(self.save_dir, self.name, time_str + random_str) if args.genotype: print('Using single genotype') self.genotype = gt.from_str(self.genotype) self.path = os.path.join(self.save_dir, 'rl', self.name) # if self.file: # print('Using multi genotypes from file') # file_ = open(self.file) # lines = file_.readlines() # for i, line in enumerate(lines): # self.path = os.path.join(self.save_dir, self.name, str(i)) # if i < self.i: # continue # if os.path.isdir(self.path): # continue # else: # self.genotype = line # print(line) # break if self.file and self.dataset == 'CIFAR10': print("automate run cifar10 epochs".format(self.epochs)) file_ = open(self.file) lines = file_.readlines() for i, line in enumerate(lines): self.path = os.path.join(self.save_dir, self.name, str(i)) if i < self.i: continue if os.path.isdir(self.path): continue else: print('run for {}th model genotype{}'.format( i, self.genotype)) self.i = i self.genotype = gt.from_str(line) print(line) break if self.file and self.dataset == 'cifar100': print("automate run cifar100") file_ = open(self.file) lines = file_.readlines() for i, line in enumerate(lines): self.path = os.path.join(self.save_dir, self.name, str(i)) if i < self.i: continue if os.path.isdir(self.path): continue else: print('run for {}th model genotype{}'.format( i, self.genotype)) self.i = i self.genotype = gt.from_str(line) print(line) break self.gpus = parse_gpus(self.gpus)