def main(): opts.parse_args() #print opts.resolved if opts.resolved.verbose: log("massaging input...") p = process(opts.inputs) if opts.resolved.verbose: log("parsing...") p = parse_program(p) if opts.resolved.verbose: log("processing...") run_chain(p) if opts.resolved.verbose: log("dumping output...") if opts.resolved.output == '-': out = sys.stdout else: out = file(opts.resolved.output, 'w') p.accept(PrintVisitor(stream = out))
def opt(): args = [ '--model', 'SimpleNet', '--dataset', 'CIFAR10', '--data_dir', '../data', '--num_classes_per_task', '1', '--num_tasks', '3', '--num_pretrain_classes', '3', '--num_pretrain_passes', '1', '--num_loops', '1' ] args = parse_args(args) return args
def caculate_from_txt(): args = parse_args() #txt_path = Path() txt_path = Path(os.path.dirname(__file__)) / args.txt_path dirs = readlines(txt_path) files_ls = [Path(dr).files() for dr in dirs] lines = [] line = [] batch = [] for files in files_ls: for img_p in files: img = cv2.imread(img_p) img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) img = torch.tensor(img).cuda() img = normalize_image(img) img = img.reshape(1, 1, img.shape[0], img.shape[1]) line.append(stat(img)) line = torch.tensor(line) line_hist = line.histc(bins=50, min=0, max=1) lines.append(line_hist.numpy()) line = [] print('ok') plt.subplot(2, 2, 1) plt.plot(lines[0], 'r') plt.subplot(2, 2, 2) plt.plot(lines[1], 'g') plt.subplot(2, 2, 3) plt.plot(lines[2], 'b') #plt.subplot(2,2,4) #plt.plot(lines[3],'y') plt.show()
torch.nn.utils.clip_grad_norm_(model.parameters(), opt.clip) # Always be safe than sorry optimizer.step() # Log losses losses.update(loss.data.item(), labels.size(0)) batch_time.update(time.time() - start) start = time.time() logger.info('==> Train:[{0}]\tTime:{batch_time.sum:.4f}\tData:{data_time.sum:.4f}\tLoss:{loss.avg:.4f}\t' .format(epoch, batch_time=batch_time, data_time=data_time, loss=losses)) return model, optimizer if __name__ == '__main__': # Parse arguments opt = parse_args() seed_everything(seed=opt.seed) # Setup logger console_logger = get_logger(folder=opt.log_dir+'/'+opt.exp_name+'/') # Handle fixed class orders. Note: Class ordering code hacky. Would need to manually adjust here to test for different datasets. console_logger.debug("==> Loading dataset..") class_order = [87, 0, 52, 58, 44, 91, 68, 97, 51, 15, 94, 92, 10, 72, 49, 78, 61, 14, 8, 86, 84, 96, 18, 24, 32, 45, 88, 11, 4, 67, 69, 66, 77, 47, 79, 93, 29, 50, 57, 83, 17, 81, 41, 12, 37, 59, 25, 20, 80, 73, 1, 28, 6, 46, 62, 82, 53, 9, 31, 75, 38, 63, 33, 74, 27, 22, 36, 3, 16, 21, 60, 19, 70, 90, 89, 43, 5, 42, 65, 76, 40, 30, 23, 85, 2, 95, 56, 48, 71, 64, 98, 13, 99, 7, 34, 55, 54, 26, 35, 39] #Currently testing using iCARL test order-- restricted to CIFAR100. For the other class orders refer to https://github.com/arthurdouillard/incremental_learning.pytorch/tree/master/options/data if opt.dataset != 'CIFAR100' and opt.dataset !='ImageNet100': class_order=None # Handle 'path does not exist errors' if not os.path.isdir(opt.log_dir+'/'+opt.exp_name): os.mkdir(opt.log_dir+'/'+opt.exp_name) if opt.old_exp_name!='test' and not os.path.isdir(opt.log_dir+'/'+opt.old_exp_name): os.mkdir(opt.log_dir+'/'+opt.old_exp_name)
if opt.tta: im_names, labels = eval_logits_tta(loader, model, device=device) else: im_names, labels = eval_logits(loader, model, device) im_labels = [] # print(im_names) for name, label in zip(im_names, labels): if name in dict_: dict_[name].append(label) else: dict_[name] = [label] header = ['filename', 'type'] utils.mkdir(opt.results_dir) result = opt.network + '-' + str(opt.layers) + '-' + str( opt.crop_size) + '_result.csv' filename = os.path.join(opt.results_dir, result) with open(filename, 'w', encoding='utf-8') as f: f_csv = csv.writer(f) f_csv.writerow(header) for key in dict_.keys(): v = np.argmax(np.sum(np.array(dict_[key]), axis=0)) + 1 # v = list(np.sum(np.array(dict_[key]), axis=0)) f_csv.writerow([key, v]) opt = opts.parse_args() main(opt)
print('Directory {} complete processing!'.format(day_dir)) print("Cost {} minutes".format((time.time() - st) / 60)) def main(args=None): source_dir = args.source_dir cur_dir_name = get_path_leaf(source_dir) if re.match('\d\d\d\d-\d\d-\d\d', cur_dir_name): day_dir = source_dir workflow(day_dir, args) else: sub_dirs = os.listdir(source_dir) day_dirs = [] for sub_dir in sub_dirs: if not re.match('\d\d\d\d-\d\d-\d\d', sub_dir): print('Sub directory {} under {} is not a date'.format( sub_dir, source_dir)) else: day_dirs.append(os.path.join(source_dir, sub_dir)) for day_dir in day_dirs: workflow(day_dir, args) if __name__ == '__main__': try: multiprocessing.set_start_method('spawn') except RuntimeError: pass main(parse_args())
writer.add_summary(summary, iteration) def check_rootfolders(trainid): """Create log and model folder""" folders_util = [args.root_log, args.root_model, args.root_output] if not os.path.exists('./data/results'): os.makedirs('./data/results') for folder in folders_util: if not os.path.exists(os.path.join('./data/results', trainid, folder)): print('creating folder ' + folder) os.makedirs(os.path.join('./data/results', trainid, folder)) if __name__ == '__main__': args = parse_args() if args.batch_size != 1: print('The batch size should always be 1 for now.') raise NotImplementedError check_rootfolders(args.train_id) summary_w = tf and tf.summary.FileWriter(\ os.path.join('./data/results', args.train_id, args.root_log)) # tensorboard print('Called with args:') print(args) np.random.seed(args.RNG_SEED) torch.backends.cudnn.enabled = False if torch.cuda.is_available() and not args.cuda: print("WARNING: You have a CUDA device, \ so you should probably run with --cuda")
frame_idx = int(random()*len(frames)) # 为了前一阵后一阵都能取到 if frame_idx<2: frame_idx=2 if frame_idx >len(frames)-2: frame_idx = len(frames)-2 s+=frames[frame_idx].stem if frames[frame_idx].stem =='0000001': print('cao!') if not s in return_list: return_list.append(s) i+=1 lenth = len(return_list) train_ = int(train_*lenth) val_ = int(val_*lenth) +train_ test_ = int(test_*lenth)+val_ writelines(return_list[:train_],train_txt_p) writelines(return_list[train_:val_],val_txt_p) writelines(return_list[val_:],test_txt_p) if __name__ == '__main__': options = parse_args() generate_mc(options)
for epoch in range(1, opt.num_pretrain_passes + 1): trainer.train(loader=vd.pretrain_loader, model=model, optimizer=optimizer, epoch=epoch) acc = trainer.test(loader=vd.pretest_loader, model=model, mask=vd.pretrain_mask, epoch_or_phase=epoch) logger.info(f'==> Pretraining completed! Acc: [{acc:.3f}]') save_pretrained_model(opt, model) if opt.num_tasks > 0: # TODO: use another optimizer? # Class-Incremental training # We start with pretrain mask bvecause in testing we want pretrained classes included logger.info(f'==> Starting Class-Incremental training') mask = vd.pretrain_mask.clone() if opt.num_pretrain_classes > 0 else torch.zeros(vd.n_classes_in_whole_dataset) dataloaders = vd.get_ci_dataloaders() cl_accuracy_meter = AverageMeter() for phase, (trainloader, testloader, class_list, phase_mask) in enumerate(dataloaders, start=1): trainer.train(loader=trainloader, model=model, optimizer=optimizer, phase=phase) # accumulate masks, because we want to test on all seen classes mask += phase_mask # this is the accuracy for all classes seen so far acc = trainer.test(loader=testloader, model=model, mask=mask, epoch_or_phase=phase) cl_accuracy_meter.update(acc) logger.info(f'==> CL training completed! AverageAcc: [{cl_accuracy_meter.avg:.3f}]') if __name__ == '__main__': cmd_opts = parse_args() exp1(cmd_opts)