cp_model = copy.deepcopy(model) if args.cuda: model.cuda() load_model = False if osp.exists(args.save): with open(args.save, 'rb') as fp: state = torch.load(fp) model.load_state_dict(state) load_model = True # Optimizer & loss optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) criterion = nn.CrossEntropyLoss(ignore_index=255) metrics = Evaluator(21) def train(epoch): model.train() for batch_idx, (data, target) in enumerate(train_loader): if args.cuda: data, target = data.cuda(), target.cuda() * 255. data, target = Variable(data), Variable(target).long().squeeze_(1) optimizer.zero_grad() output = model(data) loss = criterion(output['out'], target) loss.backward() optimizer.step() if batch_idx % args.log_interval == 0: print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
train, test = train_test_split(rows.values, cols.values, quantity) train_sparse = sparse.csr_matrix((train[2], (train[0], train[1])), shape=(len(customers), len(products))) alpha = 15 with Timer() as cython_als_t: user_vecs, item_vecs = implicit.alternating_least_squares( (train_sparse * alpha).astype('double'), factors=64, regularization=0.1, iterations=10, use_gpu=False) print(f"Time spent in implicit: {cython_als_t.interval}") evaluator = Evaluator(test[0], test[1], test[2], threshold=3.0) baseline_model = BaselinePredictor(train[1], train[2]) baseline_fpr, baseline_tpr, baseline_roc = evaluator.roc( lambda user, item: baseline_model.pred(item)) fpr, tpr, roc = evaluator.roc( lambda user, item: np.sum(user_vecs[user, :] * item_vecs[item, :])) print("AUC: %f" % roc) plt.clf() plt.plot(baseline_fpr, baseline_tpr, label='baseline') plt.plot(fpr, tpr, label='als') plt.xlabel('False positive') plt.ylabel('True positive') plt.legend() plt.show()
list_ = f.readlines() list_ = [v[:-1] for v in list_] ### gpu = 0 model_raft = RAFT() to_load = torch.load('./RAFT/models/raft-things.pth-no-zip') new_state_dict = OrderedDict() for k, v in to_load.items(): name = k[7:] # remove `module.`,表面从第7个key值字符取到最后一个字符,正好去掉了module. new_state_dict[name] = v #新字典的key值对应的value为一一对应的值。 model_raft.load_state_dict(new_state_dict) model_raft = model_raft.cuda(gpu) ### total_TC = 0. evaluator = Evaluator(num_class) for video in list_[:100]: imglist_ = sorted(os.listdir(os.path.join(data_dir, video, 'origin'))) for i, img in enumerate(imglist_[:-1]): #print('processing video : {} image: {}'.format(video,img)) next_img = imglist_[i + 1] imgname = img next_imgname = next_img img = Image.open(os.path.join(data_dir, video, 'origin', img)) next_img = Image.open(os.path.join(data_dir, video, 'origin', next_img)) image1 = torch.from_numpy(np.array(img)) image2 = torch.from_numpy(np.array(next_img)) padder = InputPadder(image1.size()[:2]) image1 = image1.unsqueeze(0).permute(0, 3, 1, 2) image2 = image2.unsqueeze(0).permute(0, 3, 1, 2)
from utils import Evaluator from networks.net_unetgenerator import UnetGenerator as G from easydict import EasyDict as edict from tqdm import tqdm from data.STB_dataset import STBdataset from data.RHD_dataset import RHDdataset if __name__ == "__main__": model = G(3, 3, 8) device = 0 cpm2d = './weights/cpm/stb_hpm2d.pth' cpm3d = './weights/cpm/stb_hpm3d.pth' weights = "./weights/pix2pix/stb_net_G.pth" evaluate = Evaluator(model, weights, cpm2d, cpm3d, device) opt = edict() opt.dataroot = "./dataset/stb-dataset/test" opt.isTrain = False dataset = STBdataset(opt) for i in tqdm(range(len(dataset))): sample = dataset[i] evaluate.feed(sample['xyz'], sample['A'] * 255.0) results = evaluate.evaluate() print(f"STB results: {results}") opt.dataroot = "./dataset/rhd-dataset/test" cpm2d = "./weights/cpm/rhd_hpm2d.pth" weights = "./weights/pix2pix/stb_net_G.pth" dataset = RHDdataset(opt) evaluate = Evaluator(model, weights, cpm2d, cpm3d, device) for i in tqdm(range(len(dataset))): sample = dataset[i]
import numpy as np import os from PIL import Image from utils import Evaluator import sys eval_ = Evaluator(124) eval_.reset() DIR = sys.argv[1] split = 'val.txt' with open(os.path.join(DIR, split), 'r') as f: lines = f.readlines() for line in lines: videolist = [line[:-1] for line in lines] PRED = sys.argv[2] for video in videolist: for tar in os.listdir(os.path.join(DIR, 'data', video, 'mask')): pred = os.path.join(PRED, video, tar) tar_ = Image.open(os.path.join(DIR, 'data', video, 'mask', tar)) tar_ = np.array(tar_) tar_ = tar_[np.newaxis, :] pred_ = Image.open(pred) pred_ = np.array(pred_) pred_ = pred_[np.newaxis, :] eval_.add_batch(tar_, pred_) Acc = eval_.Pixel_Accuracy() Acc_class = eval_.Pixel_Accuracy_Class() mIoU = eval_.Mean_Intersection_over_Union()
def mfea(config, callback): # Problem taskset = Taskset(config) # Model cf = ChromosomeFactory(config) # Simple parameter K = taskset.K config['K'] = K N = config['pop_size'] * K T = config['num_iter'] mutation_rate = config['mutation_rate'] rmp = config['rmp'] # Initialization population = cf.initialize() skill_factor = np.array([i % K for i in range(2 * N)]) factorial_cost = np.full([2 * N, K], np.inf) scalar_fitness = np.empty([2 * N]) # For parallel evaluation print('[+] Initializing evaluators') evaluators = [Evaluator(config) for _ in range(2 * N)] stabe_evaluators = [Evaluator(config) for _ in range(30)] # First evaluation (sequential) delayed_functions = [] for i in range(2 * N): sf = skill_factor[i] delayed_functions.append( delayed(evaluators[i].evaluate)(population[i], sf)) fitnesses = Parallel(n_jobs=cpu_count())(delayed_functions) for i in range(2 * N): sf = skill_factor[i] factorial_cost[i, sf] = fitnesses[i] scalar_fitness = calculate_scalar_fitness(factorial_cost) # Evolve iterator = trange(T) for t in iterator: # permute current population permutation_index = np.random.permutation(N) population[:N] = population[:N][permutation_index] skill_factor[:N] = skill_factor[:N][permutation_index] factorial_cost[:N] = factorial_cost[:N][permutation_index] factorial_cost[N:] = np.inf # select pair to crossover for i in range(0, N, 2): # extract parent p1, p2 = population[i], population[i + 1] sf1, sf2 = skill_factor[i], skill_factor[i + 1] # recombine parent if sf1 == sf2: c1, c2 = cf.one_point_crossover_adf(p1, p2) c1 = cf.uniform_mutate(c1, mutation_rate) c2 = cf.uniform_mutate(c2, mutation_rate) skill_factor[N + i] = sf1 skill_factor[N + i + 1] = sf1 elif sf1 != sf2 and np.random.rand() < rmp: c1, c2 = cf.one_point_crossover_adf_multitask(p1, p2) c1 = cf.uniform_mutate(c1, mutation_rate) c2 = cf.uniform_mutate(c2, mutation_rate) if np.random.rand() < 0.5: skill_factor[N + i] = sf1 else: skill_factor[N + i] = sf2 if np.random.rand() < 0.5: skill_factor[N + i + 1] = sf1 else: skill_factor[N + i + 1] = sf2 else: p2 = find_relative(population, skill_factor, sf1, N) c1, c2 = cf.one_point_crossover_adf(p1, p2) c1 = cf.uniform_mutate(c1, mutation_rate) c2 = cf.uniform_mutate(c2, mutation_rate) skill_factor[N + i] = sf1 skill_factor[N + i + 1] = sf1 population[N + i, :], population[N + i + 1, :] = c1[:], c2[:] # evaluation delayed_functions = [] for i in range(2 * N): sf = skill_factor[i] delayed_functions.append( delayed(evaluators[i].evaluate)(population[i], sf)) fitnesses = Parallel(n_jobs=cpu_count())(delayed_functions) for i in range(2 * N): sf = skill_factor[i] factorial_cost[i, sf] = fitnesses[i] scalar_fitness = calculate_scalar_fitness(factorial_cost) # sort sort_index = np.argsort(scalar_fitness)[::-1] population = population[sort_index] skill_factor = skill_factor[sort_index] factorial_cost = factorial_cost[sort_index] scalar_fitness = scalar_fitness[sort_index] # optimization info message = {'algorithm': 'mfea', 'instance': config['name']} results = get_optimization_results(t, population, factorial_cost, scalar_fitness, skill_factor, message) stabe_results = deepcopy(results) for i in range(len(stabe_results)): sf = stabe_results[i].sf chromosome = stabe_results[i].x stabe_delayed_functions = [] for j in range(30): stabe_delayed_functions.append( delayed(stabe_evaluators[j].evaluate)(chromosome, sf)) stabe_fitness = Parallel( n_jobs=cpu_count())(stabe_delayed_functions) stabe_results[i].fun = sum(stabe_fitness) / len(stabe_fitness) if callback: callback(results, stabe_results) desc = 'gen:{} fitness:{} message:{} K:{}'.format( t, ' '.join( '{:0.2f}|{:0.2f}|{:0.2f}'.format(res.fun, res.mean, res.std) for res in results), message, K) iterator.set_description(desc)
def train(args): base_dir = args.base_dir dirs = init_dir(base_dir) init_log(dirs['log']) config_dir = args.config_dir copy_file(config_dir, dirs['data']) config = configparser.ConfigParser() config.read(config_dir) in_test, post_test = init_test_flag(args.test_mode) # init env env = init_env(config['ENV_CONFIG']) logging.info('Training: a dim %d, agent dim: %d' % (env.n_a, env.n_agent)) # init step counter total_step = int(config.getfloat('TRAIN_CONFIG', 'total_step')) test_step = int(config.getfloat('TRAIN_CONFIG', 'test_interval')) log_step = int(config.getfloat('TRAIN_CONFIG', 'log_interval')) global_counter = Counter(total_step, test_step, log_step) # init centralized or multi agent seed = config.getint('ENV_CONFIG', 'seed') if env.agent == 'ia2c': model = IA2C(env.n_s_ls, env.n_a, env.neighbor_mask, env.distance_mask, env.coop_gamma, total_step, config['MODEL_CONFIG'], seed=seed) elif env.agent == 'ia2c_fp': model = IA2C_FP(env.n_s_ls, env.n_a, env.neighbor_mask, env.distance_mask, env.coop_gamma, total_step, config['MODEL_CONFIG'], seed=seed) elif env.agent == 'ma2c_nc': model = MA2C_NC(env.n_s, env.n_a, env.neighbor_mask, env.distance_mask, env.coop_gamma, total_step, config['MODEL_CONFIG'], seed=seed) else: model = None # disable multi-threading for safe SUMO implementation summary_writer = tf.summary.FileWriter(dirs['log']) trainer = Trainer(env, model, global_counter, summary_writer, in_test, output_path=dirs['data']) trainer.run() # save model final_step = global_counter.cur_step logging.info('Training: save final model at step %d ...' % final_step) model.save(dirs['model'], final_step) # post-training test if post_test: test_dirs = init_dir(base_dir, pathes=['eva_data']) evaluator = Evaluator(env, model, test_dirs['eva_data']) evaluator.run()
def main(): args = parse_args() print('Setting Arguments...', args) os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu os.makedirs(os.path.join(args.checkpoint_dir, args.prefix), exist_ok=True) with open(args.cfg, 'r') as f: cfg = yaml.load(f) print(cfg) batch_size = cfg['TRAIN']['BATCH_SIZE'] base_lr = cfg['MODEL']['LR'] / batch_size schedule = eval(cfg['TRAIN']['SCHEDULE']) weight_decay = eval(cfg['TRAIN']['DECAY']) max_epoch = cfg['TRAIN']['MAX_EPOCH'] valid_interval = cfg['INTERVAL'] train_transforms = transforms.Compose([ transforms.Resize((256, 256)), transforms.ToTensor(), torchvision.transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ]) valid_transforms = transforms.Compose([ transforms.Resize((256, 256)), transforms.ToTensor(), torchvision.transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ]) train_dataset = NIHDataset( cfg['PATH']['TRAIN']['ENTRY_FILE'], transforms=train_transforms, mode='train', ) subtrain_dataset = NIHDataset( cfg['PATH']['VALID']['ENTRY_FILE_A'], transforms=valid_transforms, mode='valid', ) valid_dataset = NIHDataset( cfg['PATH']['VALID']['ENTRY_FILE_B'], transforms=valid_transforms, mode='valid', ) train_loader = DataLoader( train_dataset, batch_size=batch_size, shuffle=True, num_workers=24, ) subtrain_loader = DataLoader( subtrain_dataset, batch_size=batch_size, shuffle=False, num_workers=24, ) valid_loader = DataLoader( valid_dataset, batch_size=batch_size, shuffle=False, num_workers=24, ) n_images = train_dataset.n_images def burnin_schedule(i, burn_in=1000): if i < burn_in: # factor = pow(i / burn_in, 3) factor = 1. elif i < schedule[0] * n_images / batch_size: factor = 1. elif i < schedule[1] * n_images / batch_size: factor = 0.1 elif i < schedule[2] * n_images / batch_size: factor = 0.01 return factor net = chest_net(cfg) if args.weights_path: pass elif args.checkpoint: print("loading pytorch ckpt...", args.checkpoint) state = torch.load(args.checkpoint) if 'model_state_dict' in state.keys(): net.load_state_dict(state['model_state_dict']) else: net.load_state_dict(state) net.cuda() criterion = NIHLoss(batch_size) optimizer = torch.optim.Adam(net.parameters(), lr=base_lr, amsgrad=True) scheduler = optim.lr_scheduler.LambdaLR(optimizer, burnin_schedule) params_dict = dict(net.named_parameters()) params = [] for key, value in params_dict.items(): if 'conv.weight' in key: params += [{ 'params': value, 'weight_decay': weight_decay / batch_size }] else: params += [{'params': value, 'weight_decay': 0.0}] logger = SummaryWriter(os.path.join(args.tfboard, args.prefix)) iter_train_loader = iter(train_loader) max_iter = n_images * max_epoch // batch_size epoch = 1 train_logger = TrainLogger( max_iter, max_epoch, train_loader, criterion, logger, ) for i_step in range(1, max_iter + 1, 1): try: _, images, target = next(iter_train_loader) except StopIteration: # start eval epoch += 1 train_logger = TrainLogger( max_iter, max_epoch, train_loader, criterion, logger, ) if epoch % valid_interval == 1: min_loss = np.inf train_evaluator = Evaluator( i_step, net, subtrain_loader, criterion, logger, args.prefix, mode='train', ) valid_evaluator = Evaluator( i_step, net, valid_loader, criterion, logger, args.prefix, mode='valid', ) if valid_evaluator.loss < min_loss: print('seva model...') print(f'f1 {valid_evaluator.f1}') print(f'acc {valid_evaluator.acc}') print(f'loss {valid_evaluator.loss}') min_loss = valid_evaluator.loss torch.save( { 'iter': i_step, 'epoch': epoch, 'f1': valid_evaluator.f1, 'loss': valid_evaluator.loss, 'model_state_dict': net.state_dict(), 'optimizer_state_dict': optimizer.state_dict() }, os.path.join(args.checkpoint_dir, args.prefix, f'best_model.ckpt')) iter_train_loader = iter(train_loader) _, images, target = next(iter_train_loader) images = Variable(images).cuda().float() target = Variable(target).cuda().float() net.train() optimizer.zero_grad() pred = net(images) loss = criterion(pred, target) loss.backward() optimizer.step() scheduler.step() current_lr = scheduler.get_lr()[0] * batch_size train_logger.update(i_step, pred, target, current_lr, epoch) logger.close()
def main(cfg, gpu, args): num_class = args.num_class torch.cuda.set_device(gpu) # Network Builders net_encoder = ModelBuilder.build_encoder(arch=cfg.MODEL.arch_encoder, fc_dim=cfg.MODEL.fc_dim, weights=cfg.MODEL.weights_encoder) net_decoder = ModelBuilder.build_decoder(arch=cfg.MODEL.arch_decoder, fc_dim=cfg.MODEL.fc_dim, num_class=num_class, weights=cfg.MODEL.weights_decoder, use_softmax=True) crit = nn.NLLLoss(ignore_index=-1) segmentation_module = SegmentationModule(net_encoder, net_decoder, crit) to_load = torch.load(args.load, map_location=torch.device("cuda:" + str(args.start_gpu))) new_state_dict = OrderedDict() for k, v in to_load.items(): name = k[7:] # remove `module.`,表面从第7个key值字符取到最后一个字符,正好去掉了module. new_state_dict[name] = v #新字典的key值对应的value为一一对应的值。 segmentation_module.load_state_dict(new_state_dict) print('load model parameters') segmentation_module.cuda(args.start_gpu) with open(os.path.join(args.dataroot, args.split + '.txt')) as f: lines = f.readlines() videolists = [line[:-1] for line in lines] # Dataset and Loader evaluator = Evaluator(num_class) eval_video = Evaluator(num_class) evaluator.reset() eval_video.reset() total_vmIOU = 0.0 total_vfwIOU = 0.0 total_video = len(videolists) v = [] n = [] for video in videolists: eval_video.reset() dataset_test = TestDataset(args.dataroot, video, args) loader_test = torch.utils.data.DataLoader(dataset_test, batch_size=args.batchsize, shuffle=False, num_workers=5, drop_last=False) # Main loop test(segmentation_module, loader_test, gpu, args, evaluator, eval_video, video) if args.split != 'test': v_mIOU = eval_video.Mean_Intersection_over_Union() v.append(v) n.append(video) print(video, v_mIOU) total_vmIOU += v_mIOU v_fwIOU = eval_video.Frequency_Weighted_Intersection_over_Union() total_vfwIOU += v_fwIOU if args.split != 'test': total_vmIOU = total_vmIOU / total_video total_vfwIOU = total_vfwIOU / total_video Acc = evaluator.Pixel_Accuracy() Acc_class = evaluator.Pixel_Accuracy_Class() mIoU = evaluator.Mean_Intersection_over_Union() FWIoU = evaluator.Frequency_Weighted_Intersection_over_Union() print( "Acc:{}, Acc_class:{}, mIoU:{}, fwIoU: {}, video mIOU: {}, video fwIOU: {}" .format(Acc, Acc_class, mIoU, FWIoU, total_vmIOU, total_vfwIOU)) print('Inference done!')