def main(): device = torch.device('cpu') torch.manual_seed(1234) layer = 2 size = 1024 func = "relu" model = MyModel(layer, size) model.load_state_dict(torch.load('model_state_dict_final')) model.eval() transforms = T.Compose([T.ToTensor(), T.Normalize((0.5, ), (0.5, ))]) test_dataset = MnistDatasetTest('data', 'test', transforms) test_dataloader = DataLoader(test_dataset, batch_size=64, shuffle=False, num_workers=4) result = open("result.txt", "w") # to create a result with torch.no_grad(): for images, image_name in test_dataloader: images = images.to(device) prediction = model(images, layer, func) for i in range(images.size()[0]): x = image_name[i] + ' ' + str(int(torch.argmax(prediction[i]))) result.write(x) result.write("\n") result.close()
def _load(self, filePath): checkpoint = torch.load(filePath) model = MyModel( device, checkpoint['inputSize'], checkpoint['gatedCnnOutputSize'], checkpoint['gatedCnnStride1'], checkpoint['gatedCnnStride2'], checkpoint['gatedCnnKernel1'], checkpoint['gatedCnnKernel2'], checkpoint['lstmLayer'], checkpoint['lstmHiddenSize'], checkpoint['fcOutputSize'], checkpoint['dropout']) model.load_state_dict(checkpoint['stateDict']) model.eval() if self.device.type == 'cpu': model.cpu() else: model.cuda(device=self.device) return model
def run(): use_cuda = torch.cuda.is_available() device = torch.device('cuda' if use_cuda else 'cpu') dataset = [ {"in" : [0.0, 0.0], "out" : [0.0]}, {"in" : [0.0, 1.0], "out" : [1.0]}, {"in" : [1.0, 0.0], "out" : [1.0]}, {"in" : [1.0, 1.0], "out" : [0.0]}] data_loader = get_data_loader(dataset=dataset, shuffle=True) model = MyModel() model.to(device) optimizer = optim.Adam(model.parameters()) loss_func = nn.MSELoss() model.train() for _ in range(0, 2000): for output, input in data_loader: input = input.to(device) output = output.to(device) result = model(input) loss = loss_func(result, output) optimizer.zero_grad() loss.backward() optimizer.step() test_data_loader = get_data_loader(dataset=dataset, shuffle=False) model.eval() with torch.no_grad(): for _, input in test_data_loader: print(input) input = input.to(device) result = model(input) print(result)
def train(batch_size=16, pretrain_model_path='', name='', model_type='mlp', dim=1024, lr=1e-5, epoch=12, smoothing=0.05, sample=False, open_ad='', dialog_name='xxx'): if not pretrain_model_path or not name: assert 1==-1 # print('\n********** model type:', model_type, '**********') # print('batch_size:', batch_size) # load dataset train_file = '../data/Dataset/my_train.csv' dev_file = '../data/Dataset/my_dev.csv' train_num = len(pd.read_csv(train_file).values.tolist()) val_num = len(pd.read_csv(dev_file).values.tolist()) print('train_num: %d, dev_num: %d' % (train_num, val_num)) # 选择模型 if model_type in ['siam', 'esim', 'sbert']: assert 1==-1 else: train_iter = MyDataset(file=train_file, is_train=True, sample=sample, pretrain_model_path=pretrain_model_path) train_iter = get_dataloader(train_iter, batch_size, shuffle=True, drop_last=True) dev_iter = MyDataset(file=dev_file, is_train=True, sample=sample, pretrain_model_path=pretrain_model_path) dev_iter = get_dataloader(dev_iter, batch_size, shuffle=False, drop_last=False) if model_type == 'mlp': model = MyModel(dim=dim, pretrain_model_path=pretrain_model_path, smoothing=smoothing) elif model_type == 'cnn': model = MyTextCNNModel(dim=dim, pretrain_model_path=pretrain_model_path, smoothing=smoothing) elif model_type == 'rcnn': model = MyRCNNModel(dim=dim, pretrain_model_path=pretrain_model_path, smoothing=smoothing) model.to(device) model_param_num = 0 ##### 3.24 muppti-gpu-training if n_gpu > 1: model = torch.nn.DataParallel(model) for p in model.parameters(): if p.requires_grad: model_param_num += p.nelement() print('param_num:%d\n' % model_param_num) # 加入对抗训练,提升泛化能力;但是训练速度明显变慢 (插件式调用) # 3.12 change to FGM 更快! if open_ad == 'fgm': fgm = FGM(model) elif open_ad == 'pgd': pgd = PGD(model) K = 3 # model-store-path model_path = '../user_data/model_store/' + name + '.pkl' # 输出模型默认存放在当前路径下 state = {} time0 = time.time() best_loss = 999 early_stop = 0 for e in range(epoch): print("*" * 100) print("Epoch:", e) param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [ {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01}, {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] optimizer = BertAdam(optimizer_grouped_parameters, lr=lr, warmup=0.05, t_total=len(train_iter)) # 设置优化器 train_loss = 0 train_c = 0 train_right_num = 0 model.train() # 将模型设置成训练模式(Sets the module in training mode) print('training..., %s, e:%d, lr:%7f' % (name, e, lr)) for batch in tqdm(train_iter): # 每一次返回 batch_size 条数据 optimizer.zero_grad() # 清空梯度 batch = [b.cuda() for b in batch] # cpu -> GPU # 正常训练 labels = batch[-1].view(-1).cpu().numpy() loss, bert_enc = model(batch, task='train', epoch=epoch) # 进行前向传播,真正开始训练;计算 loss right_num = count_right_num(bert_enc, labels) # multi-gpu training! if n_gpu > 1: loss = loss.mean() loss.backward() # 反向传播计算参数的梯度 if open_ad == 'fgm': # 对抗训练 fgm.attack() # 在embedding上添加对抗扰动 if model_type == 'multi-task': loss_adv, _, _ = model(batch, task='train') else: loss_adv, _ = model(batch, task='train') if n_gpu > 1: loss_adv = loss_adv.mean() loss_adv.backward() # 反向传播,并在正常的grad基础上,累加对抗训练的梯度 fgm.restore() # 恢复embedding参数 elif open_ad == 'pgd': pgd.backup_grad() # 对抗训练 for t in range(K): pgd.attack(is_first_attack=(t==0)) # 在embedding上添加对抗扰动, first attack时备份param.data if t != K-1: optimizer.zero_grad() else: pgd.restore_grad() if model_type == 'multi-task': loss_adv, _, _ = model(batch, task='train') else: loss_adv, _ = model(batch, task='train') if n_gpu > 1: loss_adv = loss_adv.mean() loss_adv.backward() # 反向传播,并在正常的grad基础上,累加对抗训练的梯度 pgd.restore() # 恢复embedding参数 optimizer.step() # 更新参数 train_loss += loss.item() # loss 求和 train_c += 1 train_right_num += right_num val_loss = 0 val_c = 0 val_right_num = 0 model.eval() print('eval...') with torch.no_grad(): # 不进行梯度的反向传播 for batch in tqdm(dev_iter): # 每一次返回 batch_size 条数据 batch = [b.cuda() for b in batch] labels = batch[-1].view(-1).cpu().numpy() loss, bert_enc = model(batch, task='train', epoch=epoch) # 进行前向传播,真正开始训练;计算 loss right_num = count_right_num(bert_enc, labels) if n_gpu > 1: loss = loss.mean() val_c += 1 val_loss += loss.item() val_right_num += right_num train_acc = train_right_num / train_num val_acc = val_right_num / val_num print('train_acc: %.4f, val_acc: %.4f' % (train_acc, val_acc)) print('train_loss: %.4f, val_loss: %.4f, time: %d' % (train_loss/train_c, val_loss/val_c, time.time()-time0)) if val_loss / val_c < best_loss: early_stop = 0 best_loss = val_loss / val_c best_acc = val_acc # 3.24 update 多卡训练时模型保存避坑: model_to_save = model.module if hasattr(model, 'module') else model state['model_state'] = model_to_save.state_dict() state['loss'] = val_loss / val_c state['acc'] = val_acc state['e'] = e state['time'] = time.time() - time0 state['lr'] = lr torch.save(state, model_path) best_epoch = e cost_time = time.time() - time0 tmp_train_acc = train_acc best_model = model else: early_stop += 1 if early_stop == 2: break model = best_model lr = lr * 0.5 print("best_loss:", best_loss) # 3.12 add 打印显示最终的最优结果 print('-' * 30) print('best_epoch:', best_epoch, 'best_loss:', best_loss, 'best_acc:', best_acc, 'reach time:', cost_time, '\n') # model-clean del model gc.collect() # 实验结果写入日志 with open('../user_data/model_dialog/' + dialog_name + '.out', 'w', encoding='utf-8') as f: f.write('*** model name:' + dialog_name + ' ***\n') f.write('best dev acc:' + str(best_acc) + '\n') f.write('best loss:' + str(best_loss) + '\n') f.write('best_epoch:' + str(best_epoch) + '\n') f.write('train acc:' + str(tmp_train_acc) + '\n') f.write('lr:' + str(state['lr']) + '\n') f.write('time:' + str(cost_time) + '\n')
def infer(PATH_IMG_INFER, PATH_MODEL_PARAMS, PATH_INFER_SAVE, DEPTH, NUM_CHANNELS, MULT_CHAN, NUM_CLASSES, Z_RANGE, IMG_WIDTH, IMG_HEIGHT): ### instantiate model and load the model parameters my_model = MyModel(n_in_channels=NUM_CHANNELS, mult_chan=MULT_CHAN, depth=DEPTH) my_model.load_state_dict( torch.load(PATH_MODEL_PARAMS, map_location=torch.device('cpu'))) my_model.eval() ### read in the tiff and place into a ZStack object stack = ZStack(path=PATH_IMG_INFER, zrange=Z_RANGE) # separate zslices; assign each to its own Slice object, and hold them all in a list slices = [Slice(x) for x in stack.images] # split each slice into tiles tiles = np.array( [slices[z].tile(IMG_WIDTH, IMG_HEIGHT) for z in range(len(slices))]) # normalize (z-score) the tiles before passing to model [[tiles[y][x].normalize() for x in range(0, tiles.shape[1])] for y in range(0, tiles.shape[0])] # reshape data; h=z_slices, w=numtiles for each z-slice h, w = np.shape(tiles) flat_tiles = np.zeros((h * w, 1, IMG_WIDTH, IMG_HEIGHT), dtype=np.float32) for i in range(h * w): flat_tiles[i] = tiles.flatten(order='C')[i].reshape( IMG_WIDTH, IMG_HEIGHT).slice # load data into pytorch data structs dataset = FormsDataset(flat_tiles, num_classes=NUM_CLASSES) hold = [] data_loader = DataLoader(dataset, batch_size=1, shuffle=False) for i, images in enumerate(data_loader, 1): images = images.type(torch.FloatTensor) hold.append(images) # now lets perform inference on the tiles, store them in 'predictions' predictions = [] for tile in hold: predictions.append(my_model(tile)) # and lets time ourselves print("--- Inference: {} seconds ---".format(time.time() - start_time)) ### now we have to stitch our images back together! unflat_predictions = np.array(predictions).reshape((h, w)) stack_infer = [] for z in unflat_predictions: stack_infer.append( stitch(z, IMG_WIDTH=IMG_WIDTH, IMG_HEIGHT=IMG_HEIGHT).toimage()) stack_infer[0].save(PATH_INFER_SAVE, save_all=True, append_images=stack_infer[1:]) print("--- Finish: {} seconds ---".format(time.time() - start_time))
watch_dataset(data_loader_train) for epoch in range(options.args.epoch): # train model.train() for data, label in data_loader_train: data, label = data.to(device), label.to(device) optimizer.zero_grad() output = model(data) loss = loss_func(output, label) loss.backward() optimizer.step() print("loss: %f" % loss.item()) # test model.eval() acc = 0 with torch.no_grad(): for data, label in data_loader_test: data, label = data.to(device), label.to(device) output = model(data) answer = torch.argmax(output, dim=-1) acc += torch.sum(answer == label).item() accuracy = acc / len(data_loader_test.dataset) print("accuracy %.6f" % accuracy) save_model(model, epoch, options)
def main(args): # device device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # tensorboard writer writer = SummaryWriter(comment='#layers{}_emb{}_multitask{}'.format(args['num_hidden_layers'], args['embedding_size'], args['multitask_weights'])) # logger logger = logging.getLogger('#layers{}_emb{}_multitask{}'.format(args['num_hidden_layers'], args['embedding_size'], args['multitask_weights'])) # experiment name formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') file_handler = logging.FileHandler("log/training_log.log") file_handler.setFormatter(formatter) logger.addHandler(file_handler) logger.setLevel(logging.DEBUG) # set random seed for reproducibility torch.manual_seed(2019) np.random.seed(2019) # load data data = utils.load_dataset(year=args['year']) # trips -- [(src, dst, cnt)] train_data = data['train'] valid_data = data['valid'] test_data = data['test'] # in/out flow counts -- [(count)] train_inflow = data['train_inflow'] train_outflow = data['train_outflow'] # geographical features -- [[features]] node_feats = data['node_feats'] # census tract adjacency as matrix ct_adj = data['ct_adjacency_withweight'] # number of nodes num_nodes = data['num_nodes'] # post-processing # trip data train_data = torch.from_numpy(train_data) trip_od_train = train_data[:, :2].long().to(device) trip_volume_train = train_data[:, -1].float().to(device) trip_od_valid = torch.from_numpy(valid_data[:, :2]).long().to(device) trip_volume_valid = torch.from_numpy(valid_data[:, -1]).float().to(device) trip_od_test = torch.from_numpy(test_data[:, :2]).long().to(device) trip_volume_test = torch.from_numpy(test_data[:, -1]).float().to(device) # in/out flow data for multitask target in/out flow train_inflow = torch.from_numpy(train_inflow).view(-1, 1).float().to(device) train_outflow = torch.from_numpy(train_outflow).view(-1, 1).float().to(device) # construct graph using adjacency matrix g = utils.build_graph_from_matrix(ct_adj, node_feats.astype(np.float32), device) g.to(device) # init model model = MyModel(g, num_nodes, in_dim = node_feats.shape[1], h_dim = args['embedding_size'], num_hidden_layers=args['num_hidden_layers'], dropout=0, device=device, reg_param=args['reg_param']) model.to(device) # training recorder model_state_file = './models/model_state_layers{}_emb{}_multitask{}.pth'.format(args['num_hidden_layers'], args['embedding_size'], args['multitask_weights']) best_rmse = 1e6 # create a optimizer optimizer = torch.optim.Adam(model.parameters(), lr=args['lr']) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 100, gamma=0.1) # training process for epoch in range(args['max_epochs']): # turn model state model.train() # create a mini-batch generator mini_batch_gen = utils.mini_batch_gen(train_data, mini_batch_size = int(args['mini_batch_size']), num_nodes=num_nodes, negative_sampling_rate = 0) # SGD from each mini-batch for mini_batch in mini_batch_gen: # clear gradients optimizer.zero_grad() # get trip od trip_od = mini_batch[:, :2].long().to(device) # get trip volume scaled_trip_volume = utils.scale(mini_batch[:, -1].float()).to(device) # evaluate loss loss = model.get_loss(trip_od, scaled_trip_volume, train_inflow, train_outflow, g, multitask_weights=args['multitask_weights']) # add to tensorboard writer.add_scalar('mini_loss', loss.item(), global_step = epoch) # back propagation to get gradients loss.backward() # clip to make stable torch.nn.utils.clip_grad_norm_(model.parameters(), args['grad_norm']) # update weights by optimizer optimizer.step() # scheduler update learning rate scheduler.step() # report training epoch if logger.level == logging.DEBUG: model.eval() # loss function with torch.no_grad(): loss = model.get_loss(trip_od_train, utils.scale(trip_volume_train), train_inflow, train_outflow, g) # metric on train dataset rmse, mae, mape, cpc, cpl = utils.evaluate(model, g, trip_od_train, trip_volume_train) # report logger.debug("Evaluation on train dataset:") logger.debug("Epoch {:04d} | Loss = {:.4f}".format(epoch, loss)) logger.debug("RMSE {:.4f} | MAE {:.4f} | MAPE {:.4f} | CPC {:.4f} | CPL {:.4f} |".format(rmse, mae, mape, cpc, cpl)) writer.add_scalar('overall-loss', loss.item(), epoch) writer.add_scalar('RMSE', rmse, epoch) writer.add_scalar('MAE', mae, epoch) writer.add_scalar('MAPE', mape, epoch) writer.add_scalar('CPC', cpc, epoch) writer.add_scalar('CPL', cpl, epoch) # validation part if epoch % args['evaluate_every'] == 0: # turn model state to eval model.eval() # loss function with torch.no_grad(): loss = model.get_loss(trip_od_valid, utils.scale(trip_volume_valid), train_inflow, train_outflow, g) # evaluate on validation set rmse, mae, mape, cpc, cpl = utils.evaluate(model, g, trip_od_valid, trip_volume_valid) # report logger.info("-----------------------------------------") logger.info("Evaluation on Validation:") logger.info("Epoch {:04d} | Loss = {:.4f}".format(epoch, loss)) logger.info("RMSE {:.4f} | MAE {:.4f} | MAPE {:.4f} | CPC {:.4f} | CPL {:.4f} |".format(rmse, mae, mape, cpc, cpl)) # save best model if rmse < best_rmse: # update indicator best_rmse = rmse # save model torch.save({'state_dict': model.state_dict(), 'epoch': epoch, 'rmse': rmse, 'mae': mae, 'mape': mape, 'cpc': cpc, 'cpl': cpl}, model_state_file) # save embeddings src_embedding = model(g).detach().cpu().numpy() # get embeddings dst_embedding = model.forward2(g).detach().cpu().numpy() # get embeddings emb_fp = "./embeddings/censustract_embeddings_year{}_layers{}_emb{}_multitask{}.npz".format(args['year'], args['num_hidden_layers'] , args['embedding_size'], args['multitask_weights']) np.savez(emb_fp, src_embedding, dst_embedding) # report logger.info('Best RMSE found on epoch {}'.format(epoch)) logger.info("-----------------------------------------")
logger.histo_summary(tag, value.data.cpu().numpy(), epoch * total_step + i) logger.histo_summary(tag + '/grad', value.grad.data.cpu().numpy(), epoch * total_step + i) # 3. Log training images (image summary) # info = {'images': joint.view(-1, 24, 2)[:10].cpu().numpy()} # for tag, images in info.items(): # logger.image_summary(tag, images, step+1) loss = 0 if epoch % 1 == 0: # Test the model model.eval( ) # eval mode (batchnorm uses moving mean/variance instead of mini-batch mean/variance) with torch.no_grad(): for joint, pose in test_loader: joint = joint.to(device) pose = pose.to(device) outputs = model(joint) loss += criterion(outputs, pose) loss = loss / len(test_loader) print( 'Test Loss of the model on the {} test images: {:.4f} '.format( total_step * batch_size, loss)) logger.scalar_summary('test_loss', loss.item(), epoch) if loss < last_loss: last_loss = loss # Save the model checkpoint torch.save(model.state_dict(),
def predict(dim, names, weight, batch_size, pretrain_model_path, model_types=None): print('-' * 100) print('multi-models begin predicting ...') print('-' * 100) # read test data test_file = '../data/Dataset/test.csv' # data test_df = pd.read_csv(test_file) test_ids = test_df['id'].values.tolist() result_prob_tmp = torch.zeros((len(test_ids), 2)) # load model for i, name in enumerate(names): # 3.17 add weight_ = weight[i] model_path = '../user_data/model_store/' + name + '.pkl' state = torch.load(model_path) # 3.10 add model_type = model_types[i] if model_type == 'mlp': test_iter = MyDataset(file=test_file, is_train=False, pretrain_model_path=pretrain_model_path[i]) test_iter = get_dataloader(test_iter, batch_size, shuffle=False, drop_last=False) model = MyModel(dim=dim[i], pretrain_model_path=pretrain_model_path[i]) elif model_type == 'cnn': test_iter = MyDataset(file=test_file, is_train=False, pretrain_model_path=pretrain_model_path[i]) test_iter = get_dataloader(test_iter, batch_size, shuffle=False, drop_last=False) model = MyTextCNNModel(dim=dim[i], pretrain_model_path=pretrain_model_path[i]) elif model_type == 'rcnn': test_iter = MyDataset(file=test_file, is_train=False, pretrain_model_path=pretrain_model_path[i]) test_iter = get_dataloader(test_iter, batch_size, shuffle=False, drop_last=False) model = MyRCNNModel(dim=dim[i], pretrain_model_path=pretrain_model_path[i]) model.to(device) model.load_state_dict(state['model_state']) model.eval() print('-'*20, 'model', i, '-'*20) print('load model:%s, loss:%.4f, e:%d, lr:%.7f, time:%d' % (name, state['loss'], state['e'], state['lr'], state['time'])) # predict with torch.no_grad(): j = 0 for batch in tqdm(test_iter): batch = [b.cuda() for b in batch] out = model(batch, task='eval') out = out.cpu() # gpu -> cpu if j == 0: tmp = out # 初始化 tmp else: tmp = torch.cat([tmp, out], dim=0) # 将之后的预测结果拼接到 tmp 中 j += 1 # 当前 模型预测完成 print('model', i, 'predict finished!\n') # 3.17 按权重融合 result_prob_tmp += (weight_ / len(names)) * tmp # 删除模型 del model gc.collect() time.sleep(1) # 3.10 当前融合策略:prob 简单的取 avg _, result = torch.max(result_prob_tmp, dim=-1) result = result.numpy() # 3.16 update: label 0的prob 大于 3,就认为是 label=0 # with open('tmp.txt', 'w', encoding='utf-8') as f: # for r in result_prob_tmp: # f.write(str(r) + '\n') # save result df = pd.DataFrame() df['id'] = test_ids df['label'] = result df.to_csv(("../prediction_result/result_"+datetime.datetime.now().strftime('%Y%m%d_%H%M%S') + ".csv"), encoding='utf-8', index=False)
def extract_cache_features(cfg, epoch, loader, dataset, test_batch_size, uuids, nls): # extract img fts first to save time if not os.path.exists('cache'): # shutil.rmtree('cache') os.mkdir('cache') if not os.path.exists(f'cache/{epoch}'): os.mkdir(f'cache/{epoch}') model = MyModel(cfg, len(dataset.nl), dataset.nl.word_to_idx['<PAD>'], nn.BatchNorm2d, num_colors=len(CityFlowNLDataset.colors), num_types=len(CityFlowNLDataset.vehicle_type) - 2).cuda() saved_dict = torch.load(f'save/{epoch}.pth') n = {} for k, v in saved_dict.items(): n[k.replace('module.', '')] = v model.load_state_dict(n, False) model.eval() with torch.no_grad(): for idx, (id, frames, _, _, _, labels) in enumerate(tqdm(loader)): frames = frames.squeeze(0).cuda() labels = labels.squeeze(0).cuda() # b = frames.shape[0] # cache = [] # version 3 # if b <= test_batch_size: # cache = model.cnn(frames) # torch.save(cache, f'cache/{epoch}/{idx}_0.pth') # else: # cache = [] for i, (f, l) in enumerate( zip(frames.split(test_batch_size), labels.split(test_batch_size))): cache = model(None, f, l) img_ft = cache[0] color = F.softmax(cache[1].mean(dim=0), dim=0).cpu() #.numpy() typ = F.softmax(cache[2].mean(dim=0), dim=0).cpu() #.numpy() # color = np.argmax(color) # typ = np.argmax(typ) cache = [img_ft, color, typ] torch.save(cache, f'cache/{epoch}/{idx}_{i}.pth') break print('saving language features..') for uuid, query_nl in zip(uuids, nls): nls_list = [] query_nl, vehicle_type = CityFlowNLDataset.type_replacer( query_nl) query_nl, vehicle_color = CityFlowNLDataset.color_replacer( query_nl) # max_len = max([len(dataset.nl.do_clean(nl)) for nl in query_nl]) for nl in query_nl: nl = torch.tensor( dataset.nl.sentence_to_index(nl, is_train=False)).cuda() # nls.append(nl.unsqueeze(0).transpose(1, 0)) nl = nl.unsqueeze(0).transpose(1, 0) # bs, len, dim nl = model.rnn(nl) nls_list.append(nl) saved_nls = { 'nls': nls_list, 'type': vehicle_type, 'color': vehicle_color } torch.save(saved_nls, f'cache/{epoch}/{uuid}.pth') # model = model.cpu() del model, saved_dict, n, nls_list, img_ft torch.cuda.empty_cache()
def test(rank, cfg, loader, dataset, epoch, uuids, nls, scene_threshold, total_threshold): dataset.load_frame = False dist.init_process_group(backend="nccl", rank=rank, world_size=cfg.num_gpu, init_method="env://") torch.cuda.set_device(rank) cudnn.benchmark = True model = MyModel(cfg, len(dataset.nl), dataset.nl.word_to_idx['<PAD>'], norm_layer=nn.BatchNorm2d, num_colors=len(CityFlowNLDataset.colors), num_types=len(CityFlowNLDataset.vehicle_type) - 2).cuda() model = DistributedDataParallel(model, device_ids=[rank], output_device=rank, broadcast_buffers=cfg.num_gpu > 1, find_unused_parameters=False) saved_dict = torch.load(f'save/{epoch}.pth', map_location=torch.device(f'cuda:{rank}')) model.load_state_dict(saved_dict, True) model.eval() final_results = {} a = len(nls) // cfg.num_gpu start = a * rank end = None if (rank + 1) == cfg.num_gpu else a * (rank + 1) end_str = 'end' if end == None else end print(f'process number: {rank}, {start}:{end_str}') for nlidx, (uuid, query_nl) in enumerate(zip(uuids[start:end], nls[start:end])): nlidx = nlidx + start print(f'{nlidx} / {len(nls)}') cache_nl = torch.load(f'cache/{epoch}/{uuid}.pth', map_location=torch.device(f'cuda:{rank}')) cache_nl, vehicle_type, vehicle_color = cache_nl['nls'], cache_nl[ 'type'], cache_nl['color'] # nls = [] # for nl in query_nl: # nl = torch.tensor(dataset.nl.sentence_to_index(nl, is_train=False)).cuda() # nls.append(nl.unsqueeze(0).transpose(1, 0)) uuids_per_nl = [] prob_per_nl = [] for idx, (id, frames, boxes, paths, rois, labels) in enumerate(loader): # print(f'{nlidx}_{idx}') with torch.no_grad(): boxes = boxes.squeeze(0).numpy() rois = rois.squeeze(0).numpy() # print(rois) frames = frames.squeeze(0) # print(frames.shape) # b = frames.shape[0] labels = labels.squeeze(0) labels = labels.cuda() cache = torch.load(f'cache/{epoch}/{idx}_0.pth', map_location=torch.device(f'cuda:{rank}')) # print(cache) frame, cs, vs = cache # if vehicle_type != -1 and vs != vehicle_type: # continue # if vehicle_color != -1 and cs != vehicle_color: # continue # print(frame.device) # results = [] nl1 = cache_nl[0] nl2 = cache_nl[1] nl3 = cache_nl[2] bs = frame.shape[0] # cache = cache[:num_of_vehicles] # print(cache.shape) if nl1.shape[0] != bs: nl1 = cache_nl[0].expand(bs, -1, -1).cuda() nl2 = cache_nl[1].expand(bs, -1, -1).cuda() nl3 = cache_nl[2].expand(bs, -1, -1).cuda() am1 = model(nl1, frame, labels) am2 = model(nl2, frame, labels) am3 = model(nl3, frame, labels) # am1, c1, v1 = model(nl1, frame, labels) # am2, c2, v2 = model(nl2, frame, labels) # am3, c3, v3 = model(nl3, frame, labels) activation_aggregation = (am1 + am2 + am3) / 3 # c_aggregation = (c1 + c2 + c3) / 3 # v_aggregation = (v1 + v2 + v3) / 3 # activation_aggregation = model(nl1, frame) +\ # model(nl2, frame) +\ # model(nl3, frame) # activation_aggregation = activation_aggregation / 3 # results.append(activation_aggregation) # cs.append(c_aggregation) # vs.append(v_aggregation) results = activation_aggregation.cpu().numpy() # cs = c_aggregation.mean(dim=0).cpu().numpy() # vs = v_aggregation.mean(dim=0).cpu().numpy() # version 1 # cache = torch.load(f'cache/{epoch}/{idx}.pth') # results = [] # for batch_idx in range(cache.shape[0]): # output = model(cache_nl[0], cache[batch_idx:batch_idx+1]).sigmoid() # results.append(output.squeeze(0).cpu().detach().numpy()) prob = compute_probability_of_activations( results, rois, scene_threshold) # if vehicle_type != -1 and np.argmax(vs) != vehicle_type: # prob = 0. # if vehicle_color != -1 and np.argmax(cs) != vehicle_color: # prob = 0. ###### visualization # if not os.path.exists('results/' + query_nl[0]): # os.mkdir('results/' + query_nl[0]) # # cs = np.argmax(cs) # cs = cs.item() # vs = vs.item() # cs = CityFlowNLDataset.colors[cs] # # vs = np.argmax(vs) # vs = CityFlowNLDataset.vehicle_type[vs] # if prob > total_threshold: # print(f'color: {cs}, type: {vs}') # save_img(np.squeeze(results[0], axis=0) * 255, cv2.imread(paths[0][0]), boxes[0], f"results/{query_nl[0]}/{idx}_{prob}.png") ###### end visualization # for submission uuids_per_nl.append(id[0]) prob_per_nl.append(prob) final_results['uuids_order'] = uuids_per_nl final_results[uuid] = prob_per_nl # uuids_per_nl = np.array(uuids_per_nl) # # print(uuids_per_nl.shape) # prob_per_nl = np.array(prob_per_nl) # prob_per_nl_arg = (-prob_per_nl).argsort(axis=0) # sorted_uuids_per_nl = uuids_per_nl[prob_per_nl_arg] # # print(prob_per_nl[prob_per_nl_arg]) # final_results[uuid] = sorted_uuids_per_nl.tolist() # print(len(final_results.keys())) with open(f'results/submit_{epoch}_{start}_{end_str}.json', 'w') as fp: json.dump(final_results, fp)
class MultiHeadTester: def __init__( self, dataset_path='./drive/MyDrive/datasets/car classification/train_data', batch_size=1, model_name='tf_efficientnet_b3_ns', test_csv='./train_labels.csv', unique_csv='./train_labels.csv', output_dir='../drive/MyDrive/ckpt/grapheme/submission.csv', ckpt='../drive/MyDrive/ckpt/grapheme/20.pth'): # initialize attributes self.dataset_path = dataset_path self.batch_size = batch_size self.model_name = model_name self.test_csv = test_csv self.unique_csv = unique_csv self.output_dir = output_dir self.ckpt = ckpt if model_name == 'tf_efficientnet_b0_ns': self.input_size = (224, 224) elif model_name == 'tf_efficientnet_b3_ns': self.input_size = (300, 300) elif model_name == 'tf_efficientnet_b4_ns': scaleelf.input_size = (380, 380) elif model_name == 'tf_efficientnet_b6_ns': self.input_size = (528, 528) else: raise Exception('non-valid model name') # Compose transforms transform = [] transform += [transforms.Resize(self.input_size)] self.transform = transforms.Compose(transform) self.test_dataset = BengaliDataset(self.test_csv, self.unique_csv, self.dataset_path, self.transform, cache=True) self.names = self.test_dataset.names self.test_dataloader = DataLoader(self.test_dataset, batch_size=self.batch_size, num_workers=0, shuffle=False) self.device = torch.device( 'cuda' if torch.cuda.is_available() else 'cpu') self.model_root = MyModel(self.input_size, self.model_name, 168, pretrained=True, dropout=0).to('cuda') self.model_consonant = MyModel(self.input_size, self.model_name, 11, pretrained=True, dropout=0).to('cuda') self.model_vowel = MyModel(self.input_size, self.model_name, 18, pretrained=True, dropout=0).to('cuda') self.model_multihead = MultiHeadModel(self.input_size, self.model_name, pretrained=True, dropout=0).to('cuda') ckpt = torch.load(self.ckpt) self.model_root.load_state_dict(ckpt['model_root_state_dict']) self.model_consonant.load_state_dict( ckpt['model_consonant_state_dict']) self.model_vowel.load_state_dict(ckpt['model_vowel_state_dict']) self.model_multihead.load_state_dict( ckpt['model_multihead_state_dict']) def test(self): pbar = tqdm.tqdm(self.test_dataloader) pbar.set_description('testing process') self.model_root.eval() self.model_consonant.eval() self.model_vowel.eval() self.model_multihead.eval() output_roots = [] output_consonants = [] output_vowels = [] count = 0 with torch.no_grad(): for it, data in enumerate(pbar): inputs = data[0].to(self.device) inputs = inputs.repeat(1, 3, 1, 1) roots = data[1].to(self.device).long() consonants = data[2].to(self.device).long() vowels = data[3].to(self.device).long() uniques = data[4].to(self.device).long() root_preds, root_preds_2 = self.model_root(inputs, roots) self.model_root.zero_grad() consonant_preds, consonant_preds_2 = self.model_consonant( inputs, consonants) self.model_consonant.zero_grad() vowel_preds, vowel_preds_2 = self.model_vowel(inputs, vowels) self.model_vowel.zero_grad() root, consonant, vowel, unique, root2, consonant2, vowel2, unique2 = self.model_multihead( inputs, roots, consonants, vowels, uniques) self.model_multihead.zero_grad() unique_prob = F.softmax(unique, dim=1) for index in range(inputs.shape[0]): #print('unique:', unique_prob.max(-1).values[index]) if unique_prob.max(-1).values[index] > 0.5: output_roots.append(root.argmax(-1)[index].item()) output_consonants.append( consonant.argmax(-1)[index].item()) output_vowels.append(vowel.argmax(-1)[index].item()) else: output_roots.append( root_preds.argmax(-1)[index].item()) output_consonants.append( consonant_preds.argmax(-1)[index].item()) output_vowels.append( vowel_preds.argmax(-1)[index].item()) row_id, target = [], [] for iid, r, v, c in zip(self.names, output_roots, output_consonants, output_vowels): row_id.append(iid + '_grapheme_root') target.append(int(r)) row_id.append(iid + '_vowel_diacritic') target.append(int(v)) row_id.append(iid + '_consonant_diacritic') target.append(int(c)) count += 1 sub_fn = self.output_dir sub = pd.DataFrame({'row_id': row_id, 'target': target}) sub.to_csv(sub_fn, index=False) print(f'Done wrote to {sub_fn}')
# udpate tensorboardX writer.add_scalar('train_loss', n_iter) # compute computation time and *compute_efficiency* process_time = start_time - time.time() - prepare_time compute_efficiency = process_time / (process_time + prepare_time) pbar.set_description( f'Compute efficiency: {compute_efficiency:.2f}, ' f'loss: {loss.item():.2f}, epoch: {epoch}/{opt.epochs}') start_time = time.time() # maybe do a test pass every N=1 epochs if epoch % 1 == 0: # bring models to evaluation mode net.eval() correct = 0 total = 0 pbar = tqdm(enumerate(test_data_loader), total=len(test_data_loader)) with torch.no_grad(): for i, data in pbar: # data preparation img, label = data if use_cuda: img = img.cuda() label = label.cuda() out = net(img)
def train(num_epochs, batch_size): current_time = time.strftime('%Y_%m_%d_%H_%M', time.localtime(time.time())) logger = utils.Logger(os.path.join('out', 'log_' + current_time + '.txt')) # logger.write(f'{current_time}logger_batch_size{batch_size}') vocab_size = ge.vocab_size answer_size = ge.answer_size train_set = GQA(root='data', split='train', transform=transform) val_set = GQA(root='data', split='val', transform=transform) train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, collate_fn=collate_data, drop_last=True) val_loader = DataLoader(val_set, batch_size=batch_size, shuffle=True, collate_fn=collate_data, drop_last=True) # add a tqdm bar d = iter(train_loader) pbar = tqdm(d) net = MyModel(out_features=answer_size).to(device) optimizer = torch.optim.Adam(net.parameters(), lr=1e-4) loss = nn.CrossEntropyLoss() def _validation(val_loader): acc_sum, n = 0.0, 0 for img, q, loq, a in val_loader: img, q, a = ( img.to(device), q.to(device), a.to(device), ) a_hat = net(img, q) acc_sum += (a_hat.detach().argmax(dim=1) == a).sum().item() n += a.shape[0] if n <= 0: print('validation loader does not work') return 0.0 return acc_sum / n for i in range(num_epochs): loss_sum = 0.0 acc_sum = 0.0 n = 0 start = time.time() for img, q, loq, a in train_loader: img, q, a = ( img.to(device), q.to(device), a.to(device), ) net.train() net.zero_grad() a_hat = net(img, q) l = loss(a_hat, a) l.backward() optimizer.step() loss_sum += l.item() acc_sum += (a_hat.detach().argmax(dim=1) == a).sum().item() n += a.shape[0] if n < 100: print('time for one batch:', time.time() - start) net.eval() with torch.no_grad(): vali_acc = _validation(val_loader) logger.write( 'Epoch #%d, Loss:%.3f, Train Acc: %.4f, Validation Acc:%.4f' % (i, loss_sum, acc_sum / n, vali_acc)) print('Epoch #%d, Loss:%.3f, Train Acc: %.4f, Validation Acc:%.4f' % (i, loss_sum, acc_sum / n, vali_acc)) try: torch.save(net.state_dict(), 'checkpoint/checkpoint_{}.pth'.format(i)) except: print('can not save checkpoint.') torch.save(net.state_dict(), 'checkpoint/model_2.pth')
TEST_INTERVAL = 200 PRINT_INTERVAL = 50 LEARNING_RATE = 1e-5 if USE_MEMORY: LEARNING_RATE = (4e-4 if DOUBLE_Q else 3e-3) RENDER_INTERVAL = 20 ENV_NAME = 'CartPole-v0' env = gym.make(ENV_NAME) state_shape = len(env.reset()) n_actions = env.action_space.n model = MyModel(state_shape, n_actions).to(device) target = MyModel(state_shape, n_actions).to(device) target.load_state_dict(model.state_dict()) target.eval() optimizer = torch.optim.SGD(model.parameters(), lr=LEARNING_RATE) loss_function = torch.nn.MSELoss(reduction='mean') memory = ReplayBuffer() def choose_action(state, test_mode=False): if random.random() < EPS_EXPLORATION: return env.action_space.sample() return model.select_action(torch.from_numpy(state)).item() # Given a tuple (s_t, a_t, s_{t+1}, r_t, done_t) update your model weights def optimize_model(state, action, next_state, reward, done): # For double Q, use the other Q function for getting values of the next state. nextStateQ = target if DOUBLE_Q else model