for word in text: if random.random() < p[word]: subsampling.append(word) vocab_count = dict(Counter(subsampling).most_common(MAX_VOCAB_SIZE - 1)) vocab_count['<UNK>'] = 1 idx2word = [word for word in vocab_count.keys()] word2idx = {word: i for i, word in enumerate(idx2word)} nc = np.array([count for count in vocab_count.values()], dtype=np.float32)**(3. / 4.) word_freqs = nc / np.sum(nc) dataset = WordEmbeddingDataset(subsampling, word2idx, word_freqs) dataloader = tud.DataLoader(dataset, BATCH_SIZE, shuffle=True) model = EmbeddingModel(len(idx2word), EMBEDDING_SIZE) model.to(device) model.train() optimizer = optim.Adam(model.parameters(), lr=LR) for epoch in range(EPOCHS): pbar = tqdm(dataloader) pbar.set_description("[Epoch {}]".format(epoch)) for i, (input_labels, pos_labels, neg_labels) in enumerate(pbar): input_labels = input_labels.to(device) pos_labels = pos_labels.to(device) neg_labels = neg_labels.to(device) model.zero_grad() loss = model(input_labels, pos_labels, neg_labels).mean()
round((real_size-crop_size)/2):round((real_size+crop_size)/2),round((real_size-crop_size)/2):round((real_size+crop_size)/2)]*\ precandidate['seg'][round((real_size-crop_size)/2):round((real_size+crop_size)/2),\ round((real_size-crop_size)/2):round((real_size+crop_size)/2),round((real_size-crop_size)/2):round((real_size+crop_size)/2)] train_data = torch.from_numpy(train_data) val_data = torch.from_numpy(val_data) column_train_label = list(map(int, column_train_label)) train_label = torch.Tensor(train_label) val_label = torch.Tensor(val_label) BATCH_SIZE = 32 torch_dataset_train = data.TensorDataset(train_data, train_label) torch_dataset_val = data.TensorDataset(val_data, val_label) train_loader = data.DataLoader(dataset=torch_dataset_train, batch_size=BATCH_SIZE, shuffle=True) val_loader = data.DataLoader(dataset=torch_dataset_val, batch_size=BATCH_SIZE, shuffle=False) test_loader = data.DataLoader(test_data, batch_size=test_num, shuffle=False) def mixup_data(x, y, alpha, device): '''Returns mixed inputs, pairs of targets, and lambda''' if alpha > 0: lam = np.random.beta(alpha, alpha) else: lam = 1 batch_size = x.size()[0]
def train(): rpf.train() lr = args.lr epoch = disp_loss = 0 eval_loss = 10000. start_time = time.time() epoch_size = len(train_dataset) // args.batch_size max_epoch = int(args.max_iter / epoch_size) step_values = [10000, 50000, 100000] step_index = 0 batch_iterator = data.DataLoader(dataset=train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers) for iteration in range(args.max_iter): if iteration % epoch_size == 0: if epoch != 0: print("Saving state, epoch:", epoch) torch.save(rpf.state_dict(), save_dir + '/epoch_{}.pth'.format(epoch)) batch_iterator = iter( data.DataLoader(dataset=train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers)) epoch += 1 demo_im, demo_action_onehot, follower_im, follower_action = next( batch_iterator) if args.cuda: demo_im = Variable(demo_im.cuda(async=True)) demo_action_onehot = Variable(demo_action_onehot.cuda(async=True)) follower_im = Variable(follower_im.cuda(async=True)) with torch.no_grad(): follower_action = Variable( follower_action.cuda(async=True).long()) record, done = rpf( demo_im, demo_action_onehot, follower_im=follower_im ) #Record: 'eta', 'h_ts', 'attention_t', 'in_rgb_feats_t', 'mu_t', 'action_pred_t' optimizer.zero_grad() loss = criterion(record['action_pred_t'].view(-1, args.action_dim), follower_action.view(-1)) loss.backward() optimizer.step() disp_loss += loss.item() end_time = time.time() if iteration % args.print_step == 0 and iteration > 0: disp_loss = disp_loss / args.print_step print( 'Epoch [%d/%d] Iter [%d/%d] Loss: %.6f, lr: %.6f, Iter time: %.5fs' % (epoch, max_epoch, iteration, args.max_iter, disp_loss, lr, (end_time - start_time))) writer.add_scalar('train_loss', disp_loss, iteration) disp_loss = 0. start_time = time.time() if iteration in step_values: step_index += 1 lr = adjust_learning_rate(optimizer, step_index, args.lr_decay) if iteration % args.test_step == 0: # and iteration > 1: rpf.eval() tot_start_time = time.time() disp_loss = total_disp_loss = 0. print("Evaluating RPF networks...") valid_iterator = iter( data.DataLoader(dataset=valid_dataset, batch_size=args.batch_size, num_workers=args.num_workers)) with torch.no_grad(): for valid_iteration in range(valid_num_batches): demo_im, demo_action_onehot, follower_im, follower_action = next( valid_iterator) if args.cuda: with torch.no_grad(): demo_im = Variable(demo_im.cuda(async=True)) demo_action_onehot = Variable( demo_action_onehot.cuda(async=True)) follower_im = Variable( follower_im.cuda(async=True)) follower_action = Variable( follower_action.cuda(async=True).long()) record, done = rpf(demo_im, demo_action_onehot, follower_im=follower_im) loss = criterion( record['action_pred_t'].view(-1, args.action_dim), follower_action.view(-1)) disp_loss += loss.item() total_disp_loss += loss.item() if valid_iteration % args.print_step == 0: disp_loss = disp_loss / args.print_step end_time = time.time() print('Iter [%d/%d] Eval loss: %.6f, Time: %.5fs' % (valid_iteration, valid_num_batches, disp_loss, (end_time - start_time))) disp_loss = 0. start_time = time.time() total_disp_loss = total_disp_loss / (valid_num_batches - 1) tot_end_time = time.time() print('[Epoch : %d] Loss : %.6f, Eval_time : %.5fs' % (epoch, total_disp_loss, (tot_end_time - tot_start_time))) writer.add_scalar('valid_loss', total_disp_loss, epoch) if eval_loss > total_disp_loss: eval_loss = total_disp_loss torch.save(rpf.state_dict(), save_dir + '/best.pth') print('Updated the best model') rpf.train()
]) transform_test = transforms.Compose([ transforms.Resize(32), transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )), ]) dataloader = datasets.MNIST trainset = dataloader(root='./data', train=True, download=True, transform=transform_train) trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testset = dataloader(root='./data', train=False, download=True, transform=transform_test) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # In[11]: # Model print("==> creating model ")
# (doc[0], indx_tokens, output_film_size1, output_film_size2, output_film_size3, output_film_size4, attention_mask, spmed, (n,doc,Entdic, Reldic)) n_Entdics_Reldics = [a[0] for a in corpus] word_input = torch.LongTensor([a[1] for a in corpus]).to(device) attention_mask = torch.LongTensor([a[2] for a in corpus]).to(device) sentencepieced = [a[3] for a in corpus] doc_correspnd_info_dict = {} #document毎にシンボリックな値をdocument名と辞書に変えるための辞書 n_doc = [] for unit in n_Entdics_Reldics: doc_correspnd_info_dict[unit[0]] = [(unit[1],unit[1][:-4]+'.ann'),{},{}] n_doc.append([unit[0]]) n_doc = torch.LongTensor(n_doc).to(device) test_dataset = D.TensorDataset(n_doc, word_input, attention_mask) test_loader = D.DataLoader(test_dataset , batch_size=int(config.get('main', 'BATCH_SIZE_TEST' )), shuffle = strtobool(config.get('main', 'BATCH_SHUFFLE_TEST'))) print('finish', end='\n') # pdb.set_trace() print('Create Model...') tokenizer = BERT_PRETRAINED_MODEL_JAPANESE(config, vocab).return_tokenizer() model = SPAN_CNN(config, vocab, REL_DIC).to(device) # relation_model = BRAN(config, vocab, REL_DIC).to(device) relation_model = RELATION(config, vocab, REL_DIC).to(device) model.load_state_dict(torch.load(NER_model_save_path, map_location=device)) relation_model.load_state_dict(torch.load(RE_model_save_path, map_location=device)) # pretrained_BERT_model = bert_pretrained_model_japanese.return_model() # tokenizer.convert_ids_to_tokens()というメソッドでindexを用語に直せる
import torch.utils.data as utils # a list of numpy arrays labels_nonmitosis = torch.ones( len(path2)) # another list of numpy arrays (targets) labels_mitosis = torch.zeros(len(path3)) train_labels = torch.cat((labels_nonmitosis, labels_mitosis)) y1_tensor = torch.tensor(train_labels, dtype=torch.long) tensor_x1 = torch.stack([transform1(i) for i in lbp_train]) # transform to torch tensors tensor_x4 = torch.stack([TRANSFORM_IMG(i) for i in tensor_x1]) train_data = utils.TensorDataset(tensor_x4, y1_tensor) train_data_loader = utils.DataLoader(train_data, batch_size=BATCH_SIZE, shuffle=True, num_workers=4) # create your dataloader labels_nonmitosis_val = torch.ones( len(path4)) # another list of numpy arrays (targets) labels_mitosis_val = torch.zeros(len(path5)) val_labels = torch.cat((labels_nonmitosis_val, labels_mitosis_val)) y2_tensor = torch.tensor(val_labels, dtype=torch.long) tensor_x2 = torch.stack([transform1(i) for i in lbp_val]) # transform to torch tensors tensor_x3 = torch.stack([TRANSFORM_IMG(i) for i in tensor_x2]) test_data = utils.TensorDataset(tensor_x3, y2_tensor) # create your datset test_data_loader = utils.DataLoader(test_data, batch_size=BATCH_SIZE, shuffle=True,
def objective(SCI_RELU, SCI_BIAS, SCI_loss_type, SCI_optimizer, SCI_LR, SCI_MM, SCI_REGULARIZATION, SCI_EPOCHS, SCI_BATCH_SIZE, SCI_DROPOUT, SCI_L_SECOND, SCI_BN_MOMENTUM, SCI_SGD_MOMENTUM, SCI_BN_EPS, SCI_BN_STATS, SCI_LAST_LAYER, SCI_ACT_LAYER): global count, PercentVector, PercentVec, device, MaxCredit SCI_BATCH_SIZE = int(SCI_BATCH_SIZE) SCI_LAST_LAYER = int(SCI_LAST_LAYER) SCI_ACT_LAYER =int(SCI_ACT_LAYER) SCI_MM = round(SCI_MM,3) # real with three decimals between (0.001, 0.999) SCI_LR = round(SCI_LR,5) # real with five decimals between(1e-4, 7e-1) SCI_DROPOUT = round(SCI_DROPOUT,2) # real with two decimals between (0, 0.4) SCI_L_SECOND = int(SCI_L_SECOND) # integer between 2 and 64 SCI_EPOCHS = int(SCI_EPOCHS) # integer between (100, 500) SCI_BN_MOMENTUM = round(SCI_BN_MOMENTUM,2) # real with two decimals between (0, 0.99) SCI_SGD_MOMENTUM = round(SCI_SGD_MOMENTUM,2) # real with two decimals between (0, 0.99) SCI_loss_type = int(SCI_loss_type) # integer between 1 and 3 ('CrossEntropyLoss', 'MultiMarginLoss','NLLLoss') SCI_BN_EPS = int(SCI_BN_EPS) if int(SCI_RELU) == 1 : # integer between 1 and 2 ('True', 'False') SCI_RELU = True else: SCI_RELU = False if int(SCI_BIAS) == 1 : # integer between 1 and 2 ('True', 'False') SCI_BIAS = True else: SCI_BIAS = False SCI_REGULARIZATION = float(str(SCI_REGULARIZATION)) if SCI_BN_EPS == 0: BN_EPS = 5e-6 if SCI_BN_EPS == 1: BN_EPS = 1e-5 if SCI_BN_EPS == 2: BN_EPS = 5e-6 if SCI_BN_EPS == 3: BN_EPS = 1e-6 if SCI_BN_EPS == 4: BN_EPS = 5e-7 if SCI_BN_EPS == 5: BN_EPS = 1e-7 if SCI_BN_EPS == 6: BN_EPS = 5e-8 if SCI_BN_EPS == 7: BN_EPS = 1e-8 if SCI_BN_EPS == 8: BN_EPS = 3e-7 if SCI_BN_EPS == 9: BN_EPS = 8e-7 if SCI_BN_EPS == 10: BN_EPS = 1e-4 if SCI_BN_EPS == 11: BN_EPS = 5e-4 if SCI_BN_EPS == 12: BN_EPS = 8e-6 if SCI_BN_EPS == 13: BN_EPS = 1e-6 if SCI_BN_EPS == 14: BN_EPS = 8e-5 print('BN Batch EPS: ', BN_EPS) SCI_BN_STATS = int(SCI_BN_STATS) if SCI_BN_STATS == 0: BN_STATS = True if SCI_BN_STATS == 1: BN_STATS = False print('BN Batch STATS: ', BN_STATS) cnn = CNN6(L_FIRST, SCI_L_SECOND, KERNEL_X, SCI_BIAS, SCI_BN_MOMENTUM, SCI_RELU, SCI_DROPOUT, dataset.CLASSES, BN_EPS, BN_STATS, SCI_LAST_LAYER, SCI_ACT_LAYER) optimizer = Utillities.optimization_algorithms(SCI_optimizer,cnn, SCI_LR, SCI_SGD_MOMENTUM, SCI_REGULARIZATION) if GPU_SELECT == 2: if torch.cuda.device_count() > 1: cnn = nn.DataParallel(cnn,device_ids=[0, 1], dim = 0) cnn = cnn.cuda() if GPU_SELECT == 1: cnn.to(device) if GPU_SELECT == 0: cnn.to(device) cnn.apply(CNN6.weights_init2) #cnn.apply(CNN6.weights_reset) cnn.share_memory() loss_func = nn.CrossEntropyLoss() def create_loss(LOSS): print('*** LOSS ******:', LOSS) if LOSS == 1: loss_func = nn.BCELoss() print('********* BCELoss') if LOSS == 2: loss_func = nn.MultiMarginLoss() print('********* MMLoss') if LOSS == 4: loss_func = nn.CrossEntropyLoss() print('********* CrossEntropyLoss ') if LOSS == 3: loss_func = nn.TripletMarginLoss() print('********* TripletMarginLoss ') return loss_func MM = float(str(SCI_MM)) LR = float(str(SCI_LR)) train_losses = [] # to track the training loss as the model trains output = 0 loss = 0 accuracy = 0 early_stopping.counter = 0 early_stopping.best_score = None early_stopping.early_stop = False early_stopping.verbose = False TEST_RESULTS = torch.zeros(1,2) loss_type = create_loss(SCI_loss_type) #cnn, optimizer = amp.initialize( # cnn, optimizer, opt_level=BITS, # keep_batchnorm_fp32=True, loss_scale="dynamic" #) Utillities.listing(optimizer, SCI_SGD_MOMENTUM, SCI_BN_MOMENTUM, SCI_L_SECOND, SCI_LR, SCI_RELU, SCI_BIAS, SCI_loss_type, SCI_REGULARIZATION, SCI_BATCH_SIZE, SCI_DROPOUT, SCI_LAST_LAYER, SCI_ACT_LAYER) # Data Loader for easy mini-batch return in training SCI_BATCH_SIZE = int(SCI_BATCH_SIZE) train_loader = Data.DataLoader(dataset = dataset.train_dataset, batch_size = SCI_BATCH_SIZE, shuffle = True, num_workers = 0, drop_last=True, pin_memory=True) validation_loader = Data.DataLoader(dataset = dataset.validation_dataset, batch_size = 30, shuffle = True, num_workers = 0, drop_last=True, pin_memory=True) test_loader = Data.DataLoader(dataset = dataset.test_dataset, batch_size = 300, shuffle = True, num_workers = 0, drop_last=True, pin_memory=True) flag = True; for epoch in range(SCI_EPOCHS): loss = None cnn.train().cuda() for step, (train_data, train_target) in enumerate(train_loader): train_data, train_target = train_data.to(device), train_target.to(device) output = cnn(train_data) # forward pass: compute predicted outputs by passing inputs to the model loss = loss_func(output, train_target) train_losses.append(loss.item()) #batch_loss.backward() loss.backward() # backward pass: compute gradient of the loss with respect to model parameters optimizer.zero_grad() optimizer.step() # perform a single optimization step (parameter update) cnn.eval().cuda() # switch to evaluation (no change) mode valid_loss = 0 accuracy = 0 running_loss = 0.0 with torch.no_grad(): for step, (validation_data, validation_target) in enumerate(validation_loader): validation_data, validation_target = validation_data.to(device), validation_target.to(device) output = cnn(validation_data) valid_loss = loss_func(output, validation_target) running_loss += valid_loss.item() epoch_val_loss = running_loss / len(validation_loader) if epoch % 3 == 0: SCI_LR, flag = Utillities.variable_learning_rate(SCI_LR, LR_MIN, LR_MAX, 2, flag) SCI_DROPOUT = SCI_DROPOUT / 1.02 early_stopping(epoch_val_loss, cnn) #print('validation loss:',epoch_val_loss) train_losses = [] if early_stopping.early_stop: if os.path.exists('checkpoint.pt'): #cnn = TheModelClass(*args, **kwargs) print("Loaded the model with the lowest Validation Loss!") cnn.load_state_dict(torch.load('checkpoint.pt')) # Choose whatever GPU device number you want cnn.to(device) break running_loss = 0.0 cnn.eval() class_correct = list(0. for i in range(1000)) class_total = list(0. for i in range(1000)) with torch.no_grad(): for (test_data, test_target) in test_loader: test_data, test_target = test_data.to(device), test_target.to(device) outputs = cnn(test_data) _, predicted = torch.max(outputs, 1) c = (predicted == test_target).squeeze() dx = ((c.cpu()).numpy()).astype(int) #dx = 600 for i in range(test_target.size(0)): label = test_target[i] class_correct[label] += c[i].item() class_total[label] += 1 for i in range(dataset.CLASSES): TEST_RESULTS[0,i] = class_correct[i] / dataset.TESTED_ELEMENTS[i] print('Class: ',i,' accuracy: ', TEST_RESULTS[0,i]) print('Class: ',i,' correct: ', class_correct[i],' of ',dataset.TESTED_ELEMENTS[i]) #mp.matshow(dx.reshape((20, 30))) #mp.ylabel('Correct Results') #mp.colorbar() #mp.show() percent = (TEST_RESULTS[0,0]+TEST_RESULTS[0,1])/2 print('Final percentage: ',percent) CreditCost = 0 CreditCost = int((1 - TEST_RESULTS[0,0]) * dataset.TESTED_ELEMENTS[0] + (1 - TEST_RESULTS[0,1]) * dataset.TESTED_ELEMENTS[1] * 5) #if TEST_RESULTS[0,0] < 0.05 or TEST_RESULTS[0,1] < 0.05 : # CreditCost = CreditCost + 300 print('Last epoch: ', epoch) if os.path.exists('checkpoint.pt'): os.remove('checkpoint.pt') #print('Credit Cost: ',CreditCost) #CreditCost = CreditCost + (SCI_SGD_MOMENTUM + SCI_DROPOUT + SCI_BATCH_SIZE + SCI_L_SECOND + SCI_optimizer + SCI_loss_type+ SCI_LR+ SCI_BN_EPS+SCI_BN_STATS+SCI_LAST_LAYER+SCI_ACT_LAYER)/10000 print('Credit Cost: ',CreditCost) if -CreditCost > MaxCredit : MaxCredit = -CreditCost print('Best Score So Far: ',MaxCredit) print() print() #CreditVector[count] = MaxCredit #CreditVec[count] = count # plot the data #fig = mp.figure() #ax = fig.add_subplot(1, 1, 1) #ax.plot(CreditVec, -CreditVector, color='tab:orange') #print(CreditVec, -CreditVector) #count = count + 1 # display the plot #mp.show() # return function (with unknown internals) we wish to maximize. return -CreditCost
def main(dataset='opencv_video_seq_dataset.VideoSeqDataset', np_transforms=None, tensor_transforms=( 'img_landmarks_transforms.ToTensor()', 'transforms.Normalize(mean=[0.5,0.5,0.5],std=[0.5,0.5,0.5])'), workers=4, batch_size=4): import time from fsgan.utils.obj_factory import obj_factory from fsgan.utils.img_utils import tensor2bgr np_transforms = obj_factory( np_transforms) if np_transforms is not None else [] tensor_transforms = obj_factory( tensor_transforms) if tensor_transforms is not None else [] img_transforms = img_landmarks_transforms.Compose(np_transforms + tensor_transforms) dataset = obj_factory(dataset, transform=img_transforms) # dataset = VideoSeqDataset(root_path, img_list_path, transform=img_transforms, frame_window=frame_window) dataloader = data.DataLoader(dataset, batch_size=batch_size, num_workers=workers, pin_memory=True, drop_last=True, shuffle=True) start = time.time() for frame_window, landmarks_window in dataloader: # print(frame_window.shape) if isinstance(frame_window, (list, tuple)): # For each batch for b in range(frame_window[0].shape[0]): # For each frame window in the list for p in range(len(frame_window)): # For each frame in the window for f in range(frame_window[p].shape[2]): print(frame_window[p][b, :, f, :, :].shape) # Render render_img = tensor2bgr( frame_window[p][b, :, f, :, :]).copy() landmarks = landmarks_window[p][b, f, :, :].numpy() # for point in np.round(landmarks).astype(int): for point in landmarks: cv2.circle(render_img, (point[0], point[1]), 2, (0, 0, 255), -1) cv2.imshow('render_img', render_img) if cv2.waitKey(0) & 0xFF == ord('q'): break else: # For each batch for b in range(frame_window.shape[0]): # For each frame in the window for f in range(frame_window.shape[2]): print(frame_window[b, :, f, :, :].shape) # Render render_img = tensor2bgr(frame_window[b, :, f, :, :]).copy() landmarks = landmarks_window[b, f, :, :].numpy() # for point in np.round(landmarks).astype(int): for point in landmarks: cv2.circle(render_img, (point[0], point[1]), 2, (0, 0, 255), -1) cv2.imshow('render_img', render_img) if cv2.waitKey(0) & 0xFF == ord('q'): break end = time.time() print('elapsed time: %f[s]' % (end - start))
parser.add_argument('--epochs', type=int, default=10, metavar='N',help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.01, metavar='LR',help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.5, metavar='M',help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False,help='disables CUDA training') parser.add_argument('--seed', type=int, default=1, metavar='S',help='random seed,default=1)') parser.add_argument('--eps', type=float, default=1e-5, metavar='LR',help='learning rate,default=1e-5') parser.add_argument('--log-interval', type=int, default=100, metavar='N',help='for printing training data is log interval') args = parser.parse_args() args.cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) if args.cuda: torch.cuda.manual_seed(args.seed) #train_loader train_loader = D.DataLoader(datasets.MNIST('./data', train=True, download=True, transform=transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])),batch_size=args.batch_size, shuffle=True) #test_loaer test_loader = D.DataLoader(datasets.MNIST('./data', train=False, transform=transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])),batch_size=args.test_batch_size, shuffle=True) ################################################################ #MODEL class Model(nn.Module): def __init__(self): super(Model, self).__init__() self.layer1 = nn.Sequential( BinaryConv2d(1, 16, kernel_size=5, padding=2),
def train(): if cfg.dataset is None: print("Missing dataset in config!") exit(-1) save_folder = Path(args.save_folder) save_folder.mkdir(exist_ok=True, parents=True) epoch_status_file_path = Path(args.epoch_status_file) dataset = COCODetection( image_path=cfg.dataset.train_images, info_file=cfg.dataset.train_info, transform=SSDAugmentation(cfg, MEANS), label_map=cfg.dataset.get_valid_label_map(), ) if args.validation_epoch > 0: setup_eval() val_dataset = COCODetection( image_path=cfg.dataset.valid_images, info_file=cfg.dataset.valid_info, transform=BaseTransform(cfg, MEANS), label_map=cfg.dataset.get_valid_label_map(), ) # Parallel wraps the underlying module, but when saving and loading we don't want that yolact_net = Yolact(cfg) net = yolact_net net.train() if args.log: log = Log( cfg.name, args.log_folder, dict(args._get_kwargs()), overwrite=(args.resume is None), log_gpu_stats=args.log_gpu, ) # I don't use the timer during training (I use a different timing method). # Apparently there's a race condition with multiple GPUs, so disable it just to be safe. timer.disable_all() # Both of these can set args.resume to None, so do them before the check if args.resume == "interrupt": args.resume = SavePath.get_interrupt(args.save_folder) elif args.resume == "latest": args.resume = SavePath.get_latest(args.save_folder, cfg.name) if args.resume is not None: print("Resuming training, loading {}...".format(args.resume)) yolact_net.load_weights(args.resume) if args.start_iter == -1: args.start_iter = SavePath.from_str(args.resume).iteration else: print("Initializing weights...") yolact_net.init_weights(backbone_path=cfg.backbone.path) optimizer = optim.SGD( net.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.decay ) criterion = MultiBoxLoss( num_classes=cfg.num_classes, pos_threshold=cfg.positive_iou_threshold, neg_threshold=cfg.negative_iou_threshold, negpos_ratio=cfg.ohem_negpos_ratio, cfg=cfg, ) if args.batch_alloc is not None: args.batch_alloc = [int(x) for x in args.batch_alloc.split(",")] if sum(args.batch_alloc) != args.batch_size: print( "Error: Batch allocation (%s) does not sum to batch size (%s)." % (args.batch_alloc, args.batch_size) ) exit(-1) net = CustomDataParallel(NetLoss(net, criterion)) if args.cuda: net = net.cuda() # Initialize everything if not cfg.freeze_bn: yolact_net.freeze_bn() # Freeze bn so we don't kill our means yolact_net(torch.zeros(1, 3, cfg.max_size, cfg.max_size).cuda()) if not cfg.freeze_bn: yolact_net.freeze_bn(True) # loss counters loc_loss = 0 conf_loss = 0 iteration = max(args.start_iter, 0) last_time = time.time() epoch_size = math.ceil(len(dataset) / args.batch_size) print(f"\n\t ==> Number of iterations per epoch: {epoch_size}") num_epochs = min(math.ceil(cfg.max_iter / epoch_size), cfg.max_num_epochs) print(f"\t ==> Number of epochs: {num_epochs}\n") # Which learning rate adjustment step are we on? lr' = lr * gamma ^ step_index step_index = 0 data_loader = data.DataLoader( dataset, args.batch_size, num_workers=args.num_workers, shuffle=True, collate_fn=detection_collate, pin_memory=True, ) save_path = lambda epoch, iteration: SavePath(cfg.name, epoch, iteration).get_path( root=args.save_folder ) time_avg = MovingAverage() global loss_types # Forms the print order loss_avgs = {k: MovingAverage(100) for k in loss_types} print("Begin training!") print() # try-except so you can use ctrl+c to save early and stop training try: for epoch in range(num_epochs): with epoch_status_file_path.open( "w", encoding="utf-8" ) as epoch_status_file: json.dump({"cur_epoch": epoch}, epoch_status_file) # Resume from start_iter if (epoch + 1) * epoch_size < iteration: continue for datum in data_loader: # Stop if we've reached an epoch if we're resuming from start_iter if iteration == (epoch + 1) * epoch_size: break # Stop at the configured number of iterations even if mid-epoch if iteration == cfg.max_iter: break # Change a config setting if we've reached the specified iteration changed = False for change in cfg.delayed_settings: if iteration >= change[0]: changed = True cfg.replace(change[1]) # Reset the loss averages because things might have changed for avg in loss_avgs: avg.reset() # If a config setting was changed, remove it from the list so we don't keep checking if changed: cfg.delayed_settings = [ x for x in cfg.delayed_settings if x[0] > iteration ] # Warm up by linearly interpolating the learning rate from some smaller value if cfg.lr_warmup_until > 0 and iteration <= cfg.lr_warmup_until: set_lr( optimizer, (args.lr - cfg.lr_warmup_init) * (iteration / cfg.lr_warmup_until) + cfg.lr_warmup_init, ) # Adjust the learning rate at the given iterations, but also if we resume from past that iteration while ( step_index < len(cfg.lr_steps) and iteration >= cfg.lr_steps[step_index] ): step_index += 1 set_lr(optimizer, args.lr * (args.gamma ** step_index)) # Zero the grad to get ready to compute gradients optimizer.zero_grad() # Forward Pass + Compute loss at the same time (see CustomDataParallel and NetLoss) losses = net(datum) losses = { k: (v).mean() for k, v in losses.items() } # Mean here because Dataparallel loss = sum([losses[k] for k in losses]) # no_inf_mean removes some components from the loss, so make sure to backward through all of it # all_loss = sum([v.mean() for v in losses.values()]) # Backprop loss.backward() # Do this to free up vram even if loss is not finite if torch.isfinite(loss).item(): optimizer.step() # Add the loss to the moving average for bookkeeping for k in losses: loss_avgs[k].add(losses[k].item()) cur_time = time.time() elapsed = cur_time - last_time last_time = cur_time # Exclude graph setup from the timing information if iteration != args.start_iter: time_avg.add(elapsed) if iteration % 10 == 0: eta_str = str( datetime.timedelta( seconds=(cfg.max_iter - iteration) * time_avg.get_avg() ) ).split(".")[0] total = sum([loss_avgs[k].get_avg() for k in losses]) loss_labels = sum( [ [k, loss_avgs[k].get_avg()] for k in loss_types if k in losses ], [], ) print( ( "[%3d] %7d ||" + (" %s: %.3f |" * len(losses)) + " T: %.3f || ETA: %s || timer: %.3f" ) % tuple( [epoch, iteration] + loss_labels + [total, eta_str, elapsed] ), flush=True, ) if args.log: precision = 5 loss_info = {k: round(losses[k].item(), precision) for k in losses} loss_info["T"] = round(loss.item(), precision) if args.log_gpu: log.log_gpu_stats = iteration % 10 == 0 # nvidia-smi is sloooow log.log( "train", loss=loss_info, epoch=epoch, iter=iteration, lr=round(cur_lr, 10), elapsed=elapsed, ) log.log_gpu_stats = args.log_gpu iteration += 1 if iteration % args.save_interval == 0 and iteration != args.start_iter: if args.keep_latest: latest = SavePath.get_latest(args.save_folder, cfg.name) print("Saving state, iter:", iteration) yolact_net.save_weights(save_path(epoch, iteration)) if args.keep_latest and latest is not None: if ( args.keep_latest_interval <= 0 or iteration % args.keep_latest_interval != args.save_interval ): print("Deleting old save...") os.remove(latest) # This is done per epoch if args.validation_epoch > 0: if epoch % args.validation_epoch == 0 and epoch > 0: compute_validation_map( epoch, iteration, yolact_net, val_dataset, log if args.log else None, ) # Compute validation mAP after training is finished compute_validation_map( epoch, iteration, yolact_net, val_dataset, log if args.log else None ) except KeyboardInterrupt: if args.interrupt: print("Stopping early. Saving network...") # Delete previous copy of the interrupted network so we don't spam the weights folder SavePath.remove_interrupt(args.save_folder) # Wait for all torch processes to finish their task time.sleep(1) yolact_net.save_weights(save_path(epoch, repr(iteration) + "_interrupt")) exit() print("Saving weights...") yolact_net.save_weights(save_path(epoch, repr(iteration) + "_end"))
self.fc = nn.Sequential(nn.Linear(3 * 3 * 128, 256), nn.ReLU(), nn.BatchNorm1d(256), nn.Linear(256, 7)) def forward(self, x): #image size (48,48) x = self.conv1(x) #(24,24) x = self.conv2(x) #(12,12) x = self.conv3(x) #(6,6) x = self.conv4(x) #(3,3) x = x.view(-1, 3 * 3 * 128) x = self.fc(x) return x if __name__ == '__main__': model = Net() model.load_state_dict(torch.load('model_70.pth')) model.eval() transform = transforms.Compose([transforms.ToTensor()]) with torch.no_grad(): test_image = sorted(glob.glob(os.path.join(sys.argv[1], '*.jpg'))) test_set = test_Dataset(test_image, transform) test_loader = Data.DataLoader(test_set, batch_size=128, shuffle=False) test_preds = get_all_preds(model, test_loader) predict = torch.max(test_preds, 1)[1] end = pd.DataFrame({'id': range(0, len(test_image)), 'label': predict}) end.to_csv(sys.argv[2], index=False)
p = (weight.t() / torch.sum(weight, dim=1)).t() log_q = torch.log(q) loss = F.kl_div(log_q, p) return loss, p loss, p = loss_func(feat_init) train_data = torchvision.datasets.MNIST( '../mnist', train=True, transform=torchvision.transforms.ToTensor(), download=False) train_loader = Data.DataLoader(dataset=train_data, batch_size=256, shuffle=True, num_workers=2) optimizer = torch.optim.SGD(list(vae.encoder.parameters()) + list(vae.fc1.parameters()) + [cluster_centers], lr=2.0) def dist_2_label(q_t): _, label = torch.max(q_t, dim=1) return label.data.cpu().numpy() for epoch in range(20): for step, (batch_x, batch_y) in enumerate(train_loader): batch_x = Variable(batch_x.view(-1, 784)).cuda() batch_feat, _ = vae.encode(batch_x)
import torch import torch.utils.data as Data torch.manual_seed(1) # reproducible BATCH_SIZE = 5 # BATCH_SIZE = 8 x = torch.linspace(1, 10, 10) # this is x data (torch tensor) y = torch.linspace(10, 1, 10) # this is y data (torch tensor) torch_dataset = Data.TensorDataset(x, y) loader = Data.DataLoader( dataset=torch_dataset, # torch TensorDataset format batch_size=BATCH_SIZE, # mini batch size shuffle=True, # random shuffle for training num_workers=2, # subprocesses for loading ) def show_batch(): for epoch in range(3): # train entire dataset 3 times for step, (batch_x, batch_y) in enumerate(loader): # for each training step # train your data... print('Epoch: ', epoch, '| Step: ', step, '| batch x: ', batch_x.numpy(), '| batch y: ', batch_y.numpy()) if __name__ == '__main__': show_batch()
parser.add_argument('--train_batch', type=int, default=16) parser.add_argument('--val_batch', type=int, default=1) parser.add_argument('--num_workers', type=int, default=8) parser.add_argument('--pretrained', type=int, default=0) parser.add_argument('--path', type=str, default=None) parser.add_argument('--epoch', type=int, default=20) parser.add_argument('--learning_rate', type=float, default=0.001) parser.add_argument('--momentum', type=float, default=0.9) parser.add_argument('--weight_decay', type=float, default=5e-4) parser.add_argument('--stage', type=int, default=1) parser.add_argument('--evaluate', type=int, default=0) args = parser.parse_args() trainset = AudioVisualData() testset = TestData() trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, collate_fn=DataAllocate, num_workers=args.num_workers) testloader = data.DataLoader(testset, batch_size=args.val_batch, shuffle=False, collate_fn=TestAllocate, num_workers=args.num_workers) vision_net = resnet18(modal='vision', pretrained=True) audio_net = resnet18(modal='audio') if args.evaluate: net = Location(vision_net, audio_net).cuda() net.load_state_dict(torch.load(args.path)) test(net, testloader) exit() net = MTask(vision_net, audio_net).cuda() if args.stage == 1 \ else Align(vision_net, audio_net).cuda() if args.pretrained:
return len(self.data_file_list) model = smp.Unet( encoder_name='resnet34', encoder_weights='imagenet', classes=1, activation='sigmoid' ) preproc_fn = smp.encoders.get_preprocessing_fn('resnet34', 'imagenet') x_train_path = r"D:\liver2\liver2\train\imgs" y_train_path = r"D:\liver2\liver2\train\masks" x_test_path = r"D:\liver2\liver2\test\imgs" y_test_path = r"D:\liver2\liver2\test\masks" train_dataset = LvDataset(x_train_path, y_train_path, [1], dataset_augmentation(), dataset_preprocessing(preproc_fn)) test_dataset = LvDataset(x_test_path, y_test_path, [1], dataset_augmentation(), dataset_preprocessing(preproc_fn)) train_loader = torchdata.DataLoader(train_dataset, batch_size=5, shuffle=True, num_workers=0) test_loader = torchdata.DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=0) best_model = torch.load('./model_est.pth') n = np.random.choice(len(test_loader)) image, mask = test_dataset[n] mask = mask.squeeze() x_tensor = torch.from_numpy(image).to("cpu").unsqueeze(0) pred_mask = best_model.predict(x_tensor) pred_mask = pred_mask.squeeze().cpu().numpy().round() image = image.transpose(1,2,0).astype("float32") visualize_help(image=image,original_mask=mask,predicted_mask=pred_mask)
torch.manual_seed(1) EPOCH = 1 BATCH_SIZE = 50 LR = 0.001 DOWNLOAD_MNIST = False train_data = torchvision.datasets.MNIST( root='./mnist/', train=True, transform=torchvision.transforms.ToTensor(), download=DOWNLOAD_MNIST, ) train_loader = Data.DataLoader(dataset=train_data, batch_size=BATCH_SIZE, shuffle=True) test_data = torchvision.datasets.MNIST(root='./mnist/', train=False) # !!!!!!!! Change in here !!!!!!!!! # test_x = Variable(torch.unsqueeze(test_data.test_data, dim=1)).type( torch.FloatTensor)[:2000].cuda() / 255. # Tensor on GPU test_y = test_data.test_labels[:2000] class CNN(nn.Module): def __init__(self): super(CNN, self).__init__() self.conv1 = nn.Sequential( nn.Conv2d(
feats_name = 'r9y9outputmel' mel_train = float_datasource( vox_dir + '/' + 'fnames.train', vox_dir + '/' + 'etc/falcon_feats.desc', feats_name, vox_dir + '/' + 'festival/falcon_' + feats_name) mel_val = float_datasource(vox_dir + '/' + 'fnames.val', vox_dir + '/' + 'etc/falcon_feats.desc', feats_name, vox_dir + '/' + 'festival/falcon_' + feats_name) # Dataset and Dataloader setup X, mfcc, mel, mol, fnames) trainset = LIDmfccmelmolDataset(X_train, mfcc_train, mel_train, mol_train, fnames_train) train_loader = data_utils.DataLoader(trainset, batch_size=hparams.batch_size, num_workers=hparams.num_workers, shuffle=True, collate_fn=collate_fn_lidmfccmelmol, pin_memory=hparams.pin_memory) valset = LIDmfccmelmolDataset(X_val, mfcc_val, mel_val, mol_val, fnames_val) val_loader = data_utils.DataLoader(valset, batch_size=hparams.batch_size, num_workers=hparams.num_workers, shuffle=True, collate_fn=collate_fn_lidmfccmelmol, pin_memory=hparams.pin_memory) # Model model = LIDMixtureofExpertsmfccattention(39, 80) model = model.cuda()
def main(): h, w = map(int, args.input_size.split(',')) input_size = (h, w) cudnn.enabled = True gpu = args.gpu # create network model = Res_Deeplab(num_classes=args.num_classes) # load pretrained parameters if args.restore_from[:4] == 'http' : saved_state_dict = model_zoo.load_url(args.restore_from) else: saved_state_dict = torch.load(args.restore_from) # only copy the params that exist in current model (caffe-like) new_params = model.state_dict().copy() for name, param in new_params.items(): print name if name in saved_state_dict and param.size() == saved_state_dict[name].size(): new_params[name].copy_(saved_state_dict[name]) print('copy {}'.format(name)) model.load_state_dict(new_params) model.train() model.cuda(args.gpu) cudnn.benchmark = True # init D model_D = FCDiscriminator(num_classes=args.num_classes) if args.restore_from_D is not None: model_D.load_state_dict(torch.load(args.restore_from_D)) model_D.train() model_D.cuda(args.gpu) if not os.path.exists(args.snapshot_dir): os.makedirs(args.snapshot_dir) train_dataset = VOCDataSet(args.data_dir, args.data_list, crop_size=input_size, scale=args.random_scale, mirror=args.random_mirror, mean=IMG_MEAN) train_dataset_size = len(train_dataset) train_gt_dataset = VOCGTDataSet(args.data_dir, args.data_list, crop_size=input_size, scale=args.random_scale, mirror=args.random_mirror, mean=IMG_MEAN) if args.partial_data is None: trainloader = data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=5, pin_memory=True) trainloader_gt = data.DataLoader(train_gt_dataset, batch_size=args.batch_size, shuffle=True, num_workers=5, pin_memory=True) else: #sample partial data partial_size = int(args.partial_data * train_dataset_size) if args.partial_id is not None: train_ids = pickle.load(open(args.partial_id)) print('loading train ids from {}'.format(args.partial_id)) else: train_ids = range(train_dataset_size) np.random.shuffle(train_ids) pickle.dump(train_ids, open(osp.join(args.snapshot_dir, 'train_id.pkl'), 'wb')) train_sampler = data.sampler.SubsetRandomSampler(train_ids[:partial_size]) train_remain_sampler = data.sampler.SubsetRandomSampler(train_ids[partial_size:]) train_gt_sampler = data.sampler.SubsetRandomSampler(train_ids[:partial_size]) trainloader = data.DataLoader(train_dataset, batch_size=args.batch_size, sampler=train_sampler, num_workers=3, pin_memory=True) trainloader_remain = data.DataLoader(train_dataset, batch_size=args.batch_size, sampler=train_remain_sampler, num_workers=3, pin_memory=True) trainloader_gt = data.DataLoader(train_gt_dataset, batch_size=args.batch_size, sampler=train_gt_sampler, num_workers=3, pin_memory=True) trainloader_remain_iter = enumerate(trainloader_remain) trainloader_iter = enumerate(trainloader) trainloader_gt_iter = enumerate(trainloader_gt) # implement model.optim_parameters(args) to handle different models' lr setting # optimizer for segmentation network optimizer = optim.SGD(model.optim_parameters(args), lr=args.learning_rate, momentum=args.momentum,weight_decay=args.weight_decay) optimizer.zero_grad() # optimizer for discriminator network optimizer_D = optim.Adam(model_D.parameters(), lr=args.learning_rate_D, betas=(0.9,0.99)) optimizer_D.zero_grad() # loss/ bilinear upsampling bce_loss = BCEWithLogitsLoss2d() interp = nn.Upsample(size=(input_size[1], input_size[0]), mode='bilinear') # labels for adversarial training pred_label = 0 gt_label = 1 for i_iter in range(args.num_steps): loss_seg_value = 0 loss_adv_pred_value = 0 loss_D_value = 0 loss_semi_value = 0 optimizer.zero_grad() adjust_learning_rate(optimizer, i_iter) optimizer_D.zero_grad() adjust_learning_rate_D(optimizer_D, i_iter) for sub_i in range(args.iter_size): # train G # don't accumulate grads in D for param in model_D.parameters(): param.requires_grad = False # do semi first if args.lambda_semi > 0 and i_iter >= args.semi_start : try: _, batch = trainloader_remain_iter.next() except: trainloader_remain_iter = enumerate(trainloader_remain) _, batch = trainloader_remain_iter.next() # only access to img images, _, _, _ = batch images = Variable(images).cuda(args.gpu) pred = interp(model(images)) D_out = interp(model_D(F.softmax(pred))) D_out_sigmoid = F.sigmoid(D_out).data.cpu().numpy().squeeze(axis=1) # produce ignore mask semi_ignore_mask = (D_out_sigmoid < args.mask_T) semi_gt = pred.data.cpu().numpy().argmax(axis=1) semi_gt[semi_ignore_mask] = 255 semi_ratio = 1.0 - float(semi_ignore_mask.sum())/semi_ignore_mask.size print('semi ratio: {:.4f}'.format(semi_ratio)) if semi_ratio == 0.0: loss_semi_value += 0 else: semi_gt = torch.FloatTensor(semi_gt) loss_semi = args.lambda_semi * loss_calc(pred, semi_gt, args.gpu) loss_semi = loss_semi/args.iter_size loss_semi.backward() loss_semi_value += loss_semi.data.cpu().numpy()[0]/args.lambda_semi else: loss_semi = None # train with source try: _, batch = trainloader_iter.next() except: trainloader_iter = enumerate(trainloader) _, batch = trainloader_iter.next() images, labels, _, _ = batch images = Variable(images).cuda(args.gpu) ignore_mask = (labels.numpy() == 255) pred = interp(model(images)) loss_seg = loss_calc(pred, labels, args.gpu) D_out = interp(model_D(F.softmax(pred))) loss_adv_pred = bce_loss(D_out, make_D_label(gt_label, ignore_mask)) loss = loss_seg + args.lambda_adv_pred * loss_adv_pred # proper normalization loss = loss/args.iter_size loss.backward() loss_seg_value += loss_seg.data.cpu().numpy()[0]/args.iter_size loss_adv_pred_value += loss_adv_pred.data.cpu().numpy()[0]/args.iter_size # train D # bring back requires_grad for param in model_D.parameters(): param.requires_grad = True # train with pred pred = pred.detach() D_out = interp(model_D(F.softmax(pred))) loss_D = bce_loss(D_out, make_D_label(pred_label, ignore_mask)) loss_D = loss_D/args.iter_size/2 loss_D.backward() loss_D_value += loss_D.data.cpu().numpy()[0] # train with gt # get gt labels try: _, batch = trainloader_gt_iter.next() except: trainloader_gt_iter = enumerate(trainloader_gt) _, batch = trainloader_gt_iter.next() _, labels_gt, _, _ = batch D_gt_v = Variable(one_hot(labels_gt)).cuda(args.gpu) ignore_mask_gt = (labels_gt.numpy() == 255) D_out = interp(model_D(D_gt_v)) loss_D = bce_loss(D_out, make_D_label(gt_label, ignore_mask_gt)) loss_D = loss_D/args.iter_size/2 loss_D.backward() loss_D_value += loss_D.data.cpu().numpy()[0] optimizer.step() optimizer_D.step() print('exp = {}'.format(args.snapshot_dir)) print('iter = {0:8d}/{1:8d}, loss_seg = {2:.3f}, loss_adv_p = {3:.3f}, loss_D = {4:.3f}, loss_semi = {5:.3f}'.format(i_iter, args.num_steps, loss_seg_value, loss_adv_pred_value, loss_D_value, loss_semi_value)) if i_iter >= args.num_steps-1: print 'save model ...' torch.save(model.state_dict(),osp.join(args.snapshot_dir, 'VOC_'+str(args.num_steps)+'.pth')) torch.save(model_D.state_dict(),osp.join(args.snapshot_dir, 'VOC_'+str(args.num_steps)+'_D.pth')) break if i_iter % args.save_pred_every == 0 and i_iter!=0: print 'taking snapshot ...' torch.save(model.state_dict(),osp.join(args.snapshot_dir, 'VOC_'+str(i_iter)+'.pth')) torch.save(model_D.state_dict(),osp.join(args.snapshot_dir, 'VOC_'+str(i_iter)+'_D.pth')) end = timeit.default_timer() print end-start,'seconds'
def custom_nmf2(V, Sw, Se, params, hyperparams, title, initE, initW): # decide where to draw negative samples from - entities (= for every (i,j) positive, find (i, *) neg) or words '''density_column = np.max(V.sum(0).A1)/V.shape[0] density_row = np.max(V.sum(1).A1)/V.shape[1] from_entities = True if density_row <= density_column else False ''' from_entities = check_densities(V) n_words = Sw.shape[0] n_entities = Se.shape[0] print('V shape: ', V.shape) print('n entities - ', n_entities) print('n words - ', n_words) losses = [] # define model '''initE, initW = init_values(V, hyperparams.n_features) np.save('initE.npy', initE) np.save('initW.npy', initW) initW = np.load('initW.npy') initE = np.load('initE.npy') ''' initW = np.abs(initW) initE = np.abs(initE) print('Computed initial values') print('initW min = {}, mean = {}'.format(np.min(initW), np.mean(initW))) print('initE min = {}, mean = {}'.format(np.min(initE), np.mean(initE))) model = NMF(n_entities, n_words, hyperparams.n_features, initE, initW, params) initW = None initE = None print('Created NMF model') loss = CustomLoss(params.device) optimizer = hyperparams.optim(model.parameters(), **hyperparams.optim_settings) # construct tensors V_tensor = torch.Tensor(V.todense().astype(np.float32)) V_var = Variable(V_tensor, requires_grad=False) MatrixDataset = DS(V, hyperparams.n_negatives, from_entities) trainloader = data.DataLoader(MatrixDataset, batch_size=5, shuffle=True) losses = [] differences = [] for epoch in range(hyperparams.n_epochs): # construct batch = one row, n positive samples, 3n negative samples total_loss = 0 bcount = -1 for batch2 in trainloader: # transform batch batch = [] for row, col in batch2: batch.extend(list(zip(row, col))) bcount += 1 optimizer.zero_grad() #print('On batch ', bcount) if print_time: # construct sample t_start = timeit.default_timer() target = get_target_values(batch, V) if print_time: t_diff = (timeit.default_timer() - t_start) * 100 print('Time to collect target - ', t_diff) if print_time: t_start = timeit.default_timer() prediction = model(batch) if print_time: t_diff = (timeit.default_timer() - t_start) * 100 print('Time to compute prediction - ', t_diff) if print_time: t_start = timeit.default_timer() l = loss(target, prediction, hyperparams.lambdas, batch, model, Sw, Se) if print_time: t_diff = (timeit.default_timer() - t_start) * 100 print('Time to compute loss - ', t_diff) l.backward(retain_graph=True) optimizer.step() model.positivise() total_loss += l.item() # test the model after each epoch #total_loss /= hyperparams.n_batches losses.append(total_loss) print('Epoch - {}: loss - {}'.format(epoch, total_loss)) if epoch % 50 == 0 or epoch == hyperparams.n_epochs - 1: t_start = timeit.default_timer() V_prediction = torch.mm(model.E, model.W) t_diff = timeit.default_timer() - t_start #print('time to compute difference - ', t_diff * 100) diff = (V_tensor - V_prediction).norm(2).item() differences.append(diff) print('\t\t \t \t difference - {}'.format(diff)) MatrixDataset.reset() plot_results(losses, differences, hyperparams.n_epochs, model, title) return model.E, model.W
num_crop=8 scores_pred = np.zeros((num_crop, len(data_set['test']),4),dtype=np.float32) q = np.linspace(0, 2600-int(2600*0.8), num_crop,dtype=int) for aug_cnt, crop_start in enumerate(q): crop_end = crop_start+int(2600*0.8) data_set = {} data_set['test'] = TWdata(index_pd=test_index, data_root=test_data_root, classes=le.classes_, transform=TWAugVal(start_point=crop_start,end_point=crop_end), ) data_loader = {} data_loader['test'] = torchdata.DataLoader(data_set['test'], batch_size=bs, num_workers=4, shuffle=False, pin_memory=True) idx = 0 for batch_cnt, data in enumerate(data_loader['test']): print 'aug %d, %d/%d' % (aug_cnt, batch_cnt, len(data_set['test']) // bs) inputs, labels = data if usecuda: inputs = Variable(inputs.cuda()) else: inputs = Variable(inputs) # forward gb,outputs = model(inputs) # score = F.softmax(outputs) score = outputs
print(f'Training MNIST Model on {device}\n{"=" * 44}') # MNIST Data Loading train_dataset = datasets.MNIST(root='./mnist_data/', train=True, transform=transforms.ToTensor(), download=True) test_dataset = datasets.MNIST(root='./mnist_data/', train=False, transform=transforms.ToTensor()) # Data Loader (Input Pipeline) train_loader = data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True) test_loader = data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False) class MNISTClassifier(nn.Module): def __init__(self): super(MNISTClassifier, self).__init__() self.l1 = nn.Linear(784, 520) self.l2 = nn.Linear(520, 320) self.l3 = nn.Linear(320, 240) self.l4 = nn.Linear(240, 120) self.l5 = nn.Linear(120, 10)
def main(): ##################### # Generate data ##################### # data loader - if isServerRun: path = '/home/[email protected]/thesisML/' else: path = '/Users/chanaross/dev/Thesis/UberData/' fileName = '3D_allDataLatLonCorrected_20MultiClass_500gridpickle_30min.p' dataInput = np.load(path + fileName) flag_save_network = True xmin = 0 xmax = dataInput.shape[0] ymin = 0 ymax = dataInput.shape[1] zmin = 48 zmax = np.floor(dataInput.shape[2]*0.7).astype(int) dataInput = dataInput[xmin:xmax, ymin:ymax, zmin:zmax] # shrink matrix size for fast training in order to test model dataInput = dataInput[5:6, 10:11, :] smoothParam = [10, 30, 40] #[10, 20, 30, 40] #[10, 15, 30] testSize = 0.2 # define hyper parameters - hidden_sizeVec = [64, 128, 256] # [20, 64, 256, 512] #[20, 64, 264, 512] # [20, 40, 64, 128] sequence_sizeVec = [50, 60] # [5, 10, 20] # [5, 20, 30, 40] # [5, 10, 15] # length of sequence for lstm network batch_sizeVec = [40] num_epochs = 500 # optimizer parameters - lrVec = [0.05, 0.01] #[0.05, 0.2, 0.5] # [0.1, 0.5, 0.9] #[0.1, 0.5, 0.9] # [0.1, 0.01, 0.001] otVec = [1] # [1, 2] dmp = 0 mm = 0.9 eps = 1e-08 wdVec = [2e-3] # create case vectors networksDict = {} itr = itertools.product(smoothParam, sequence_sizeVec, batch_sizeVec, hidden_sizeVec, lrVec, otVec, wdVec) for i in itr: networkStr = 'smooth_{0}_seq_{1}_bs_{2}_hs_{3}_lr_{4}_ot_{5}_wd_{6}'.format(i[0], i[1], i[2], i[3], i[4], i[5], i[6]) networksDict[networkStr] = {'seq': i[1], 'bs': i[2], 'hs': i[3], 'lr': i[4], 'ot': i[5], 'wd': i[6], 'sm': i[0]} for netConfig in networksDict: dataInputSmooth = moving_average(dataInput, networksDict[netConfig]['sm']) # smoothing data so that results are more clear to network # dataInput[dataInput>1] = 1 # limit all events larger than 10 to be 10 # define important sizes for network - x_size = dataInputSmooth.shape[0] y_size = dataInputSmooth.shape[1] dataSize = dataInputSmooth.shape[2] class_size = (np.max(np.unique(dataInputSmooth)) + 1).astype(int) num_train = int((1 - testSize) * dataSize) grid_size = x_size * y_size # output file outFile = open('LSTM_networksOutput.csv', 'w') outFile.write('Name;finalAcc;finalLoss;trainTime;numWeights;NumEpochs\n') print('Net Parameters: ' + netConfig) # create network based on input parameter's - hidden_size = networksDict[netConfig]['hs'] batch_size = networksDict[netConfig]['bs'] sequence_size = networksDict[netConfig]['seq'] lr = networksDict[netConfig]['lr'] ot = networksDict[netConfig]['ot'] wd = networksDict[netConfig]['wd'] my_net = Model(grid_size, hidden_size, batch_size, sequence_size, class_size) my_net.lstm = my_net.create_lstm(grid_size) # lstm receives all grid points and seq length of my_net.fc_after_lstm = my_net.create_fc_after_lstm(my_net.hiddenSize, grid_size*class_size) my_net.to(device) print("model device is:") print(next(my_net.parameters()).device) numWeights = sum(param.numel() for param in my_net.parameters()) print('number of parameters: ', numWeights) my_net.optimizer = CreateOptimizer(my_net.parameters(), ot, lr, dmp, mm, eps, wd) my_net.lossCrit = nn.NLLLoss(size_average=True) # nn.BCELoss(size_average=True) my_net.maxEpochs = num_epochs my_net.lr = lr my_net.wd = wd my_net.smoothingParam = networksDict[netConfig]['sm'] # network_path = '/Users/chanaross/dev/Thesis/MachineLearning/forGPU/GPU_results/limitedZero_500grid/' # network_name = 'gridSize11_epoch4_batch5_torch.pkl' # my_net = torch.load(network_path + network_name, map_location=lambda storage, loc: storage) # load data from data loader and create train and test sets data_train = dataInputSmooth[:, :, 0:num_train] data_test = dataInputSmooth[:, :, num_train:] dataset_uber_train = DataSet_oneLSTM_allGrid(data_train, sequence_size) dataset_uber_test = DataSet_oneLSTM_allGrid(data_test , sequence_size) # creating data loader dataloader_uber_train = data.DataLoader(dataset=dataset_uber_train, batch_size=batch_size, shuffle=False) dataloader_uber_test = data.DataLoader(dataset=dataset_uber_test , batch_size=batch_size, shuffle=False) netOutDict = {} labelsOutDict = {} for numEpoch in range(num_epochs): my_net.loss = None # for each epoch, calculate loss for each batch - my_net.train() localLoss = [4] accTrain = [0] rmseTrain = [1] trainCorr = 0.0 trainTot = 0.0 if (1+numEpoch)%40 == 0: if my_net.optimizer.param_groups[0]['lr'] > 0.001: my_net.optimizer.param_groups[0]['lr'] = my_net.optimizer.param_groups[0]['lr']/2 else: my_net.optimizer.param_groups[0]['lr'] = 0.001 print('lr is: %.6f' % my_net.optimizer.param_groups[0]['lr']) netOutList = [] labelOutList = [] for i, (input, labels) in enumerate(dataloader_uber_train): inputD = input.to(device) labelsD = labels.to(device) my_net.loss = None # create torch variables # input is of size [batch_size, grid_id, seq_size] inputVar = Variable(inputD).to(device) labVar = Variable(labelsD).to(device) # if isServerRun: # labVar = labVar.type(torch.cuda.FloatTensor) # else: # labVar = labVar.type(torch.FloatTensor) # reset gradient my_net.optimizer.zero_grad() # forward grid_size = labels.shape[1] local_batch_size = input.shape[0] # input to LSTM is [seq_size, batch_size, grid_size] , will be transferred as part of the forward netOut = my_net.forward(inputVar) netOut = netOut.view(local_batch_size, class_size, grid_size) _, labTrain = torch.max(torch.exp(netOut.data), 1) my_net.calcLoss(netOut, labVar) # backwards my_net.backward() # optimizer step my_net.optimizer.step() # local loss function list localLoss.append(my_net.loss.item()) # if isServerRun: # labTrain = labTrain.cpu() if isServerRun: labTrainNp = labTrain.type(torch.cuda.LongTensor).cpu().detach().numpy() # print("number of net labels different from 0 is:" + str(np.sum(labTrainNp > 0))) # print("number of net labels 0 is:"+str(np.sum(labTrainNp == 0))) labelsNp = labels.cpu().detach().numpy() # print("number of real labels different from 0 is:" + str(np.sum(labelsNp > 0))) # print("number of real labels 0 is:" + str(np.sum(labelsNp == 0))) trainCorr = torch.sum(labTrain.type(torch.cuda.LongTensor) == labels).cpu().detach().numpy() + trainCorr else: labTrainNp = labTrain.long().detach().numpy() labelsNp = labels.detach().numpy() trainCorr = torch.sum(labTrain.long() == labels).detach().numpy() + trainCorr netOutList.append(labTrainNp) labelOutList.append(labelsNp) trainTot = labels.size(0) * labels.size(1) + trainTot rmse = sqrt(metrics.mean_squared_error(labTrainNp.reshape(-1), labelsNp.reshape(-1))) accTrain.append(100 * trainCorr / trainTot) rmseTrain.append(rmse) # output current state if (i + 1) % 2 == 0: print('Epoch: [%d/%d1 ], Step: [%d/%d], Loss: %.4f, Acc: %.4f, RMSE: %.4f' % (numEpoch + 1, my_net.maxEpochs, i + 1, dataloader_uber_train.batch_size, my_net.loss.item(), accTrain[-1], rmseTrain[-1])) # if (i+1) % 20 == 0: # if ((localLoss[-1] < np.max(np.array(localLoss[0:-1]))) or (accTrain[-1] > np.max(np.array(accTrain[0:-1])))) and flag_save_network: # # pickle.dump(my_net, open("gridSize" + str(xmax - xmin) + "_epoch" + str(numEpoch+1) + "_batch" + str(i+1) + ".pkl", 'wb')) # my_net.saveModel("gridSize" + str(xmax - xmin) + "_epoch" + str(numEpoch+1) + "_batch" + str(i+1) + "_torch.pkl") # # networkStr = "gridSize" + str(xmax - xmin) + "_epoch" + str(numEpoch+1) + "_batch" + str(i+1) # # outArray = np.stack([np.array(localLoss), np.array(accTrain)]) # # np.save(networkStr + "_oArrBatch.npy", outArray) my_net.lossVecTrain.append(np.average(localLoss)) my_net.accVecTrain.append(np.average(accTrain)) my_net.rmseVecTrain.append(np.average(rmseTrain)) # test network for each epoch stage accEpochTest, lossEpochTest, rmseEpochTest = my_net.test_spesific(testLoader=dataloader_uber_test) my_net.accVecTest.append(accEpochTest) my_net.lossVecTest.append(lossEpochTest) my_net.rmseVecTest.append(rmseEpochTest) netOutDict[numEpoch] = netOutList labelsOutDict[numEpoch] = labelOutList if (flag_save_network): my_net.saveModel(netConfig + "_torch.pkl") # outArray = np.stack([np.array(my_net.lossVecTest), np.array(my_net.lossVecTrain), # np.array(my_net.accVecTest), np.array(my_net.accVecTrain)]) # np.save("gridSize" + str(xmax - xmin) + "_epoch" + str(numEpoch) + "_oArrBatch.npy", outArray) my_net.finalAcc = accEpochTest my_net.finalLoss = lossEpochTest my_net.finalRmse = rmseEpochTest # name, HyperPerams, accur, num total weights # err vs epoch, loss vs epoch, saveFile(netOutDict, 'netDict') saveFile(labelsOutDict, 'labelsDict') strWrite = '{0};{1};{2};{3};{4}\n'.format(netConfig, my_net.finalAcc, my_net.finalLoss, numWeights, my_net.maxEpochs) outFile.write(strWrite) outFile.close() return
flat_list.append(item) """ flatten = lambda l: [item for sublist in l for item in sublist] x_train_flat = flatten(x_train.values.tolist()) x_test_flat = flatten(x_test.values.tolist()) # Datasets partition = {'train': x_train_flat, 'validation': x_test_flat} labels = data.set_index('name').to_dict() """ partition={'train': ['id-1', 'id-2', 'id-3'], 'validation': ['id-4']} labels={'id-1': 0, 'id-2': 1, 'id-3': 2, 'id-4': 1} """ # Generators training_set = Dataset(partition['train'], labels) training_generator = Data.DataLoader(training_set, **params) #inputs, labels,area=next(iter(training_generator)) #plt.imshow(moveaxis(a[0,:,:,:],2, 0)) validation_set = Dataset(partition['validation'], labels) validation_generator = Data.DataLoader(validation_set, **params) dataloaders_dict = {'train': training_generator, 'val': validation_generator} MINIBATCH_SIZE = 8 """ train_inputs, train_labels = torch.FloatTensor(normalizedImage(x_train_image)), torch.FloatTensor(y_train[:]) test_inputs, test_labels = torch.FloatTensor(normalizedImage(x_test_image)), torch.FloatTensor(y_test[:]) loader_train = Data.DataLoader(dataset=Data.TensorDataset(train_inputs, train_labels), batch_size=MINIBATCH_SIZE,shuffle=True, num_workers=0)
def train_single_domain_transfer(args, wf, src, repeat_seed): # tb_dir = 'runs/{}_sup_base_{}_source_{}_train_num_{}_tune_{}_{}'.format( # args.test, args.base_model, src, args.train_num, args.n_tune, repeat_seed) # if os.path.exists(tb_dir) and os.path.isdir(tb_dir): # shutil.rmtree(tb_dir) # writer = SummaryWriter(tb_dir) args.cuda = not args.no_cuda and torch.cuda.is_available() say('cuda is available %s\n' % args.cuda) np.random.seed(args.seed + repeat_seed) torch.manual_seed(args.seed + repeat_seed) if args.cuda: torch.cuda.manual_seed(args.seed + repeat_seed) pretrain_emb = torch.load( os.path.join(settings.OUT_DIR, "rnn_init_word_emb.emb")) src_model_dir = os.path.join(settings.OUT_DIR, src) if args.base_model == "cnn": encoder_src = CNNMatchModel(input_matrix_size1=args.matrix_size1, input_matrix_size2=args.matrix_size2, mat1_channel1=args.mat1_channel1, mat1_kernel_size1=args.mat1_kernel_size1, mat1_channel2=args.mat1_channel2, mat1_kernel_size2=args.mat1_kernel_size2, mat1_hidden=args.mat1_hidden, mat2_channel1=args.mat2_channel1, mat2_kernel_size1=args.mat2_kernel_size1, mat2_hidden=args.mat2_hidden) elif args.base_model == "rnn": encoder_src = BiLSTM(pretrain_emb=pretrain_emb, vocab_size=args.max_vocab_size, embedding_size=args.embedding_size, hidden_size=args.hidden_size, dropout=args.dropout) else: raise NotImplementedError if args.cuda: encoder_src.load_state_dict( torch.load( os.path.join( src_model_dir, "{}-match-best-now-train-num-{}-try-{}.mdl".format( args.base_model, args.train_num, repeat_seed)))) else: encoder_src.load_state_dict( torch.load(os.path.join( src_model_dir, "{}-match-best-now-train-num-{}-try-{}.mdl".format( args.base_model, args.train_num, repeat_seed)), map_location=torch.device('cpu'))) dst_model_dir = os.path.join(settings.OUT_DIR, args.test) if args.base_model == "cnn": encoder_dst_pretrain = CNNMatchModel( input_matrix_size1=args.matrix_size1, input_matrix_size2=args.matrix_size2, mat1_channel1=args.mat1_channel1, mat1_kernel_size1=args.mat1_kernel_size1, mat1_channel2=args.mat1_channel2, mat1_kernel_size2=args.mat1_kernel_size2, mat1_hidden=args.mat1_hidden, mat2_channel1=args.mat2_channel1, mat2_kernel_size1=args.mat2_kernel_size1, mat2_hidden=args.mat2_hidden) elif args.base_model == "rnn": encoder_dst_pretrain = BiLSTM(pretrain_emb=pretrain_emb, vocab_size=args.max_vocab_size, embedding_size=args.embedding_size, hidden_size=args.hidden_size, dropout=args.dropout) else: raise NotImplementedError encoder_dst_pretrain.load_state_dict( torch.load( os.path.join( dst_model_dir, "{}-match-best-now-train-num-{}-try-{}.mdl".format( args.base_model, args.train_num, repeat_seed)))) # args = argparser.parse_args() say(args) print() say("Transferring from %s to %s\n" % (src, args.test)) if args.base_model == "cnn": train_dataset_dst = ProcessedCNNInputDataset(args.test, "train", args.train_num) valid_dataset = ProcessedCNNInputDataset(args.test, "valid") test_dataset = ProcessedCNNInputDataset(args.test, "test") elif args.base_model == "rnn": train_dataset_dst = ProcessedRNNInputDataset(args.test, "train", args.train_num) valid_dataset = ProcessedRNNInputDataset(args.test, "valid") test_dataset = ProcessedRNNInputDataset(args.test, "test") else: raise NotImplementedError print("train num", len(train_dataset_dst)) train_loader_dst = data.DataLoader(train_dataset_dst, batch_size=args.batch_size, shuffle=False, num_workers=0) valid_loader = data.DataLoader(valid_dataset, batch_size=args.batch_size, shuffle=False, num_workers=0) test_loader = data.DataLoader(test_dataset, batch_size=args.batch_size, shuffle=False, num_workers=0) say("Corpus loaded.\n") if args.cuda: encoder_dst_pretrain.cuda() encoder_src.cuda() # classifier.cuda() requires_grad = lambda x: x.requires_grad # task_params = list(classifier.parameters()) # if args.base_model == "cnn": # optim_model = optim.Adagrad( # filter(requires_grad, task_params), # lr=args.lr, # weight_decay=args.weight_decay # ) # elif args.base_model == "rnn": # optim_model = optim.Adam( # filter(requires_grad, task_params), # lr=args.lr, # weight_decay=args.weight_decay # ) # else: # raise NotImplementedError say("Training will begin from scratch\n") iter_cnt = 0 min_loss_val = None max_auc_val = None best_test_results = None weights_sources = None model_dir = os.path.join(settings.OUT_DIR, args.test) cur_src_idx = source_to_idx[src] for epoch in range(args.max_epoch): print("training epoch", epoch) # iter_cnt = train_epoch( # iter_cnt, # [encoder_src, encoder_dst_pretrain], classifier, # train_loader_dst, # args, # optim_model, # epoch, # writer # ) thr, metrics_val = evaluate( epoch, [encoder_src, encoder_dst_pretrain], valid_loader, True, args, # writer ) _, metrics_test = evaluate( epoch, [encoder_src, encoder_dst_pretrain], test_loader, False, args, # writer, thr=thr) if min_loss_val is None or min_loss_val > metrics_val[0]: print("change val loss from {} to {}".format( min_loss_val, metrics_val[0])) min_loss_val = metrics_val[0] best_test_results = metrics_test # torch.save(classifier, # os.path.join(model_dir, "{}_{}_classifier_from_src_{}_train_num_{}_try_{}.mdl".format( # args.test, args.base_model, cur_src_idx, args.train_num, repeat_seed # ))) print() print( "src: {}, min valid loss {:.4f}, best test metrics: AUC: {:.2f}, Prec: {:.2f}, Rec: {:.2f}, F1: {:.2f}\n" .format(src, min_loss_val, best_test_results[1] * 100, best_test_results[2] * 100, best_test_results[3] * 100, best_test_results[4] * 100)) wf.write( "from src {}, min valid loss {:.4f}, best test metrics: AUC: {:.2f}, Prec: {:.2f}, Rec: {:.2f}, F1: {:.2f}\n" .format(src, min_loss_val, best_test_results[1] * 100, best_test_results[2] * 100, best_test_results[3] * 100, best_test_results[4] * 100)) # writer.close() return min_loss_val, best_test_results[1:]
parser.add_argument("--out", default=True, help="save model or not") parser.add_argument("--gpu", type=str, default="0", help="gpu card ID") args = parser.parse_args() os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu cudnn.benchmark = True ############################## PREPARE DATASET ########################## train_data, test_data, user_num, item_num, train_mat = data_utils.load_all( ) # construct the train and test datasets train_dataset = data_utils.BPRData(train_data, item_num, train_mat, args.num_ng, True) test_dataset = data_utils.BPRData(test_data, item_num, train_mat, 0, False) train_loader = data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=4) test_loader = data.DataLoader(test_dataset, batch_size=args.test_num_ng + 1, shuffle=False, num_workers=0) ########################### CREATE MODEL ################################# model = model.BPR(user_num, item_num, args.factor_num) #model.cuda() optimizer = optim.SGD(model.parameters(), lr=args.lr, weight_decay=args.lamda) # writer = SummaryWriter() # for visualization
desc="", removeHs=args.removeHs, labelspath=args.vscreening, ) amap = utils.load_amap(args.amap) testdata.atomicnums_to_idxs(amap) n_species = len(amap) mlflow.log_param("n_species", n_species) testloader = data.DataLoader( testdata, batch_size=args.batchsize, shuffle=False, collate_fn=loaders.pad_collate, ) AEVC = utils.loadAEVC(args.aev) models = [utils.loadmodel(m) for m in args.models] evaluate( models, testloader, AEVC, args.outpath, stage="predict", baseline=None, plt=args.plot,
l.backward() optimizer.step() train_l_sum += l.item() train_acc_sum += (outputs.argmax(dim=1) == y).sum().item() n += y.shape[0] print('epoch %d loss %f train acc %f' % (epoch + 1, train_l_sum / n, train_acc_sum / n)) if __name__ == '__main__': device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') raw_data, cat_cols, num_cols = load_data() train_data = preprocess_data(raw_data, cat_cols, num_cols) # train_data['label'] = raw_data['label'] train_data['label'] = raw_data['Label'] cat_tuple_list = get_cat_tuple_list(train_data, cat_cols) X = torch.tensor(train_data.drop(['label'], axis=1).values, dtype=torch.float) y = torch.tensor(train_data.label.values, dtype=torch.long) dataset = Data.TensorDataset(X, y) data_iter = Data.DataLoader(dataset=dataset, batch_size=128, shuffle=True) model = DeepFM(num_cols, cat_cols, cat_tuple_list) # print(model) loss = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=0.001) # print(model) train(model, data_iter, device, optimizer, loss, epochs=100)
def main(): logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S", filename="log.txt", filemode='w') console = logging.StreamHandler() console.setLevel(logging.INFO) formatter = logging.Formatter('%(message)s') console.setFormatter(formatter) logging.getLogger('').addHandler(console) model = Dronet(PreActBlock, [1, 1, 1], True) ModelManager.Read('../PyTorch/Models/DronetGray.pt', model) DATA_PATH = "/Users/usi/PycharmProjects/data/" picklename = "HimaxDynamic_12_03_20.pickle" [x_test, y_test, z_test] = DataProcessor.ProcessTestData(DATA_PATH + picklename, True) t_test = DataProcessor.GetTimeStampsFromTestData(DATA_PATH + picklename) if picklename.find(".pickle"): picklename = picklename.replace(".pickle", '') x_test2 = [] y_test2 = [] z_test2 = [] # for i in range(len(x_test)): # gt = y_test[i] # if ((gt[0] > 1.0) and (gt[0] < 2.0)): # x_test2.append(x_test[i]) # y_test2.append(y_test[i]) # z_test2.append(z_test[i]) # # x_test = np.asarray(x_test2) # y_test = np.asarray(y_test2) # z_test = np.asarray(z_test2) test_set = Dataset(x_test, y_test) params = {'batch_size': 1, 'shuffle': False, 'num_workers': 1} test_generator = data.DataLoader(test_set, **params) trainer = ModelTrainer(model) MSE, MAE, r2_score, outputs, gt_labels = trainer.Test(test_generator) # utils.SaveResultsToCSV(gt_labels, outputs, t_test, "wow.csv") model2 = Dronet(PreActBlock, [1, 1, 1], True) ModelManager.Read('../PyTorch/Models/DronetGrayAug120.pt', model2) trainer2 = ModelTrainer(model2) MSE, MAE, r2_score, outputs2, gt_labels2 = trainer2.Test(test_generator) utils.SaveResultsToCSVWithCamPoses(gt_labels, outputs2, t_test, z_test, picklename + ".csv") outputs2 = np.reshape(outputs2, (-1, 4)) h = x_test.shape[2] w = x_test.shape[3] x_test = np.reshape(x_test, (-1, h, w)) VizWorldTopView(x_test, y_test, z_test, outputs, outputs2, True, picklename)
def train(config): ## set pre-process prep_dict = {} prep_config = config["prep"] prep_dict["train_set1"] = prep.image_train( \ resize_size=prep_config["resize_size"], \ crop_size=prep_config["crop_size"]) prep_dict["train_set2"] = prep.image_train( \ resize_size=prep_config["resize_size"], \ crop_size=prep_config["crop_size"]) ## prepare data dsets = {} dset_loaders = {} data_config = config["data"] dsets["train_set1"] = ImageList(open(data_config["train_set1"]["list_path"]).readlines(), transform=prep_dict["train_set1"]) dset_loaders["train_set1"] = util_data.DataLoader(dsets["train_set1"], batch_size=data_config["train_set1"]["batch_size"], shuffle=True, num_workers=data_config["train_set2"]['workers'], pin_memory=True) dsets["train_set2"] = ImageList(open(data_config["train_set2"]["list_path"]).readlines(), transform=prep_dict["train_set2"]) dset_loaders["train_set2"] = util_data.DataLoader(dsets["train_set2"], batch_size=data_config["train_set2"]["batch_size"], shuffle=True, num_workers=data_config["train_set2"]['workers'], pin_memory=True) hash_bit = config["hash_bit"] ## set base network net_config = config["network"] if net_config["net_snapshot"] is None: base_network = net_config["type"](**net_config["params"]) else: base_network = torch.load(config["snapshot_path"]) use_gpu = torch.cuda.is_available() if use_gpu: base_network = base_network.cuda() ## collect parameters parameter_list = [{"params": base_network.feature_layers.parameters(), "lr": 1}, \ {"params": base_network.hash_layer.weight, "lr": 10}, {"params": base_network.hash_layer.bias, "lr": 20}] ## set optimizer optimizer_config = config["optimizer"] optimizer = optim_dict[optimizer_config["type"]](parameter_list, **(optimizer_config["optim_params"])) ## set loss function loss_config = config["loss"] loss_f = loss_dict[loss_config["type"]] param_lr = [] for param_group in optimizer.param_groups: param_lr.append(param_group["lr"]) schedule_param = optimizer_config["lr_param"] lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] ## train len_train1 = len(dset_loaders["train_set1"]) - 1 len_train2 = len(dset_loaders["train_set2"]) - 1 transfer_loss_value = classifier_loss_value = total_loss_value = 0.0 best_acc = 0.0 t_range = trange(config["num_iterations"], desc="Loss Value", leave=True) for i in t_range: if i % config["snapshot_interval"] == 0 or i == config["num_iterations"]-1: if i == config["num_iterations"]-1: j = i+1 else: j = i torch.save(base_network, osp.join(config["output_path"], \ "iter_{:05d}_model.pth.tar".format(j))) ## train one iter base_network.train(True) optimizer = lr_scheduler(param_lr, optimizer, i, **schedule_param) optimizer.zero_grad() if i % len_train1 == 0: iter1 = iter(dset_loaders["train_set1"]) inputs1, labels1 = iter1.next() if use_gpu: inputs1, labels1, = Variable(inputs1).cuda(), Variable(labels1).cuda() else: inputs1, labels1 = Variable(inputs1), Variable(labels1) if config["dual_batch"]: if i % len_train2 == 0: iter2 = iter(dset_loaders["train_set2"]) inputs2, labels2 = iter2.next() if use_gpu: inputs2, labels2 = Variable(inputs2).cuda(), Variable(labels2).cuda() else: inputs2, labels2 = Variable(inputs2), Variable(labels2) else: inputs2 = inputs1 labels2 = labels1 inputs = torch.cat((inputs1, inputs2), dim=0) outputs = base_network(inputs) similarity_loss = loss_f(outputs.narrow(0, 0, inputs1.size(0)), labels1, outputs.narrow(0, inputs1.size(0), inputs2.size(0)), labels2, config_loss=loss_config["loss_param"]) similarity_loss.backward() if i > 0: t_range.set_description("Loss Value: %f" % similarity_loss.float()) t_range.refresh() # print("Iter: {:05d}, loss: {:.3f}".format(i, similarity_loss.float().data[0])) config["out_file"].write("Iter: {:05d}, loss: {:.3f}".format(i, \ similarity_loss.float())) optimizer.step()
LR = 0.01 BATCH_SIZE = 32 EPOCH = 12 # fake dataset x = torch.unsqueeze(torch.linspace(-1, 1, 1000), dim=1) y = x.pow(2) + 0.1*torch.normal(torch.zeros(*x.size())) # plot dataset plt.scatter(x.numpy(), y.numpy()) plt.show() # 使用上节内容提到的 data loader torch_dataset = Data.TensorDataset(data_tensor=x, target_tensor=y) loader = Data.DataLoader(dataset=torch_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2,) # 默认的 network 形式 class Net(torch.nn.Module): def __init__(self): super(Net, self).__init__() self.hidden = torch.nn.Linear(1, 20) # hidden layer self.predict = torch.nn.Linear(20, 1) # output layer def forward(self, x): x = F.relu(self.hidden(x)) # activation function for hidden layer x = self.predict(x) # linear output return x # 为每个优化器创建一个 net net_SGD = Net()