def input_index_data(num_docs=None, batch_size=8, dataset='f30k'): from dataset import get_data_loader captions = 'dataset_flickr30k.json' if dataset == 'f30k' else 'captions.txt' data_loader = get_data_loader(root=os.path.join(cur_dir, f'data/{dataset}/images'), captions=os.path.join( cur_dir, f'data/{dataset}/{captions}'), split='test', batch_size=batch_size, dataset_type=dataset) for i, (images, captions) in enumerate(data_loader): for image in images: with Document() as document: document.buffer = image document.modality = 'image' document.mime_type = 'image/jpeg' yield document for caption in captions: with Document() as document: document.text = caption document.modality = 'text' document.mime_type = 'text/plain' yield document if num_docs and (i + 1) * batch_size >= num_docs: break
def run(self): train_dl, valid_dl = dataset.get_data_loader(self.bs) train_dl = WrappedDataLoader(train_dl, preprocess) valid_dl = WrappedDataLoader(valid_dl, preprocess) model, opt = self.get_model() loss_func = F.cross_entropy self.fit(self.epochs, model, loss_func, opt, train_dl, valid_dl)
def test1(): # visualize spectrograms in a batch root_dir = "data/LibriSpeech/dev-clean" loader = get_data_loader(root_dir, batch_size=4, shuffle=True) for i, batch in enumerate(loader): spec = batch["specgram"] break fig = plt.figure() for i in range(4): ax = fig.add_subplot(4, 1, i + 1) ax.imshow(spec.log2()[i, :, :].detach().numpy(), cmap='gray') plt.savefig(visual_path + "specgram.png")
def main(args): # Image preprocessing transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)) ]) image_dir = "data/" json_path = image_dir + "annotations/captions_train2014.json" root_dir = image_dir + "train2014" dataset = CocoDataset(json_path=json_path, root_dir=root_dir, transform=transform) data_loader = get_data_loader(dataset, batch_size=32) # Build models encoder = FeatureExtractor(args.embed_size).eval( ) # eval mode (batchnorm uses moving mean/variance) decoder = CaptionGenerator(args.embed_size, args.hidden_size, len(dataset.vocabulary), args.num_layers) encoder = encoder.to(device) decoder = decoder.to(device) # Load the trained model parameters encoder.load_state_dict(torch.load(args.encoder_path)) decoder.load_state_dict(torch.load(args.decoder_path)) # Prepare an image image = load_image(args.image, transform) image_tensor = image.to(device) # Generate an caption from the image feature = encoder(image_tensor) sampled_ids = decoder.sample(feature) sampled_ids = sampled_ids[0].cpu().numpy( ) # (1, max_seq_length) -> (max_seq_length) # Convert word_ids to words sampled_caption = [] for word_id in sampled_ids: word = data_loader.dataset.id_to_word[word_id] sampled_caption.append(word) if word == '<end>': break sentence = ' '.join(sampled_caption) # Print out the image and the generated caption print(sentence) image = Image.open(args.image) plt.imshow(np.asarray(image))
def main(args): set_seed(args.seed) # load data loaders = (get_data_loader(args=args, data_path=args.data_path, bert_path=args.bert_path, batch_size=args.batch_size, num_workers=args.num_workers, split=split) for split in ['train', 'dev']) trn_loader, dev_loader = loaders # initialize model model = MultimodalTransformer( n_layers=args.n_layers, n_heads=args.n_heads, n_classes=args.n_classes, only_audio=args.only_audio, only_text=args.only_text, d_audio_orig=args.n_mfcc, d_text_orig=768, # BERT hidden size d_model=args.d_model, attn_dropout=args.attn_dropout, relu_dropout=args.relu_dropout, emb_dropout=args.emb_dropout, res_dropout=args.res_dropout, out_dropout=args.out_dropout, attn_mask=args.attn_mask).to(args.device) # warmup scheduling args.total_steps = round(len(trn_loader) * args.epochs) args.warmup_steps = round(args.total_steps * args.warmup_percent) # optimizer & scheduler optimizer, scheduler = get_optimizer_and_scheduler(args, model) logging.info('training starts') model.zero_grad() args.global_step = 0 for epoch in tqdm(range(1, args.epochs + 1), desc='epochs'): # training and evaluation steps train(args, model, trn_loader, optimizer, scheduler) loss, f1 = evaluate(model, dev_loader, args.device) # save model model_name = "epoch{}-loss{:.4f}-f1{:.4f}.bin".format(epoch, loss, f1) model_path = os.path.join(args.save_path, model_name) torch.save(model.state_dict(), model_path) logging.info('training ended')
def test2(): # LAS forward pass root_dir = "data/LibriSpeech/dev-clean" loader = get_data_loader(root_dir, batch_size=4, shuffle=True) for i, batch in enumerate(loader): break spec = batch["specgram"] print(spec.size()) h_dim = 512 listen = Listen(n_mels=40, h_dim=h_dim) spell = AttendAndSpell(h_dim=h_dim, voc_size=voc_size()) h = listen(spec) print(h.size()) spell(h)
def main(): mp.set_start_method('spawn') args = parse_args() torch.set_grad_enabled(True) data_loader_train, data_loader_dev, data_loader_test = get_data_loader( args) print("Data loaded.") model = Net(args).to(args.device) print("Model set.") optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) print("Optimizer set.") train(args, data_loader_train, data_loader_dev, data_loader_test, model, loss_fn, optimizer)
def evaluation_generator(num_docs=None, batch_size=8, dataset_type='f8k', mode='text2image'): from dataset import get_data_loader captions = 'dataset_flickr30k.json' if dataset_type == 'f30k' else 'captions.txt' data_loader = get_data_loader( root=os.path.join(cur_dir, f'data/{dataset_type}/images'), captions=os.path.join(cur_dir, f'data/{dataset_type}/{captions}'), split='test', batch_size=batch_size, dataset_type=dataset_type) for i, (images, captions) in enumerate(data_loader): for image, caption in zip(images, captions): hashed = hashlib.sha1(image).hexdigest() if mode == 'text2image': with Document() as document: document.text = caption document.modality = 'text' document.mime_type = 'text/plain' with Document() as gt: match = Document() match.tags['id'] = hashed gt.matches.append(match) yield document, gt elif mode == 'image2text': with Document() as document: document.buffer = image document.modality = 'image' document.mime_type = 'image/jpeg' document.convert_buffer_to_uri() with Document() as gt: match = Document() match.tags['id'] = caption gt.matches.append(match) yield document, gt else: msg = f'Not supported mode {mode}.' msg += 'expected `image2text` or `text2image`' raise ValueError(msg) if num_docs and (i + 1) * batch_size >= num_docs: break
def test(model_filename, img_type, dir_name): print("Testing {}".format(model_filename)) model = torch.load(os.path.join(dir_name, model_filename)) base_dirs = ['KKI', 'NeuroIMAGE', 'OHSU', 'Peking_1', 'Peking_2', 'Peking_3', 'Pittsburgh','WashU'] model.eval() total_attempts = 0 total_correct = 0 res = collections.defaultdict(dict) for base_dir in base_dirs: d = dataset.get_data_loader(base_dir, img_type, batch_size=util.BATCH_SIZE, train=False) dataset_correct = 0 dataset_attempt = 0 for idx, (img, label, err) in enumerate(d): err_exists = sum(err) != 0 if err_exists: print("Error in loading {}".format(base_dir)) continue label = label.cuda() img = Variable(img.float()).cuda().contiguous() img = img.view(img.shape[0], 1, img.shape[1], img.shape[2], img.shape[3]) # print(img.shape) with torch.no_grad(): out = model(img) prediction = out.data.max(1)[1] correct = float(prediction.eq(label.data).sum()) dataset_attempt += len(label) dataset_correct += correct if dataset_attempt != 0: accuracy = (dataset_correct / float(dataset_attempt)) * 100.0 res[base_dir]['correct'] = dataset_correct res[base_dir]['attempt'] = dataset_attempt res[base_dir]['accuracy'] = accuracy total_correct += dataset_correct total_attempts += dataset_attempt print('Test Accuracy {}: {}/{}={}'.format(base_dir, dataset_correct, dataset_attempt, accuracy)) res['summary']['correct'] = total_correct res['summary']['attempt'] = total_attempts res['summary']['accuracy'] = float(total_correct)/total_attempts * 100.0 util.save_data(res, os.path.join(dir_name, 'test_result.pckl')) print('Total Accuracy: {}'.format(res['summary']['accuracy'])) return res
def main(args): data_loader = get_data_loader(args=args, data_path=args.data_path, bert_path=args.bert_path, num_workers=args.num_workers, batch_size=args.batch_size, split=args.split) model = MultimodalTransformer( n_layers=args.n_layers, n_heads=args.n_heads, n_classes=args.n_classes, only_audio=args.only_audio, only_text=args.only_text, d_audio_orig=args.n_mfcc, d_text_orig=768, # BERT hidden size d_model=args.d_model, attn_mask=args.attn_mask).to(args.device) model.load_state_dict(torch.load(args.model_path)) # evaluation logging.info('evaluation starts') model.zero_grad() evaluate(model, data_loader, args.device)
input_lengths = torch.full(size=(QUERY_NUM*CLASS_NUM,), fill_value=WINDOW_NUM, dtype=torch.long).to(device) target_lengths = torch.full(size=(QUERY_NUM*CLASS_NUM,), fill_value=1, dtype=torch.long).to(device) blank_prob = torch.full(size=(QUERY_NUM*CLASS_NUM, WINDOW_NUM, 1), fill_value=1, dtype=torch.float).to(device) # Training Loop train_ep = 0 while train_ep < args.train_ep: # Load Data if train_ep % args.load_frq == 0: try: if dataset_info["name"] != "finegym": the_dataset = dataset.StandardDataset(dataset_info["folders"], "train", dataset_info["split"], CLASS_NUM, INST_NUM, FRAME_NUM, CLIP_NUM, WINDOW_NUM) else: the_dataset = dataset.FinegymDataset(dataset_info["folder"], dataset_info["finegym_info"], "train", dataset_info["split"], CLASS_NUM, INST_NUM, FRAME_NUM, CLIP_NUM, WINDOW_NUM) dataloader = dataset.get_data_loader(the_dataset, num_per_class=SAMPLE_NUM+QUERY_NUM, num_workers=0) data, data_labels = dataloader.__iter__().next() # [class*(support+query), window*clip, RGB, frame, H, W] except Exception: continue data = data.view(-1, 3, FRAME_NUM, 128, 128) print("Train_Ep[{}] Current_Accuracy = {}".format(train_ep, max_accuracy), end="\t") # Generate support & query split query_index = [] support_index = [] for i in range(CLASS_NUM): start = (SAMPLE_NUM+QUERY_NUM) * i end = (SAMPLE_NUM+QUERY_NUM) * (i+1) index = list(range(start, end)) q = random.sample(index, QUERY_NUM)
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--train-set', type=str, dest="train_set", default="/store/slowmoyang/TopTagging/toptagging-training.root") parser.add_argument( '--valid-set', type=str, dest="valid_set", default="/store/slowmoyang/TopTagging/toptagging-validation.root") parser.add_argument( '--test-set', dest="test_set", default="/store/slowmoyang/TopTagging/toptagging-test.root", type=str) parser.add_argument('--batch-size', dest="batch_size", default=128, type=int, help='batch size') parser.add_argument('--test-batch-size', type=int, default=2048, dest="test_batch_size", help='batch size for test and validation') parser.add_argument('--epoch', dest="num_epochs", default=2, type=int, help='number of epochs to train for') parser.add_argument('--lr', default=0.005, type=float, help='learning rate, default=0.005') parser.add_argument("--logdir", dest="log_dir", default="./logs/untitled", type=str, help="the path to direactory") parser.add_argument("--verbose", default=True, type=bool) args = parser.parse_args() ##################################### # ###################################### log_dir = Directory(args.log_dir, create=True) log_dir.mkdir("state_dict") log_dir.mkdir("validation") log_dir.mkdir("roc_curve") ################################################## # Logger ################################################ logger = logging.getLogger("TopTagging") logger.setLevel(logging.INFO) format_str = '[%(asctime)s] %(message)s' date_format = '%Y-%m-%d %H:%M:%S' formatter = logging.Formatter(format_str, date_format) stream_handler = logging.StreamHandler(sys.stdout) stream_handler.setFormatter(formatter) logger.addHandler(stream_handler) log_file_path = log_dir.concat("log.txt") file_handler = logging.FileHandler(log_file_path) file_handler.setFormatter(formatter) logger.addHandler(file_handler) ############################### # #################################### device = torch.device("cuda:0") ###################################### # Data Loader ######################################## train_loader = get_data_loader(path=args.train_set, batch_size=args.batch_size) valid_loader = get_data_loader(path=args.valid_set, batch_size=args.test_batch_size) test_loader = get_data_loader(path=args.test_set, batch_size=args.test_batch_size) ##################### # Model ###################### model = Classifier() model.apply(init_weights) model.cuda(device) if args.verbose: logger.info(model) ################################## # Objective, optimizer, ################################## criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=args.lr) ################################ # Callbacks ################################ scheduler = ReduceLROnPlateau(optimizer, 'min', verbose=args.verbose) ######################################## # NOTE ####################################### for epoch in xrange(1, args.num_epochs + 1): logger.info("Epoch: [{:d}/{:d}]".format(epoch, args.num_epochs)) train(model=model, data_loader=train_loader, optimizer=optimizer, criterion=criterion, device=device, logger=logger) results = validate(model, valid_loader, device, logger) # Callbacks scheduler.step(results["loss"]) save_model(model, log_dir, epoch, results) good_states = find_good_state(log_dir.state_dict.path) for each in good_states: model.load_state_dict(torch.load(each)) # evaluate(model, test_loader, log_dir) logger.info("END")
default='shufflenetv2x0.5', help='shufflenetv2x0.5 supported') parser.add_argument('--load-model', type=str, default='None', help='the path of the model to be loaded') parser.add_argument('--preprocess', type=str, default='randaffine') args = parser.parse_args() if __name__ == '__main__': batch_size = args.batch_size preprocess = args.preprocess device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") testloader = get_data_loader(load_set='val', preprocess=preprocess, batch_size=batch_size) model = get_model(model_name=args.model, load_model=args.load_model, batch_size=batch_size, device=device) best_acc = 0 t = datetime.datetime.now().strftime("%Y%m%d_%H%M") model_name = args.model + '_' + t + '.pth' print('model_name: ', model_name) print('Number of parameters: {}'.format( sum([p.numel() for p in model.parameters()]))) print('testing') # test
def main(num_epochs=10, embedding_dim=256, data_dir="data/"): """ Function to train the model. Args: num_epochs: int Number of full dataset iterations to train the model. embedding_dim: int Output of the CNN model and input of the LSTM embedding size. data_dir: str Path to the folder of the data. """ device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") print(f"WORKING WITH: {device}") # Define the paths for train and validation train_json_path = data_dir + "annotations/captions_train2014.json" train_root_dir = data_dir + "train2014" valid_json_path = data_dir + "annotations/captions_val2014.json" valid_root_dir = data_dir + "val2014" transform = transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) train_dataset = CocoDataset(json_path=train_json_path, root_dir=train_root_dir, transform=transform) train_coco_dataset = get_data_loader(train_dataset, batch_size=128) valid_dataset = CocoDataset(json_path=valid_json_path, root_dir=valid_root_dir, transform=transform) valid_coco_dataset = get_data_loader(valid_dataset, batch_size=1) encoder = FeatureExtractor(embedding_dim).to(device) decoder = CaptionGenerator(embedding_dim, 512, len(train_dataset.vocabulary), 1).to(device) criterion = nn.CrossEntropyLoss() # params = list(decoder.parameters()) + list(encoder.linear.parameters()) + list(encoder.bn.parameters()) params = list(decoder.parameters()) + list( encoder.linear.parameters()) + list(encoder.bn.parameters()) optimizer = optim.Adam(params, lr=0.01) print(f"TRAIN DATASET: {len(train_coco_dataset)}") print(f"VALID DATASET: {len(valid_coco_dataset)}") total_step = len(train_coco_dataset) for epoch in range(num_epochs): encoder.train() decoder.train() train_loss = 0.0 valid_loss = 0.0 for i, (images, captions, descriptions) in enumerate(train_coco_dataset): # targets = pack_padded_sequence(caption, 0, batch_first=True)[0] images = images.to(device) captions = captions.to(device) # targets = pack_padded_sequence(captions, lengths, batch_first=True)[0] features = encoder(images) outputs = decoder(features, captions) loss = criterion(outputs.view(-1, len(train_dataset.vocabulary)), captions.view(-1)) # bleu = calculate_bleu(decoder, features, descriptions, coco_dataset) # print(bleu) encoder.zero_grad() decoder.zero_grad() loss.backward() optimizer.step() # Print log info train_loss += loss.item() ''' if i % 10 == 0: print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}, Perplexity: {:5.4f}' .format(epoch, num_epochs, i, total_step, loss.item(), np.exp(loss.item()))) ''' # Save the model checkpoints if (i + 1) % 1000 == 0: torch.save( decoder.state_dict(), os.path.join("models", 'decoder-{}-{}.ckpt'.format(epoch + 1, i + 1))) torch.save( encoder.state_dict(), os.path.join("models", 'encoder-{}-{}.ckpt'.format(epoch + 1, i + 1))) encoder.eval() decoder.eval() bleu = 0.0 for i, (images, captions, descriptions) in enumerate(valid_coco_dataset): if (i > 80000): break images = images.to(device) captions = captions.to(device) features = encoder(images) outputs = decoder(features, captions) loss = criterion(outputs.view(-1, len(train_dataset.vocabulary)), captions.view(-1)) valid_loss += loss.item() bleu += calculate_bleu(decoder, features, descriptions, train_coco_dataset) # print(f"BLEU: {bleu / 10000}") print( "Epoch: {}, Train Loss: {:.4f}, Valid Loss: {:.4f}, BLEU: {:.4f}". format(epoch, train_loss / len(train_coco_dataset), valid_loss / 80000, bleu / 80000))
def run(optimizer_type, img_type, lr = 0.005, large=False): if large: _model = model.StructuralModel3DFullImageLarge() _model_dir = 'model_large' else: _model = model.StructuralModel3DFullImage() _model_dir = 'model' _model = _model.cuda() if optimizer_type == 'adam': optimizer = optim.Adam(_model.parameters(), lr=lr) save_dir = '{}/{}/{}/{}'.format(_model_dir, str(img_type), optimizer_type, lr) util.mkdir(save_dir) base_dirs = ['KKI', 'NeuroIMAGE', 'OHSU', 'Peking_1', 'Peking_2', 'Peking_3', 'Pittsburgh','WashU'] criterion = nn.CrossEntropyLoss() count = 0 begin_time = time.time() train_acc_dict = {} for epoch in range(0, num_of_epochs): ###### TRAIN train_accu = [] train_loss = [] _model.train() start_time = time.time() print("Epoch: {}".format(epoch)) random.shuffle(base_dirs) for base_dir in base_dirs: print('Using {}'.format(base_dir)) d = dataset.get_data_loader(base_dir, img_type, batch_size=util.BATCH_SIZE, train=True) if optimizer_type == 'adam': for group in optimizer.param_groups: for p in group['params']: state = optimizer.state[p] if ('step' in state and state['step'] >= 1024): state['step'] = 1000 num_correct= 0 num_total = 0 for idx, (img, label, err) in enumerate(d): if(sum(err) != 0): print("Error occured") continue # ipdb.set_trace() label = label.cuda() img = Variable(img.float()).cuda().contiguous() img = img.view(img.shape[0], 1, img.shape[1], img.shape[2], img.shape[3]) # print(img.shape) out = _model(img) # print(out) optimizer.zero_grad() loss = criterion(out, label) loss.backward() optimizer.step() prediction = out.data.max(1)[1] correct = float(prediction.eq(label.data).sum()) num_correct += correct num_total += len(label) accuracy = (correct / float(len(label))) * 100.0 train_loss.append(loss.item()) train_accu.append(accuracy) # if count % 100 == 0: print("Epoch {} iter {}: Training accuracy = {}/{}={} Loss = [{}]".format(epoch, idx, correct, len(label), accuracy, loss.item())) count+= 1 if num_total != 0: base_train_acc = float(num_correct)/num_total * 100.0 print("Epoch {} {}: Training accuracy = {}/{}={}".format(epoch, base_dir, num_correct, num_total, base_train_acc)) train_acc_dict[base_dir] = base_train_acc torch.save(_model, './{}/{}'.format(save_dir, '{}.ckpt'.format(epoch))) train_accu = np.asarray(train_accu) train_loss = np.asarray(train_loss) np.save(file=os.path.join(save_dir, 'loss.npy'), arr=train_loss) np.save(file=os.path.join(save_dir, 'train_acc.npy'), arr=train_accu) print("Ran for {}s".format(time.time()-begin_time)) util.save_data(filename=os.path.join(save_dir, 'train_dict.pckl'), d=train_acc_dict)
help='shufflenetv2x0.5 supported') parser.add_argument('--load-model', type=str, default='None', help='the path of the model to be loaded') parser.add_argument('--preprocess', type=str, default='randaffine') args = parser.parse_args() if __name__ == '__main__': batch_size = args.batch_size preprocess = args.preprocess device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") trainloader = get_data_loader(load_set='train', preprocess=preprocess, batch_size=batch_size) valloader = get_data_loader(load_set='val', preprocess=preprocess, batch_size=batch_size) model = get_model(model_name=args.model, load_model=args.load_model, batch_size=batch_size, device=device) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=0.9) best_acc = 0 t = datetime.datetime.now().strftime("%Y%m%d_%H%M") model_name = args.model + '_' + t + '.pth'
def train(dataset_path: str): device = torch.device(config["train"]["device"]) print("Device: {}".format(device)) # device = torch.device("cpu") # gpu not enough memory :( model = OpenAIGPTDoubleHeadsModel.from_pretrained("openai-gpt") model.to(device) tokenizer = OpenAIGPTTokenizer.from_pretrained("openai-gpt") orig_num_tokens = len(tokenizer.encoder) num_added_tokens = tokenizer.add_special_tokens(SPECIAL_TOKENS) model.resize_token_embeddings(new_num_tokens=orig_num_tokens + num_added_tokens) # dataloader = get_data_loader(dataset_path, tokenizer, batch_size=4, shuffle=False, num_workers=1) full_dataset = get_dataset(dataset_path, tokenizer) assert len(full_dataset) > 0 train_size = int( len(full_dataset) * config["train"]["train_dataset_proportion"] + 1) test_size = len(full_dataset) - train_size print("Full dataset has {} dialogs. Splitting into train: {} and test: {}". format(len(full_dataset), train_size, test_size)) train_dataset, test_dataset = random_split( full_dataset, [train_size, test_size], torch.Generator().manual_seed(42)) print(len(train_dataset), len(test_dataset)) train_loader = get_data_loader(train_dataset, tokenizer, config["train"]["batch_size"], True, 0) test_loader = get_data_loader(test_dataset, tokenizer, 1, False, 0) lr = config["train"]["learning_rate"] print("lr: {}".format(lr)) optimizer = AdamW(model.parameters(), lr=lr) # init logging start_time = datetime.datetime.now() save_path = os.path.join( os.path.dirname(__file__), "log/log-{}.txt".format(start_time.strftime("%y-%m-%d-%H-%M-%S"))) print(os.path.dirname(__file__), save_path) f = open(save_path, "w+") f.close() epochs = config["train"]["num_epochs"] eval_every = config["train"]["evaluate_interval_iters"] num_tests = config["train"]["num_tests"] last_model_save = datetime.datetime.now() iteration = 0 for epoch in range(epochs): print("Starting epoch {}/{}".format(epoch, epochs)) for batch in train_loader: if iteration % eval_every == 0: results = evaluate_model(model, test_loader, device, num_tests) add_log( save_path, "test,{0},{1},{2[mc_correct]},{2[num_tests]},{2[lm_correct]},{2[lm_tested]}\n" .format(iteration, epoch, results)) model.train() input_ids = batch["input_ids"].to(device) mc_token_ids = batch["mc_token_ids"].to(device) token_type_ids = batch["token_type_ids"].to(device) lm_labels = batch["lm_labels"].to(device) mc_labels = batch["correct"].to(device) try: model_output = model(input_ids, token_type_ids=token_type_ids, mc_token_ids=mc_token_ids, mc_labels=mc_labels, labels=lm_labels) except Exception as e: print(input_ids, token_type_ids, mc_token_ids, lm_labels, mc_labels, sep="\n") raise e # print("input_ids: {}\ntoken_type_ids: {}\nmc_token_ids: {}\nlm_labels: {}\nmc_labels: {}" # .format(input_ids, token_type_ids, mc_token_ids, lm_labels, mc_labels)) # print(model_output.loss.item(), model_output.mc_loss.item()) lm_loss = model_output.loss mc_loss = model_output.mc_loss loss = lm_loss * config["train"]["lm_coeff"] + mc_loss * config[ "train"]["mc_coeff"] add_log( save_path, "train,{},{},{},{},{}\n".format(iteration, epoch, loss, lm_loss, mc_loss)) loss.backward() nn.utils.clip_grad_norm_(model.parameters(), config["train"]["max_norm"]) optimizer.step() optimizer.zero_grad() # TODO: evaluation if iteration % 50 == 0: print( "Time: {} Epoch: {}/{} Iteration: {}/{} Loss: {} ({} {})". format( datetime.datetime.now() - start_time, epoch, epochs, iteration, epochs * (len(train_dataset) // config["train"]["batch_size"]), loss.item(), lm_loss.item(), mc_loss.item())) if datetime.datetime.now() - last_model_save > datetime.timedelta( minutes=config["train"]["save_interval_mins"]): print("Saving model...") torch.save( model.state_dict(), os.path.join(os.path.dirname(__file__), "checkpoints/model-{}-iter{}.pt").format( start_time.strftime("%y-%m-%d-%H-%M-%S"), iteration)) last_model_save = datetime.datetime.now() iteration += 1 print("Saving model...") torch.save( model.state_dict(), os.path.join(os.path.dirname(__file__), "checkpoints/model-{}-iter{}.pt").format( start_time.strftime("%y-%m-%d-%H-%M-%S"), iteration))
test_ep = 0 while test_ep < args.test_ep: # Data Loading try: if dataset_info["name"] != "finegym": the_dataset = dataset.StandardDataset( dataset_info["folders"], "test", dataset_info["split"], CLASS_NUM, INST_NUM, FRAME_NUM, CLIP_NUM, WINDOW_NUM) else: the_dataset = dataset.FinegymDataset( dataset_info["folder"], dataset_info["finegym_info"], "test", dataset_info["split"], CLASS_NUM, INST_NUM, FRAME_NUM, CLIP_NUM, WINDOW_NUM) dataloader = dataset.get_data_loader(the_dataset, num_per_class=SAMPLE_NUM + QUERY_NUM, num_workers=0) data, data_labels = dataloader.__iter__().next( ) # [class*(support+query), window*clip, RGB, frame, H, W] except Exception: continue data = data.view(-1, 3, FRAME_NUM, 128, 128) print("Test_Epi[{}]".format(test_ep), end="\t") # Generate support & query split query_index = [] support_index = [] for i in range(CLASS_NUM): start = (SAMPLE_NUM + QUERY_NUM) * i end = (SAMPLE_NUM + QUERY_NUM) * (i + 1)
def train_model(args): bn_training = tf.placeholder(dtype=tf.bool, shape=[], name='bn_training') x = tf.placeholder(dtype=tf.float32, shape=[None, 3, 32, 32], name='x') y = tf.placeholder(dtype=tf.float32, shape=[None, args.num_classes], name='y') weight_decay = args.weight_decay with tf.name_scope('resnet18'): pred = resnet18(x, args.num_classes, bn_training) with tf.variable_scope('train'): global_step = tf.Variable(0, name='global_step', trainable=False) learning_rate = tf.placeholder(dtype=tf.float32, shape=[], name='learning_rate') cross_entropy_loss = tf.reduce_mean( slim.nn.softmax_cross_entropy_with_logits_v2(labels=y, logits=pred)) weight_decay_loss = weight_decay * tf.add_n([ tf.nn.l2_loss(tf.cast(v, tf.float32)) for v in tf.trainable_variables() if weight_decay_param_filter(v.name) ]) loss = cross_entropy_loss + weight_decay_loss train_optimizer = slim.train.MomentumOptimizer( learning_rate=learning_rate, momentum=0.9) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): train_op = train_optimizer.minimize(loss, global_step) tf.summary.scalar('learning_rate', learning_rate) tf.summary.scalar('loss', loss) with tf.variable_scope('test'): correct = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct, tf.float32)) tf.summary.scalar('accuracy', accuracy) config = tf.ConfigProto() config.gpu_options.allow_growth = True train_loader, test_loader = get_data_loader(args.data_dir, batch_size=args.batch_size, num_workers=args.num_workers) base_lr = args.lr best_acc = 0.0 merged = tf.summary.merge_all() i = 0 accuracy_list = [] with tf.Session(config=config) as sess: summary_writer = tf.summary.FileWriter(args.log_dir, sess.graph) sess.run(tf.global_variables_initializer()) pbar = tqdm.tqdm(range(args.epoch)) for epoch in pbar: lr = get_learning_rate(base_lr, epoch) _losses = [] for idx, batch_data in enumerate(train_loader): i += 1 images, labels = batch_data _loss, _, rs = sess.run([loss, train_op, merged], feed_dict={ x: images, y: labels, learning_rate: lr, bn_training: True }) _losses.append(_loss) summary_writer.add_summary(rs, i) _test_accuracy = [] for _, batch_data in enumerate(test_loader): images, labels = batch_data acc = sess.run(accuracy, feed_dict={ x: images, y: labels, bn_training: True }) _test_accuracy.append(acc) cur_acc = 100 * np.mean(_test_accuracy) accuracy_list.append(cur_acc) best_acc = max(cur_acc, best_acc) pbar.set_description( "e:{} loss:{:.3f} acc:{:.2f}% best:{:.2f}% lr:{:.5f}".format( epoch, np.mean(_losses), cur_acc, best_acc, lr)) with open(os.path.join(args.log_dir, 'acc.pkl'), 'wb') as f: pickle.dump(accuracy_list, f)
def main(arg): ################################## for key in arg: parse[key] = arg[key] global args args = SimpleNamespace(**parse) ''' print('seed:{:}'.format(args.seed)) print('dataset:{:}'.format(args.dataset)) print('hidden_layers:{:}'.format(args.hidden_layers)) print('first_neurons:{:}'.format(args.first_neurons)) print('cross_link:{:}'.format(args.cross_link)) print('fully_cross:{:}'.format(args.fully_cross)) print() exit(0) ''' ################################## seed = util.prepare(args) if not cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(seed) random.seed(seed) torch.manual_seed(seed) cuda.manual_seed(seed) cuda.set_device(args.gpu) cudnn.benchmark = False cudnn.deterministic = True logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) logging.info('hidden_layers:{:}'.format(args.hidden_layers)) logging.info('first_neurons:{:}'.format(args.first_neurons)) logging.info('change:{:}'.format(args.change)) logging.info('activate_func:{:}'.format(args.activate_func)) logging.info('opt:{:}'.format(args.opt)) logging.info('cross_link:{:}'.format(args.cross_link)) logging.info('fully_cross:{:}'.format(args.fully_cross)) model = Network(args) model = model.cuda() logging.info("param size = %fMB", util.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) # scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, args.epochs) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.7) train_data, valid_data = dataset.get_dataset(args.data, args.dataset) train_queue, valid_queue = dataset.get_data_loader(train_data, valid_data, 2) early_stop = util.EarlyStop(patience=10, delta=0.0001, save_path=args.save + '/best.pt') for epoch in range(args.epochs): logging.info('epoch %d lr %.6f', epoch, scheduler.get_lr()[0]) epoch_str = '[{:03d}/{:03d}]'.format(epoch, args.epochs) train_acc, train_obj = train(train_queue, model, criterion, optimizer, epoch_str) logging.info('train_acc %.2f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion, epoch_str) logging.info('valid_acc %.2f', valid_acc) if early_stop.check(train_obj, valid_acc, model): logging.info('Early stopping at {:}'.format(epoch)) break scheduler.step()
decay_lr = args.decay_lr decay_equilibrium = args.decay_equilibrium slurm = args.slurm writer = SummaryWriter(comment="_CELEBA_ALL") device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") net = VaeGan(z_size=z_size, recon_level=recon_level).to(device) net = nn.DataParallel(net) # DATASET # dataloader = torch.utils.data.DataLoader(CELEBA(train_folder), batch_size=64, # shuffle=True, num_workers=4) # dataloader_test = torch.utils.data.DataLoader(CELEBA(test_folder), batch_size=100, # shuffle=False, num_workers=1) dataloader, dataloader_test = get_data_loader(img_size, train_folder, test_folder, n_batch_train, n_batch_test, num_workers_train, num_workers_test) #margin and equilibirum margin = 0.35 equilibrium = 0.68 #mse_lambda = 1.0 # OPTIM-LOSS # an optimizer for each of the sub-networks, so we can selectively backprop optimizer_encoder = RMSprop(params=net.module.encoder.parameters(), lr=lr, alpha=0.9, eps=1e-8, weight_decay=0, momentum=0, centered=False)
parser.add_argument('--resume_training', type=str2bool, default=False) parser.add_argument('--to_train', type=str2bool, default=True) parser.add_argument('--conditional', type=str2bool, default=False) opt = parser.parse_args() print(opt) manual_seed = random.randint(1, 10000) random.seed(manual_seed) T.manual_seed(manual_seed) if T.cuda.is_available(): T.cuda.manual_seed_all(manual_seed) train_loader = get_data_loader(opt, True) test_loader = get_data_loader(opt, False) E = get_cuda(Encoder(opt)) G = get_cuda(Generator(opt)).apply(weights_init) D = get_cuda(Discriminator(opt)).apply(weights_init) device_ids = range(T.cuda.device_count()) E = nn.DataParallel(E, device_ids) G = nn.DataParallel(G, device_ids) D = nn.DataParallel(D, device_ids) E_trainer = T.optim.Adam(E.parameters(), lr=opt.lr_e) G_trainer = T.optim.Adam(G.parameters(), lr=opt.lr_g, betas=(0.5, 0.999)) D_trainer = T.optim.Adam(D.parameters(), lr=opt.lr_d, betas=(0.5, 0.999))
train_epoch = 80 resume_training = False z_size = 512 def load_model_from_checkpoint(): global ae_model checkpoint = torch.load(save_path) vae.load_state_dict(checkpoint['ae_model']) return checkpoint['epoch'] # load dataSet train_loader = get_data_loader(img_size=im_size, img_path=img_path, shuffle=True, batch_size=batch_size, num_workers=8) loss_function = pytorch_ssim.SSIM() def main(): #build Neural Network ae_model = ResNet_autoencoder(Bottleneck, DeconvBottleneck, [3, 4, 6, 3], 3) ae_model.apply(weights_init_kaiming) if torch.cuda.is_available(): torch.backends.cudnn.benchmark = True ae_model.cuda() ae_model.train()
def main(): args = parser.parse_args() # define args more args.train_meta = './meta/CARS196/train.txt' args.test_meta = './meta/CARS196/test.txt' args.lr_decay_epochs = [ int(epoch) for epoch in args.lr_decay_epochs.split(',') ] args.recallk = [int(k) for k in args.recallk.split(',')] os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu_idx) args.ctx = [mx.gpu(0)] print(args) # Set random seed mx.random.seed(args.seed) np.random.seed(args.seed) random.seed(args.seed) # Load image transform train_transform, test_transform = T.get_transform( image_size=args.image_size) # Load data loader train_loader, test_loader = D.get_data_loader( args.data_dir, args.train_meta, args.test_meta, train_transform, test_transform, args.batch_size, args.num_instances, args.num_workers) # Load model model = Model(args.embed_dim, args.ctx) model.hybridize() # Load loss loss = HPHNTripletLoss(margin=args.margin, soft_margin=False, num_instances=args.num_instances, n_inner_pts=args.n_inner_pts, l2_norm=args.ee_l2norm) # Load logger and saver summary_writer = SummaryWriter( os.path.join(args.save_dir, 'tensorboard_log')) print("steps in epoch:", args.lr_decay_epochs) steps = list(map(lambda x: x * len(train_loader), args.lr_decay_epochs)) print("steps in iter:", steps) lr_schedule = mx.lr_scheduler.MultiFactorScheduler( step=steps, factor=args.lr_decay_factor) lr_schedule.base_lr = args.lr # Load optimizer for training optimizer = mx.gluon.Trainer(model.collect_params(), 'adam', { 'learning_rate': args.lr, 'wd': args.wd }, kvstore=args.kvstore) # Load trainer & evaluator trainer = Trainer(model, loss, optimizer, train_loader, summary_writer, args.ctx, summary_step=args.summary_step, lr_schedule=lr_schedule) evaluator = Evaluator(model, test_loader, args.ctx) best_metrics = [0.0] # all query global_step = args.start_epoch * len(train_loader) # Enter to training loop print("base lr mult:", args.base_lr_mult) for epoch in range(args.start_epoch, args.epochs): model.backbone.collect_params().setattr('lr_mult', args.base_lr_mult) trainer.train(epoch) global_step = (epoch + 1) * len(train_loader) if (epoch + 1) % args.eval_epoch_term == 0: old_best_metric = best_metrics[0] # evaluate_and_log(summary_writer, evaluator, ranks, step, epoch, best_metrics) best_metrics = evaluate_and_log(summary_writer, evaluator, args.recallk, global_step, epoch + 1, best_metrics=best_metrics) if best_metrics[0] != old_best_metric: save_path = os.path.join( args.save_dir, 'model_epoch_%05d.params' % (epoch + 1)) model.save_parameters(save_path) sys.stdout.flush()
return False parser.add_argument('--resume_training', type=str2bool, default=False) parser.add_argument('--to_train', type=str2bool, default=True) opt = parser.parse_args() print(opt) manual_seed = random.randint(1, 10000) random.seed(manual_seed) T.manual_seed(manual_seed) if T.cuda.is_available(): T.cuda.manual_seed_all(manual_seed) train_loader = get_data_loader(opt) E = get_cuda(Encoder(opt)) G = get_cuda(Generator(opt)).apply(weights_init) D = get_cuda(Discriminator()).apply(weights_init) device_ids = range(T.cuda.device_count()) E = nn.DataParallel(E, device_ids) G = nn.DataParallel(G, device_ids) D = nn.DataParallel(D, device_ids) E_trainer = T.optim.Adam(E.parameters(), lr=opt.lr_e) G_trainer = T.optim.Adam(G.parameters(), lr=opt.lr_g, betas=(0.5, 0.999)) D_trainer = T.optim.Adam(D.parameters(), lr=opt.lr_d, betas=(0.5, 0.999))