def main(args): #load best init model model_path = os.path.join(args.output_dir, "models", f'best_{args.emotion}_init_model.pkl') model=torch.load(model_path) args.output_dir = args.output_dir + "_train_trainedInitialmodel" os.makedirs(os.path.join(args.output_dir,"logs"), exist_ok=True) os.makedirs(os.path.join(args.output_dir, "models"), exist_ok=True) os.makedirs(os.path.join(args.output_dir, "gradient_similarities"), exist_ok=True) logging.basicConfig(filename=os.path.join(args.output_dir,"logs","output.log"), level=logging.DEBUG, format='%(asctime)s %(message)s') # Reproducibility torch.manual_seed(args.seed) torch.backends.cudnn.deterministic = True # Load metalearning data tokenizer = RobertaTokenizer.from_pretrained(args.bert_name) train_data, val_data, test_data = load_emotion_data("meta_all", args.seed, args.balance_datasets) # Determine number of classes n_classes=[] emotions_all = args.emotion.split("&") train_data_loader = {} val_data_loader = {} test_data_loader = {} train_data_loader_list = [] for emo in emotions_all: c_num = train_data[train_data['task'] == emo]['emotion_ind'].value_counts() n_classes.append(len(c_num)) train_data_task = train_data[train_data['task'] == emo] val_data_task = val_data[val_data['task'] == emo] test_data_task = test_data[test_data['task'] == emo] train_data_loader[emo] = create_data_loader(train_data_task, tokenizer, args.max_len, args.batch_size, args) val_data_loader[emo] = create_data_loader(val_data_task, tokenizer, args.max_len, args.batch_size, args) test_data_loader[emo]= create_data_loader(test_data_task, tokenizer, args.max_len, args.batch_size, args) train_data_loader_list.append(train_data_loader[emo]) print(f'n_classes = {n_classes}') # Define loss function loss_fn = nn.CrossEntropyLoss() # Start training train(model, train_data_loader, val_data_loader, args, loss_fn, emotions_all) # test model test(test_data_loader, args, loss_fn)
def infer(args): print(args) device = paddle.set_device(args.device) _, _, _, vocab, bos_id, eos_id, _ = create_data_loader(args) net = VAESeq2SeqInferModel(args.embed_dim, args.hidden_size, args.latent_size, len(vocab) + 2) model = paddle.Model(net) model.prepare() model.load(args.init_from_ckpt) infer_output = paddle.ones((args.batch_size, 1), dtype='int64') * bos_id space_token = ' ' line_token = '\n' with io.open(args.infer_output_file, 'w', encoding='utf-8') as out_file: predict_lines = model.predict_batch(infer_output)[0] for line in predict_lines: end_id = -1 if eos_id in line: end_id = np.where(line == eos_id)[0][0] new_line = [vocab.to_tokens(e[0]) for e in line[:end_id]] out_file.write(space_token.join(new_line)) out_file.write(line_token)
def train(args): print(args) device = paddle.set_device(args.device) train_loader, dev_loader, test_loader, vocab, bos_id, pad_id, train_data_len = create_data_loader( args) net = VAESeq2SeqModel(embed_dim=args.embed_dim, hidden_size=args.hidden_size, latent_size=args.latent_size, vocab_size=len(vocab) + 2, num_layers=args.num_layers, init_scale=args.init_scale, enc_dropout=args.enc_dropout, dec_dropout=args.dec_dropout) gloabl_norm_clip = paddle.nn.ClipGradByGlobalNorm(args.max_grad_norm) anneal_r = 1.0 / (args.warm_up * train_data_len / args.batch_size) cross_entropy = CrossEntropyWithKL(base_kl_weight=args.kl_start, anneal_r=anneal_r) model = paddle.Model(net) optimizer = paddle.optimizer.Adam(args.learning_rate, parameters=model.parameters(), grad_clip=gloabl_norm_clip) if args.init_from_ckpt: model.load(args.init_from_ckpt) print("Loaded checkpoint from %s" % args.init_from_ckpt) ppl_metric = Perplexity(loss=cross_entropy) nll_metric = NegativeLogLoss(loss=cross_entropy) model.prepare(optimizer=optimizer, loss=cross_entropy, metrics=[ppl_metric, nll_metric]) model.fit(train_data=train_loader, eval_data=dev_loader, epochs=args.max_epoch, save_dir=args.model_path, shuffle=False, callbacks=[TrainCallback(ppl_metric, nll_metric, args.log_freq)], log_freq=args.log_freq) # Evaluation print('Start to evaluate on test dataset...') model.evaluate(test_loader, log_freq=len(test_loader))
def main(): experiment_name = "Resnet_DY_05" epoch = 100 experiment_path = os.path.join("experiments", experiment_name) opt = Options(suppress_parse=True) opt.load_from_file(os.path.join(experiment_path, "config.yaml")) opt.checkpoint_path = os.path.join(experiment_path, f"{experiment_name}_{epoch}.pth") assert opt.use_dynamic model = get_inspect_model(opt) test_dl = data.create_data_loader(opt, "test") test_score = test(model, opt.temperature[1], test_dl, opt.device) entropy = compute_entropy(attentions_register) print(test_score) print(entropy)
def main(args): args.output_dir = os.path.join(args.output_dir, args.emotion + "_MAML") os.makedirs(os.path.join(args.output_dir, "logs"), exist_ok=True) os.makedirs(os.path.join(args.output_dir, "models"), exist_ok=True) logging.basicConfig(filename=os.path.join(args.output_dir, "logs", "output.log"), level=logging.DEBUG, format='%(asctime)s %(message)s') # Reproducibility torch.manual_seed(args.seed) torch.backends.cudnn.deterministic = True # Load metalearning data tokenizer = RobertaTokenizer.from_pretrained(args.bert_name) train_data, val_data, test_data = load_emotion_data("meta_all", args.seed) # Determine number of classes n_classes = [] emotions_all = args.emotion.split("&") train_data_loader = {} val_data_loader = {} test_data_loader = {} for emo in emotions_all: c_num = train_data[train_data['task'] == emo]['emotion_ind'].value_counts() n_classes.append(len(c_num)) train_data_task = train_data[train_data['task'] == emo] val_data_task = val_data[val_data['task'] == emo] test_data_task = test_data[test_data['task'] == emo] train_data_loader[emo] = create_data_loader(train_data_task, tokenizer, args.max_len, args.batch_size, args) val_data_loader[emo] = create_data_loader(val_data_task, tokenizer, args.max_len, args.batch_size, args) test_data_loader[emo] = create_data_loader(test_data_task, tokenizer, args.max_len, args.batch_size, args) print(f'n_classes = {n_classes}') # Load metalearning data # tokenizer = RobertaTokenizer.from_pretrained(args.bert_name) # train_data, val_data, test_data = load_emotion_data("meta_all", args.seed) # train_data_loader = creat_metadataLoader(train_data, tokenizer, args.max_len, args.tasks_selected, args.num_task_eachtime, args.num_sample_pertask, args) #build model classifier = EmoClassifier_MulTask(args.bert_name, n_classes, emotions_all) MAML_Classifier_model = MAML_multiTask_framework(args, classifier) MAML_Classifier_model.to(args.device) # optimizer criterion = nn.CrossEntropyLoss() #train train(MAML_Classifier_model, train_data_loader, val_data_loader, args, criterion, emotions_all) #test MAML_Classifier_model.test_classifier(test_data_loader, args, criterion)
z_size = 3 writer = SummaryWriter( '/home/test2/Documents/Data/post_processing/weights_sep_traj/120319/GAN/runs/exp18' ) #Instantiate dataset #DMP dataset ##data_loc='/home/test2/Documents/Data/post_processing/weights_sep_traj/120319/combinedWts_transformedRefFrame/XYZABG_weights_separated_by_cut_new_generated_data/generated_data_combined_weights_all_6dims_with_initial_orientations_num_labels.txt' #data_loc='/home/test2/Documents/Data/post_processing/weights_sep_traj/120319/combinedWts_transformedRefFrame/XYZABG_weights_combined/combined_weights_all_6dims_with_initial_orientations_num_labels.txt' data_loc = '/home/test2/Documents/Data/post_processing/weights_sep_traj/120319/combinedWts_transformedRefFrame/XYZABG_weights_combined/combined_wts_xyz_dims.txt' #data_loc='/home/test2/Documents/Data/post_processing/weights_sep_traj/120319/combinedWts_transformedRefFrame/XYZABG_weights_combined/combined_weights_6dims_only_2_cuts.txt' dataset = DMPWeightData(data_loc, num_dims, num_labels, MNIST=False) batch_size = 39 #156 data_loader = create_data_loader(dataset, batch_size) #Create instances of generator and discrim classes #DMP dataset generator = Generator(num_labels, num_dims, z_size) discriminator = Discriminator(num_labels, num_dims) #Train num_epochs = 5000 n_critic = 5 display_step = 300 loss = nn.BCELoss() d_optimizer = torch.optim.Adam(discriminator.parameters(), lr=0.0003) g_optimizer = torch.optim.Adam(generator.parameters(), lr=0.0001) gen_loss = []
#underfit => 20201001_143458 (fonctionne bien) #overfit => 0200930_111537 MODEL_PATH = "/media/nas/samir-data/CamemBertPunc/models/20201001_143458/model" checkpoint = torch.load(MODEL_PATH, map_location=lambda storage, loc: storage) from collections import OrderedDict new_checkpoint = OrderedDict() for k, v in checkpoint.items(): new_k = k[7:] new_checkpoint[new_k] = v #load params bert_punc.load_state_dict(new_checkpoint) batch_size = 64 data_loader_test = create_data_loader(X_test, y_test, False, batch_size) #data_loader_test_asr = create_data_loader(X_test_asr, y_test_asr, False, batch_size) def predictions(data_loader): y_pred = [] y_true = [] for inputs, attentions, labels in tqdm(data_loader, total=len(data_loader)): with torch.no_grad(): #inputs, labels = inputs.cuda(), labels.cuda() output = bert_punc(inputs, attentions) #print("output0 : ", output[0]) #print("output1 : ", output[1]) #print("output2 : ", output[2]) #print("shape output : ", output.shape)
tokenizer = CamembertTokenizer.from_pretrained('camembert-base') print('PREPROCESSING DATA...') X_train, y_train = encode_data3(data_train, tokenizer, puncs, punctuation_enc, segment_size) X_valid, y_valid = encode_data3(data_valid, tokenizer, puncs, punctuation_enc, segment_size) print('INITIALIZING MODEL...') output_size = len(punctuation_enc) #bert_punc = BertPunc_ner(segment_size, output_size).cuda() bert_punc = nn.DataParallel(BertPunc_ner(segment_size, output_size).cuda()) print('TRAINING TOP LAYER...') data_loader_train = create_data_loader(X_train, y_train, True, batch_size_top) data_loader_valid = create_data_loader(X_valid, y_valid, False, batch_size_top) for name, param in bert_punc.named_parameters(): if 'classifier' not in name: # classifier layer param.requires_grad = False else: param.requires_grad = True optimizer = optim.AdamW(bert_punc.parameters(), lr=learning_rate_top) criterion = nn.CrossEntropyLoss() bert_punc, optimizer, best_val_loss = train(bert_punc, optimizer, criterion, epochs_top, data_loader_train, data_loader_valid,
def train(args): paddle.set_device(args.device) set_seed(102) trainer_num = paddle.distributed.get_world_size() if trainer_num > 1: paddle.distributed.init_parallel_env() rank = paddle.distributed.get_rank() word_vocab, label_vocab, train_loader, test_loader = create_data_loader( args) # Define the model netword and its loss model = BiGruCrf(args.emb_dim, args.hidden_size, len(word_vocab), len(label_vocab), crf_lr=args.crf_lr) # Prepare optimizer, loss and metric evaluator optimizer = paddle.optimizer.Adam(learning_rate=args.base_lr, parameters=model.parameters()) chunk_evaluator = ChunkEvaluator(label_list=label_vocab.keys(), suffix=True) if args.init_checkpoint: if os.path.exists(args.init_checkpoint): logger.info("Init checkpoint from %s" % args.init_checkpoint) model_dict = paddle.load(args.init_checkpoint) model.load_dict(model_dict) else: logger.info("Cannot init checkpoint from %s which doesn't exist" % args.init_checkpoint) logger.info("Start training") # Start training global_step = 0 last_step = args.epochs * len(train_loader) train_reader_cost = 0.0 train_run_cost = 0.0 total_samples = 0 reader_start = time.time() max_f1_score = -1 for epoch in range(args.epochs): for step, batch in enumerate(train_loader): train_reader_cost += time.time() - reader_start global_step += 1 token_ids, length, label_ids = batch train_start = time.time() loss = model(token_ids, length, label_ids) avg_loss = paddle.mean(loss) train_run_cost += time.time() - train_start total_samples += args.batch_size if global_step % args.logging_steps == 0: logger.info( "global step %d / %d, loss: %f, avg_reader_cost: %.5f sec, avg_batch_cost: %.5f sec, avg_samples: %.5f, ips: %.5f sequences/sec" % (global_step, last_step, avg_loss, train_reader_cost / args.logging_steps, (train_reader_cost + train_run_cost) / args.logging_steps, total_samples / args.logging_steps, total_samples / (train_reader_cost + train_run_cost))) train_reader_cost = 0.0 train_run_cost = 0.0 total_samples = 0 avg_loss.backward() optimizer.step() optimizer.clear_grad() if global_step % args.save_steps == 0 or global_step == last_step: if rank == 0: paddle.save( model.state_dict(), os.path.join(args.model_save_dir, "model_%d.pdparams" % global_step)) logger.info("Save %d steps model." % (global_step)) if args.do_eval: precision, recall, f1_score = evaluate( model, chunk_evaluator, test_loader) if f1_score > max_f1_score: max_f1_score = f1_score paddle.save( model.state_dict(), os.path.join(args.model_save_dir, "best_model.pdparams")) logger.info("Save best model.") reader_start = time.time()
from data import create_data_loader from models import create_model from options import options from utils.summary_utils import SummaryHelper from utils.image_utils import write_2images if __name__ == '__main__': opt = options.BaseOptions().parse() cudnn.benchmark = True model = create_model(opt) train_loader = create_data_loader(opt, 'train', opt.batch_size, not opt.serial_batches, num_workers=opt.num_workers) valid_loader = create_data_loader(opt, 'valid', 1, False, 1) # valid 数据集 valid_data = model.get_valid_dataset(valid_loader) del valid_loader train_Summary = SummaryHelper(save_path=os.path.join(opt.log_dir, 'train'), comment=opt.model, flush_secs=20) total_iters = 0 epoch_start, summary_step = model.setup() iter_start_time = time.time() for epoch in range(epoch_start + 1, opt.niter + opt.niter_decay + 1):
# hard-code some parameters for test opt.num_threads = 0 # test code only supports num_threads = 1 opt.batch_size = 1 # test code only supports batch_size = 1 opt.serial_batches = True # disable data shuffling; comment this line if results on randomly chosen images are needed. opt.no_flip = True # no flip; comment this line if results on flipped images are needed. opt.phase = 'test' opt.num_test = 4 opt.num_style = 9 model = create_model(opt) device = torch.device('cuda:{}'.format( opt.gpu_ids[0])) if opt.gpu_ids else torch.device('cpu') test_loader = create_data_loader(opt, 'test', opt.batch_size, opt.serial_batches, num_workers=opt.num_workers) model.setup() # regular setup: load and print networks; create schedulers A_encode = model.netG_A.encode B_encode = model.netG_B.encode B_decode = model.netG_B.decode # if opt.eval: # model.eval() for i, data in enumerate(test_loader): if i >= opt.num_test: # only apply our model to opt.num_test images. break model.set_input(data) # unpack data from data loader