def benchmark(weights_filename, r, verbose = "True"): filepath = 'model/weights/' + weights_filename # filepath of weights num_images = 729 # number of testing images to benchmark on (<=729) if verbose == "True": verbose = True else: verbose = False # Compile model opt = tf.keras.optimizers.Adam(learning_rate=0.001) # Peak Signal-to-Noise Ratio def PSNR(y_true, y_pred): max_pixel = 1.0 return tf.image.psnr(y_true, y_pred, max_val=max_pixel) model = espcn(r) model.compile(optimizer=opt, loss='mse', metrics=[PSNR]) # Initialize testing generator testing_generator = DataGenerator('LRbicx' + str(r), batch_size = 1, dictionary = "test") # Load weights model.load_weights(filepath) # Calculate average PSNR of all testing data average_psnr = 0 for i in range(0, num_images): lr, hr = testing_generator.__getitem__(i) sr = model.predict(lr) result = psnr(sr[0], hr[0]) average_psnr += result if verbose: print('Image: ' + str(i) + ', PSNR: ' + str(result) + ', Average: ' + str(average_psnr/(i+1))) print("Average PSNR: " + str(average_psnr/num_images))
def regulate(model, iters, batch_size, data_dir, dataset, embed_path, save_iter, saver, sess): data = DataGenerator(data_dir, dataset=dataset, portion='tra') optimizer = tf.train.AdamOptimizer(learning_rate=1e-3) train_op = optimizer.minimize(model.loss, var_list=[model.embed]) var_list = [v.name for v in tf.trainable_variables()] sess.run(tf.global_variables_initializer()) for i in range(iters): if i % (data.size / batch_size) == 0: data.shuffle() seqs, lens, gold = data.get_batch(i) loss, acc, w, _ = sess.run( [model.loss, model.accuracy, var_list, train_op], { model.x_ph: seqs, model.y_ph: gold, model.len_ph: lens }) if ((i + 1) % 100 == 0): print('| Iter {:3}: loss {:.4f}, acc {:.4f} |'.format( i + 1, loss, acc)) if ((i + 1) == save_iter): weight_dict = {} for key, value in zip(var_list, w): weight_dict[key] = value embed = weight_dict['embed:0'] np.save(embed_path, embed)
def main(): batch_size = 1 coord = tf.train.Coordinator() with tf.name_scope('create_inputs'): reader = DataGenerator(coord) input_batch = reader.dequeue(batch_size) sess = tf.Session(config=tf.ConfigProto(log_device_placement=False)) init = tf.global_variables_initializer() sess.run(init) threads = reader.start_threads(sess) net = define_net(input_batch) queue_size = reader.queue_size for step in range(10000): print('size queue =', queue_size.eval(session=sess)) print(sess.run(net)) # Make this thread slow. You can comment this line. If you do so, you will dequeue # faster than you enqueue, so expect the queue not to reach its maximum (32 by default) time.sleep(1) coord.request_stop() print("stop requested.") for thread in threads: thread.join()
def train(model, iters, batch_size, data_dir, dataset, ckpt_path, save_iter, saver, sess): data = DataGenerator(data_dir, dataset=dataset, portion='tra') optimizer = tf.train.AdamOptimizer(learning_rate=1e-3) train_op = optimizer.minimize(model.loss) sess.run(tf.global_variables_initializer()) for i in range(iters): if i % (data.size / batch_size) == 0: data.shuffle() seqs, lens, gold = data.get_batch(i) loss, acc, _ = sess.run([model.loss, model.accuracy, train_op], { model.x_ph: seqs, model.y_ph: gold, model.len_ph: lens }) if ((i + 1) % 100 == 0): print('| Iter {:3}: loss {:.4f}, acc {:.4f} |'.format( i + 1, loss, acc)) if ((i + 1) == save_iter): saver.save(sess, ckpt_path, write_meta_graph=False) print('| Saved to {}'.format(ckpt_path)) break
def fit_model(model, train_indexes, valid_indexes, test_indexes, const, p, verbose=1): ''' train_history has train and valid ''' train_gen = DataGenerator(train_indexes, const['batch_size'], const['datadir']) valid_gen = DataGenerator(valid_indexes, const['batch_size'], const['datadir']) test_gen = DataGenerator(test_indexes, const['batch_size'], const['datadir']) callbacks = [] # tensorboard_logdir_path = p2logdir_path(folder_path=const['tensorboard_dir'],p=p) # callbacks.append( tensorboard(tensorboard_logdir_path, batch_size=const['batch_size']) ) callbacks.append(EvaluateData(test_gen, log_word='test')) callbacks.append(PredictData(test_gen, denormalize, log_word='test')) callbacks.append(PredictData(valid_gen, denormalize, log_word='val')) callbacks.append(PredictData(train_gen, denormalize, log_word='')) history = model.fit_generator(generator=train_gen, epochs=const['epochs'], steps_per_epoch=len(train_gen), validation_data=valid_gen, validation_steps=len(valid_gen), verbose=verbose, callbacks=callbacks) return history.history
def main(): args = parse_args() gen = DataGenerator(args.num_vocab, args.max_length, args.num_examples) train_set, test_set = gen.get_tarin_test_datasets() train_loader = DataLoader(train_set, args.batch_size, True, pin_memory=True) test_loader = DataLoader(test_set, args.batch_size, True, pin_memory=True) if not args.attention: Model = Network else: Model = AttnNetwork network = Model(args.num_vocab + 2, args.num_vocab + 2, args.emb_dim, args.hidden_dim, args.max_length, args.num_layers, args.drop, args.rnn_type, args.tied, args.device) # optimizer = optim.SGD(network.parameters(), args.lr, 0.99) optimizer = optim.RMSprop(network.parameters(), args.lr) scheduler = optim.lr_scheduler.StepLR(optimizer, 1, args.lr_decay) best_loss = float('inf') for epoch in range(args.epochs): for param_group in optimizer.param_groups: print('#%d: \tLR %f' % (epoch, param_group['lr'])) acc, loss = single_step(network, train_loader, optimizer, .8 - epoch / args.epochs) print("#%d: \tTRAIN loss:%.03f \t acc:%.03f" % (epoch, loss, acc)) acc, loss = single_step(network, test_loader) print("#%d: \tTEST loss:%.03f \t acc:%.03f" % (epoch, loss, acc)) scheduler.step() if best_loss > loss: best_loss = loss print('Saving model to %s' % args.save) torch.save(network, args.save)
def test(model, batch_size, data_dir, dataset, sess): data = DataGenerator(data_dir, dataset=dataset, portion='test') iters = int(np.ceil(1.0 * data.size / batch_size)) preds = 0 for i in range(iters): seqs, lens, gold = data.get_batch(i) pred = sess.run(model.correct, { model.x_ph: seqs, model.y_ph: gold, model.len_ph: lens }) if (i + 1) * batch_size < data.size: preds += len(pred[pred == True]) else: remain = data.size - i * batch_size pred = pred[:remain] preds += len(pred[pred == True]) if i % 100 == 0: sys.stdout.write('\r{}/{}'.format(i, iters)) sys.stdout.flush() accuracy = 100. * preds / data.size print('\rTest accuracy: {:.2f}'.format(accuracy))
def train(conf): gan = KernelGAN(conf) learner = Learner() data = DataGenerator(conf, gan) for iteration in tqdm.tqdm(range(conf.max_iters), ncols=60): [g_in, d_in] = data.__getitem__(iteration) gan.train(g_in, d_in) learner.update(iteration, gan) gan.finish()
def estimate_kernel(img_file): conf = config_kernelGAN(img_file) kgan = KernelGAN(conf) learner = Learner() data = DataGenerator(conf, kgan) for iteration in tqdm.tqdm(range(conf.max_iters), ncols=70): [g_in, d_in, _] = data.__getitem__(iteration) kgan.train(g_in, d_in) learner.update(iteration, kgan) kgan.finish()
def train(conf): sr_net = DBPISR(conf) learner = Learner() data = DataGenerator(conf, sr_net) for iteration in tqdm.tqdm(range(conf.max_iters), ncols=60): g_in = data.__getitem__(iteration) sr_net.train(g_in) learner.update(iteration, sr_net) sr_net.finish(data.input_image)
def visualize_algorithm(algorithm, data): """ Responsible for user interaction, using cli """ g = DataGenerator() options = ['bubble_sort', 'insertion_sort', 'merge_sort', 'quick_sort', 'selection_sort'] options_data = {'random': g.random(), 'reversed' : g.reversed(), 'partsorted' : g.part_sorted()} algorithm = algorithm.lower() if algorithm.lower() in options else 'bubble_sort' data= options_data[data.lower()] if data.lower() in options_data.keys() else options_data['random'] v = Visualizer(data) v.visualize_algorithm(algorithm)
def main(): NUM_THREADS = multiprocessing.cpu_count() COMMON_PATH = os.path.join(os.path.expanduser("~"), 'local_tensorflow_content') pickle_file = 'titles_CBOW_data.pkl' pickle_file_path = os.path.join(os.path.expanduser("~"), pickle_file) dataGen = DataGenerator(pickle_file_path) model_config, training_config = {}, {} model_config['vocab_size'] = dataGen.vocab_size model_config['batch_size'] = 32 model_config['context_window'] = 2 model_config['embedding_size'] = 128 model_config['neg_sample_size'] = 2 model_config['learning_rate'] = 0.0005 model_config['model_name'] = 'word2vec' batches = dataGen.generate_sequence(model_config['batch_size']) model = word2vec(**model_config) use_gpu = False if use_gpu: training_config['sess_config'] = tf.ConfigProto( log_device_placement=False, gpu_options=tf.GPUOptions(per_process_gpu_memory_fraction=0.5)) else: os.environ[ 'CUDA_VISIBLE_DEVICES'] = '-1' # the only way to completely not use GPU training_config['sess_config'] = tf.ConfigProto( intra_op_parallelism_threads=NUM_THREADS) training_config['model_path'] = create_local_model_path( COMMON_PATH, model_config['model_name']) training_config['log_path'] = create_local_log_path( COMMON_PATH, model_config['model_name']) generate_tensorboard_script( training_config['log_path'] ) # create the script to start a tensorboard session training_config['epoch_num'] = 20000 training_config['display_steps'] = 1000 training_config['saving_steps'] = 1 * training_config['display_steps'] training_config['num_batches'] = int(dataGen.data_size * training_config['epoch_num'] / model_config['batch_size']) print 'total #batches: {}, vocab_size: {}'.format( training_config['num_batches'], model_config['vocab_size']) model.train(batches, training_config, restore_model=False)
def main(params): data = polyvore_dataset() transforms = data.get_data_transforms() X_train, X_test, y_train, y_test, n_classes = data.create_dataset() train_set = (X_train, y_train, transforms['train']) test_set = (X_test, y_test, transforms['test']) dataset_size = {'train': len(y_train), 'test': len(y_test)} params = { 'batch_size': Config['batch_size'], 'n_classes': n_classes, 'shuffle': True } train_generator = DataGenerator(train_set, dataset_size, params) test_generator = DataGenerator(test_set, dataset_size, params)
def compile_model(): model = create_model() model.compile(optimizer=optimizers.Adam(), loss=deepball_loss_function, metrics=([deepball_precision])) model_checkpoint = callbacks.ModelCheckpoint(filepath='footballcnn.h5', verbose=1) train_datagen = DataGenerator(file_path=IMAGE_PATH, config_path=CONFIG_PATH) model.fit(x=train_datagen, epochs=6, callbacks=[model_checkpoint]) model.save_weights('footballcnn.h5') """"img = preprocessing.image.load_img('1frame1199.jpg', target_size=(360, 640, 3)) input1 = preprocessing.image.img_to_array(img) input1 = input1.reshape([1, 360, 640, 3]) input1 = input1 / 255. b = model.predict(input1) print(b.shape) b = b[0, :, :, 0] b = np.expand_dims(b, axis=2) preprocessing.image.save_img('pred.jpg', b) c = np.unravel_index(b.argmax(), b.shape) print(c)""" return
def train(args): if args.load_trained: last_epoch, arch, model, tokenizer, scores = load_checkpoint(args.pytorch_dump_path) else: # May load local file or download from huggingface model, tokenizer = load_pretrained_model_tokenizer(base_model=args.local_model, base_tokenizer=args.local_tokenizer, device=args.device) last_epoch = 1 train_dataset = DataGenerator(args.data_path, args.data_name, args.batch_size, tokenizer, 'train', args.device) validate_dataset = DataGenerator(args.data_path, args.data_name, args.batch_size, tokenizer, 'dev', args.device) test_dataset = DataGenerator(args.data_path, args.data_name, args.batch_size, tokenizer, 'test', args.device) optimizer = init_optimizer(model, args.learning_rate, args.warmup_proportion, args.num_train_epochs, train_dataset.data_size, args.batch_size) model.train() best_score = 0 step = 0 for epoch in range(last_epoch, args.num_train_epochs + 1): print('Epoch: {}'.format(epoch)) tr_loss = 0 while True: batch = train_dataset.load_batch() if batch is None: break tokens_tensor, segments_tensor, mask_tensor, label_tensor = batch[:4] loss = model(tokens_tensor, segments_tensor, mask_tensor, label_tensor) loss.backward() tr_loss += loss.item() optimizer.step() model.zero_grad() if args.eval_steps > 0 and step % args.eval_steps == 0: print('Step: {}'.format(step)) best_score = eval_select(model, tokenizer, validate_dataset, test_dataset, args.pytorch_dump_path, best_score, epoch) step += 1 print('[train] loss: {}'.format(tr_loss)) best_score = eval_select(model, tokenizer, validate_dataset, test_dataset, args.pytorch_dump_path, best_score, epoch) scores = test(args, split='test') print_scores(scores)
def main(): model_dir = "/home/tensor/tensor/scene/DataSet/checkpoints/" train_image_dir = "/home/tensor/tensor/scene/DataSet/train/" validate_image_dir = "/home/tensor/tensor/scene/DataSet/validation/" pretrained_model_path = "/home/tensor/tensor/scene/DataSet/pre_trained/inception_resnet_v2.ckpt" datagen = DataGenerator(train_image_dir, validate_image_dir) model = ModelFactory(datagen, net='INCEPTION_RESNET_V2', model_dir=model_dir, fine_tune=True, pretrained_path=pretrained_model_path) with tf.Session() as session: model.train(session)
def main(images_path, labels_path): keras.backend.clear_session() data_df = get_data(images_path, labels_path) raw_train, valid = split_data(data_df) model = create_model(num_classes=28, input_shape=input_shape) model.compile(loss="binary_crossentropy", optimizer=Adam(), metrics=["acc", f1]) # model.compile(loss=[_focal_loss(gamma=2,alpha=0.75)], optimizer=Adam(), metrics=["acc", f1]) epochs = 50 batch_size = 64 checkpointer = ModelCheckpoint("../working/InceptionResNetV2.model", verbose=2, save_best_only=True) early_stopping = EarlyStopping(monitor="val_loss", patience=2) reduce_lr = ReduceLROnPlateau(monitor="val_loss", patience=1, factor=0.1) train_generator = DataGenerator.create_train(raw_train, batch_size, DEFAULT_IMG_SIZE_WHC, augument=True) validation_generator = DataGenerator.create_train(valid, 100, DEFAULT_IMG_SIZE_WHC, augument=False) train_steps = raw_train.shape[0] // batch_size valid_steps = valid.shape[0] // batch_size # train model history = model.fit_generator( train_generator, steps_per_epoch=train_steps, validation_data=next(validation_generator), validation_steps=valid_steps, epochs=epochs, verbose=1, callbacks=[checkpointer, reduce_lr], )
def main(): datagen = DataGenerator(FLAGS.train_json, FLAGS.train_image_dir, FLAGS.validate_json, FLAGS.validate_image_dir) model = ModelFactory(datagen, net='INCEPTION_RESNET_V2', model_dir=FLAGS.model_dir, fine_tune=True, pretrained_path=FLAGS.pretrained_model_path) with tf.Session() as session: model.train(session)
def predict_to_file(infile, out_file="result.json"): def save_json(record_dict): fw = open(out_file, 'w', encoding='utf-8') json_str = json.dumps(record_dict, ensure_ascii=False, indent=4) fw.write(json_str) fw.close() model=torch.load("dureder_model").to(device) model.eval() R = {} ignore_id=set() if resume_predict: with open(out_file,'r') as load_f: R = json.load(load_f) for key in R: ignore_id.add(key) data_gen = DataGenerator(infile, tokenizer,device) batch_cnt=0 for batch in data_gen.batchIterNoAnswer(predict_batch_size,ignore_id): batch_cnt+=1 batch_id=batch["batch_id"] batch_pair_ids = batch["batch_pair_ids"] batch_token_type_ids = batch["batch_token_type_ids"] batch_attention_mask = batch["batch_attention_mask"] batch_context_len=batch["batch_context_len"] batch_question_len=batch["batch_question_len"] batch_context_ids=batch["batch_context_ids"] p1, p2 = model(batch_pair_ids,batch_token_type_ids,batch_attention_mask,batch_context_len,batch_question_len) answers=get_batch_predict_answer(batch_context_ids, p1, p2) print(answers) for i in range(len(batch_id)): R[batch_id[i]] = answers[i] if(batch_cnt % 200 == 0): save_json(R) save_json(R)
def train(model, r, batch_size, epochs): # Fit model training_generator = DataGenerator('LRbicx' + str(r), batch_size=batch_size) model.fit_generator(generator=training_generator, epochs=epochs, verbose=1) # Save weights # Filepath where the weights will be saved to filepath = 'model/weights/r' + str(r) + 'bs' + str( batch_size) + 'epochs' + str(epochs) + 'weights.h5' model.save_weights(filepath) print("Saved weights at : " + filepath)
def main(): print('loading modules ...') gen = DataGenerator('protain/all/train/', 'protain/all/test/', 'protain/all/train.csv') model = Resnet(resnet_layers=4, channels=[4, 16, 16, 32, 32]) print('Done') epoch = 10 for i in range(epoch): val_x, val_y = gen.get_validation_set() bar = tqdm(gen.get_batch(), total=len(gen.train_ids) // 8) for x, y in bar: loss = model.train(x, y) bar.set_description('loss = {:.5f}'.format(loss)) preds = np.array([[int(y >= 0.5) for y in model.predict([x])[0]] for x in tqdm(val_x)]) print('[epoch {}]: f1_macro = {}'.format(i, f1_score(val_y, preds, average='macro'))) preds_test = [(name, [i for i, y in enumerate(model.predict([x])[0]) if y >= 0.5]) for name, x in gen.get_test_set()] with open('submission.csv', 'w') as f: f.write('Id,Predicted\n') for id_, preds in preds_test: f.write('{},{}\n'.format(id_, ' '.join(list(map(str, preds)))))
def train(infile): if resume_train==False: encode_model = AlbertModel.from_pretrained(pretrained).to(device) model=BertBidaf(tokenizer,encode_model,device) else: model=torch.load("dureder_model") model=model.to(device) data_gen=DataGenerator(infile,tokenizer,device) optimizer = optim.Adam(model.parameters(),lr=learning_rate) criterion = nn.CrossEntropyLoss() model.train() for e in range(epoch): batch_cnt=0 for batch in data_gen.batchIter(train_batch_size): batch_cnt+=1 batch_pair_ids=batch["batch_pair_ids"] batch_token_type_ids=batch["batch_token_type_ids"] batch_attention_mask=batch["batch_attention_mask"] batch_start=batch["batch_start"] batch_end=batch["batch_end"] batch_context_len=batch["batch_context_len"] batch_question_len=batch["batch_question_len"] p1,p2=model(batch_pair_ids,batch_token_type_ids,batch_attention_mask,batch_context_len,batch_question_len) optimizer.zero_grad() batch_loss = criterion(p1, batch_start) + criterion(p2, batch_end) print(e,batch_cnt*train_batch_size,batch_loss.item()) # print(get_batch_predict_answer(batch_pair_ids, p1, p2)) # print(batch_ans) batch_loss.backward() optimizer.step() if(batch_cnt % 20 == 0): torch.save(model,'dureder_model') torch.save(model,'dureder_model')
def test (model, testDirectory, classList, INPUT_FRAMES = 64, batchSize = 10): print("\n\n\ngenerating Annotation List...") annotationList = generateDatasetList(testDirectory,INPUT_FRAMES,classList=classList) print("creating data generator...") dataGenerator = DataGenerator(annotationList,INPUT_FRAMES,batch_size=batchSize) print("starting test...\n") out_logits = model.predict_generator(dataGenerator, steps=None, max_queue_size=10, workers=1, use_multiprocessing=False, verbose=1) out_logits = out_logits[:len(annotationList)] predictions = getPredictions(out_logits) output = generateFormatedOutput(predictions,annotationList,classList) writeJsontoFile("results.json",output) np.save("logits.npy", out_logits)
def main(): batch_size = 32 num_classes = 4 epochs = 100 save_dir = os.path.join(os.getcwd(), 'saved_models') model_name = 'orientation-inception.h5' data_train, data_test = load_data() # Use Google Inception v3 model model = InceptionV3( include_top=False, weights=None, input_shape=(192, 192, 3), pooling='softmax', classes=4, ) # initiate RMSprop optimizer opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6) # Let's train the model using RMSprop model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) if not os.path.isdir(save_dir): os.makedirs(save_dir) checkpointer = ModelCheckpoint( filepath=os.path.join(save_dir, 'checkpoint.h5'), verbose=1, save_best_only=True, ) early_stopping = EarlyStopping(monitor='val_loss', patience=2) train_generator = DataGenerator(data_train) val_generator = DataGenerator(data_test) model.fit_generator( train_generator.flow(batch_size=batch_size), epochs=epochs, validation_data=val_generator.flow(batch_size=batch_size), shuffle=True, callbacks=[checkpointer, early_stopping], ) # Save model and weights model_path = os.path.join(save_dir, model_name) model.save(model_path) print('Saved trained model at %s' % model_path) # Score trained model. scores = model.evaluate_generator( val_generator.flow(batch_size=batch_size)) print('Test loss:', scores[0]) print('Test accuracy:', scores[1])
def __init__(self): try: from faker import Faker from faker.config import DEFAULT_PROVIDERS self.fake = Faker() self.providers = DEFAULT_PROVIDERS except: self.fake = None self.providers = None self.outfile = StringIO() self.section = None self.section_titles = [] self.section_structure = [] self.data_gen = DataGenerator()
def main(): conf = load_conf() wandb.init(project=conf.proj_name, config=dict(conf)) agent = Agent(embed_hidden=conf.embed_hidden, enc_stacks=conf.enc_stacks, ff_hidden=conf.ff_hidden, enc_heads=conf.enc_heads, query_hidden=conf.query_hidden, att_hidden=conf.att_hidden, crit_hidden=conf.crit_hidden, n_history=conf.n_history, p_dropout=conf.p_dropout) wandb.watch(agent) dataset = DataGenerator() trainer = Trainer(conf, agent, dataset) trainer.run() # Save trained agent torch.save(agent.state_dict(), conf.model_path) if conf.test: device = torch.device(conf.device) # Load trained agent agent.load_state_dict(torch.load(conf.model_path)) agent.eval() agent = agent.to(device) running_reward = 0 for _ in range(conf.test_steps): input_batch = dataset.test_batch(conf.batch_size, conf.max_len, conf.dimension, shuffle=False) input_batch = torch.Tensor(input_batch).to(device) tour, *_ = agent(input_batch) reward = reward_fn(input_batch, tour) # Find best solution j = reward.argmin() best_tour = tour[j][:-1].tolist() # Log running_reward += reward[j] # Display print('Reward (before 2 opt)', reward[j]) opt_tour, opt_length = dataset.loop2opt( input_batch.cpu()[0][best_tour]) print('Reward (with 2 opt)', opt_length) dataset.visualize_2D_trip(opt_tour) wandb.run.summary["test_reward"] = running_reward / conf.test_steps
def train(conf): gan = KernelGAN(conf) learner = Learner() data = DataGenerator(conf, gan) dataloader = DataLoader(data, batch_size=batch_size, shuffle=False) timer = 0 for i_batch, sample_batched in enumerate(tqdm.tqdm(dataloader)): g_in,d_in = sample_batched gan.train(g_in,d_in) learner.update(i_batch*batch_size, gan) if learner.flag: timer += 1 if timer > 10: break gan.finish()
def main(): #模型训练预测时的参数 mp = model_params() #构建bert的参数 bp = bert_params(with_pool=True) datagen = DataGenerator(bp, batch_size=mp.batch_size, num_neg=mp.num_neg, shuffle=mp.shuffle) #后续再尝试用其它的优化器 optimizer = Adagrad(learning_rate=mp.learning_rate) my_model = Bert4QA(bp) #训练主类 t = TrainOrPredict(mp) #final_model就是训练好的模型 final_model = t.train(my_model, optimizer, datagen) data = datagen.data_faq tokenizer = datagen.tokenizer #训练完成后查看效果 real_query_text = "月球和地球是什么关系?" question_score = {} for query_name in data.query_dict.keys(): query_text = data.query_dict[query_name] token_ids, segment_ids = tokenizer.encode(real_query_text, query_text) question_score[query_name] = final_model.predict( [token_ids, segment_ids]) question_score = {k: v.numpy() for k, v in question_score.items()} qs = dict(sorted(question_score.items(), key=lambda x: x[1], reverse=True)) c = 0 for k, v in qs.items(): c += 1 print(k, data.query_dict[k], v) if c == 10: break return final_model
def main(): if os.path.exists(JOB_NAME): raise AssertionError("Job name already exists") else: os.mkdir(JOB_NAME) f = open(os.path.join(JOB_NAME, "train_params.txt"), 'w') f.write("META_LEARNER " + str(META_LEARNER) + '\n') f.write("FUNCTION " + str(FUNCTION_TRAIN) + '\n') f.write("K_TRAIN " + str(K_TRAIN) + '\n') f.write("SGD_STEPS_TRAIN " + str(SGD_STEPS_TRAIN) + '\n') f.write("NOISE_PERCENT_TRAIN " + str(NOISE_PERCENT_TRAIN) + '\n') f.write("ITERATIONS_TRAIN " + str(ITERATIONS_TRAIN) + '\n') f.write("OUTER_LR_TRAIN " + str(OUTER_LR_TRAIN) + '\n') f.write("INNER_LR_TRAIN " + str(INNER_LR_TRAIN) + '\n') f.write("AVERAGER_SIZE_TRAIN " + str(AVERAGER_SIZE_TRAIN) + '\n') f.close() model = Net() if META_LEARNER == "reptile": learning_alg = Reptile(lr_inner=INNER_LR_TRAIN, lr_outer=OUTER_LR_TRAIN, sgd_steps_inner=SGD_STEPS_TRAIN) elif META_LEARNER == "maml": learning_alg = MAML(lr_inner=INNER_LR_TRAIN, lr_outer=OUTER_LR_TRAIN, sgd_steps_inner=SGD_STEPS_TRAIN) else: learning_alg = Insect(lr_inner=INNER_LR_TRAIN, lr_outer=OUTER_LR_TRAIN, sgd_steps_inner=SGD_STEPS_TRAIN, averager=AVERAGER_SIZE_TRAIN) meta_train_data = DataGenerator(function=FUNCTION_TRAIN, size=ITERATIONS_TRAIN, K=K_TRAIN, noise_percent=NOISE_PERCENT_TRAIN) learning_alg.train(model, meta_train_data) torch.save(model, os.path.join(JOB_NAME, "trained_model.pth")) test(model)
def evaluate(base_model, function, K, noise_percent, SGD_steps, inner_lr, runs=100): all_losses = [] test_tasks = DataGenerator(function, runs, K, noise_percent).shuffled_set() for test_task in test_tasks: xeval, yeval = test_task.eval_set(size=100) model = copy.deepcopy(base_model) optim = torch.optim.SGD(model.parameters(), lr=inner_lr) losses = [] predicted = model(xeval) losses.append(F.mse_loss(predicted, yeval).item()) for i in range(SGD_steps): inner_train(model, test_task, optim) predicted = model(xeval) losses.append(F.mse_loss(predicted, yeval).item()) norm_losses = np.array(losses) / test_task.amp * 100 all_losses.append(norm_losses) return np.array(all_losses)
# -*- coding: utf-8 -*- import json from data import DataParser, CSVParser, DataGenerator, DEPDataParser import mturk, pickle from pprint import pprint import nltk from random import shuffle parser = DataParser() csv_parser = CSVParser() data_generator = DataGenerator() dep_data_generator = DEPDataParser() #### # INPUT FOR THE SCRIPT #### main_data_file = "data/data_all.json" split_values = {"train": 0.7, "validate": 0.1, "test": 0.2} fix_file = "fix_turk_result.csv" #### # This is main data (from Tim) with open(main_data_file) as data_file: data = json.load(data_file) # Segment it into json
# -*- coding: utf-8 -*- import json from data import DataParser, CSVParser, DataGenerator, DEPDataParser import mturk from pprint import pprint parser = DataParser() csv_parser = CSVParser() data_generator = DataGenerator() dep_data_generator = DEPDataParser() #### # INPUT FOR THE SCRIPT #### main_data_file = "data/data_all.json" #### # This is main data (from Tim) with open(main_data_file) as data_file: data = json.load(data_file) # Segment it into json segmented_data = parser.parse_data(data) annotated_paragraphs = []
def train(args): path = args.save_dir try: os.makedirs(path) except OSError as exc: if exc.errno == errno.EEXIST and os.path.isdir(path): pass else: raise with open(args.vocab, 'r') as fin: vocab = json.load(fin) args.vocab_size = len(vocab) args.docs_looped = True train = DataGenerator(args, '../data/processed/train.npy') args.docs_looped = False val = DataGenerator(args, '../data/processed/test.npy') args.iterations_per_epoch = int(train.samples / args.batch_size) args.iterations_per_val = int(val.samples / args.batch_size) if args.init_from is not None: # check if all necessary files exist assert os.path.isdir(args.init_from), \ " %s must be a a path" % args.init_from assert os.path.isfile(os.path.join(args.init_from, "config.pkl")), \ "config.pkl file does not exist in path %s" % args.init_from ckpt = tf.train.get_checkpoint_state(args.init_from) assert ckpt, "No checkpoint found" assert ckpt.model_checkpoint_path, "No model path found in checkpoint" # open old config and check if models are compatible with open(os.path.join(args.init_from, 'config.pkl')) as f: saved_model_args = pickle.load(f) need_be_same = ["model", "rnn_size", "num_layers", "seq_length"] for checkme in need_be_same: assert vars(saved_model_args)[checkme] == vars(args)[checkme], \ "Command line argument and saved model disagree on '%s' " % checkme with open(os.path.join(args.save_dir, 'config.pkl'), 'wb') as f: pickle.dump(args, f) model = Model(args) with tf.Session() as sess: tf.initialize_all_variables().run() saver = tf.train.Saver(tf.all_variables()) # restore model if args.init_from is not None: saver.restore(sess, ckpt.model_checkpoint_path) for e in range(args.num_epochs): sess.run(tf.assign(model.lr, args.learning_rate * (args.decay_rate ** e))) train.reset() state = model.initial_state.eval() pbar = tqdm(range(args.iterations_per_epoch)) for b in pbar: x, y = next(train) feed = {model.input_data: x, model.targets: y, model.initial_state: state} train_loss, state, _ = sess.run([model.cost, model.final_state, model.train_op], feed) if b % 10 == 0: pbar.set_description('train_loss: {:.3f}' .format(train_loss)) print('Checkpoint') checkpoint_path = os.path.join(args.save_dir, 'model.ckpt') saver.save(sess, checkpoint_path, global_step=e * args.iterations_per_epoch) print('Validation...') # we can restore training states # train_states = state[:] val_loss = 0.0 state = model.initial_state.eval() val.reset() pbar = tqdm(range(args.iterations_per_val)) for b in pbar: x, y = next(val) feed = {model.input_data: x, model.targets: y, model.initial_state: state} (loss, state) = sess.run([model.cost, model.final_state, ], feed) pbar.set_description('val_loss: {:.3f}' .format(loss)) val_loss += loss val_loss /= args.iterations_per_val print('Mean val_loss is {:.3f}'.format(val_loss))