def add_target(img): # generate augmentations poses = [{ "yaw": -45, "pitch": 0, "roll": 0 }, { "yaw": -30, "pitch": 0, "roll": 0 }, { "yaw": -15, "pitch": 0, "roll": 0 }, { "yaw": 15, "pitch": 0, "roll": 0 }, { "yaw": 30, "pitch": 0, "roll": 0 }, { "yaw": 45, "pitch": 0, "roll": 0 }] folder_name = str(time.time()) os.mkdir("./tmp/" + folder_name) cv2.imwrite("./tmp/" + folder_name + "/" + "original.png", cv2.cvtColor(img, cv2.COLOR_RGB2BGR)) bundle_loc = "./tmp/" + folder_name + ".zip" prepare_batch(["./tmp/" + folder_name + "/" + "original.png"], bundle_loc) process_id = post_batch(poses=poses, bundle_loc=bundle_loc) processed_loc = "./tmp/" + folder_name + "_processed.zip" time.sleep(3) finished = False while not finished: finished = get_processed(process_id, processed_loc) # downloads the zip time.sleep(5) unzip_and_move(processed_loc, "./tmp/" + folder_name + "/", ["henk"]) img_list = [] for file in sorted(os.listdir("./tmp/" + folder_name))[::-1]: img = cv2.cvtColor(cv2.imread("./tmp/" + folder_name + "/" + file), cv2.COLOR_BGR2RGB) pipeline.add_to_targets(img) img_list.append(np_array_to_string(img)) os.remove("./tmp/" + folder_name + "/" + file) os.rmdir("./tmp/" + folder_name) return img_list
def main(): # get file iterator source_loc = "dataset" if ARGS.source == "" else ARGS.source target_loc = "dataset_processed" bundle_loc = "batch.zip" processed_loc = "batch_processed.zip" batch_size = 25 poses = [ { "yaw": -8, "pitch": -8, "roll": 0 }, { "yaw": -1, "pitch": 8, "roll": 0 }, { "yaw": 5, "pitch": 5, "roll": 0 } ] total_files = number_of_files(source_loc) counter = 0 current_progress = 0 iterator = get_files(source_loc, batch_size=batch_size) for batch_filenames in iterator: current_progress += len(batch_filenames) pretty_print(counter, "processing: {}/{}".format(current_progress, total_files)) prepare_batch(batch_filenames, bundle_loc) pretty_print(counter, "posting batch request") process_id = post_batch(poses=poses, bundle_loc=bundle_loc) pretty_print(counter, "request id: {}".format(process_id)) # do some waiting until it is finished finished = False while not finished: finished = get_processed(process_id, processed_loc) # downloads the zip time.sleep(10) # pretty_print(counter, "{} is available".format(process_id)) # get_processed(process_id, processed_loc) # downloads the zip names = [name.split('/')[1].split('.')[0] for name in batch_filenames] # extract the zip, go into the 3duniversum structure and move the files to the target location unzip_and_move(processed_loc, target_loc, names=names) # Copy the originals copy_originals(batch_filenames, target_loc) pretty_print(counter, "batch finished") counter += 1
def make_train_inputs(self, x, y): inputs, num_inputs = utils.prepare_batch(x) targets, num_targets = utils.prepare_batch(y) return { self.encoder_inputs: inputs, self.encoder_inputs_length: num_inputs, self.decoder_targets: targets, self.decoder_targets_length: num_targets }
def evaluate_by_logic_level(args, model, iterator, print_total=False): from tqdm import tqdm vocab = model.vocab model.eval() cor_que_n = torch.zeros(5) # level: 1 ~ 4 (0: pad) all_que_n = torch.zeros(5) # level: 1 ~ 4 (0: pad) all_que_n[0] = 1 # Prevent DivisionByZero with torch.no_grad(): for batch in tqdm(iterator, desc='Calculating accuracy by question logic level'): net_inputs, target = prepare_batch(args, batch, vocab) q_level_logic = net_inputs['q_level_logic'] if net_inputs['subtitle'].nelement() == 0: import ipdb; ipdb.set_trace() # XXX DEBUG y_pred = model(**net_inputs) _, pred_idx = y_pred.max(dim=1) result = pred_idx == target for i, lev in enumerate(q_level_logic): if result[i]: # correct cor_que_n[lev] += 1 all_que_n[lev] += 1 accuracys = cor_que_n / all_que_n.float() print('Accuracy by question logic level: ') for i in range(1, 5): print('Level %d: %.4f' % (i, accuracys[i])) if print_total: print('Total Accuracy:', cor_que_n.sum().item() / (all_que_n.sum().item() - 1))
def update_model(trainer, batch): args.pretraining = True model.train() optimizer.zero_grad() net_inputs, target = prepare_batch(args, batch, model.module.vocab) #net_inputs, target = prepare_batch(args, batch, model.vocab) y_pred, char_pred, mask_pred = model(**net_inputs) batch_size = y_pred.shape[0] # get person ground truth and compute character loss n_char = 21 visual_char = net_inputs['filtered_visual'].view(batch_size, -1, 3)[:,:,0] char_target = visual_char.unsqueeze(2).view(batch_size, -1) char_target = char_target.view(-1) char_pred = char_pred.view(-1, n_char) character_loss = nn.CrossEntropyLoss(ignore_index=-1).cuda()(char_pred, char_target) # get ground truth labels and compute MLM loss vocab_size = mask_pred.size(-1) mask_target = net_inputs['labels'] mask_target = mask_target.view(-1) mask_pred = mask_pred.view(-1, vocab_size) mlm_loss = nn.CrossEntropyLoss(ignore_index=-1).cuda()(mask_pred, mask_target) # compute QA loss loss, stats = loss_fn(y_pred, target) # compute total loss loss = character_loss + mlm_loss loss.backward() optimizer.step() return loss.item(), stats, batch_size, y_pred.detach(), target.detach()
def decode(): os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.gpu # Load model config config = load_config(FLAGS) print(config) # Load source data to decode test_set = TextIterator(source=config['decode_input'], split_sign=config['split_sign'], batch_size=config['decode_batch_size'], source_dict=config['source_vocabulary'], n_words_source=config['num_encoder_symbols']) # Load inverse dictionary used in decoding target_inverse_dict = load_inverse_dict(config['target_vocabulary']) # Initiate TF session with tf.Session(config=tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement, gpu_options=tf.GPUOptions(allow_growth=True))) as sess: # Reload existing checkpoint model = load_model(sess, config) try: print('Decoding {}..'.format(FLAGS.decode_input)) if FLAGS.write_n_best: fout = [open(("%s_%d" % (FLAGS.decode_output, k)), 'w') \ for k in range(FLAGS.beam_width)] else: fout = [open(FLAGS.decode_output, 'w')] test_set.reset() for idx, source_seq in enumerate(test_set.next()): source, source_len = prepare_batch(source_seq) print('Get Batch', len(source), len(source_len)) print('Source', list(source[0]), 'Source Len', source_len[0]) # predicted_ids: GreedyDecoder; [batch_size, max_time_step, 1] # BeamSearchDecoder; [batch_size, max_time_step, beam_width] predicted_ids = model.predict(sess, encoder_inputs=source, encoder_inputs_length=source_len) # Write decoding results for k, f in reversed(list(enumerate(fout))): for seq in predicted_ids: result = str(seq2words(seq[:, k], target_inverse_dict)) f.write(result + '\n') f.flush() if not FLAGS.write_n_best: break print('{}th line decoded'.format(idx * FLAGS.decode_batch_size)) print('Decoding terminated') except IOError: pass finally: [f.close() for f in fout]
def make_infer_inputs(self, x): inputs, num_inputs = utils.prepare_batch(x) return { self.encoder_inputs: inputs, self.encoder_inputs_length: num_inputs }
def warp_images(): print('building model') layers = vgg16.build_model((None, 3, 227, 227)) batch_size = 32 infer_dir = join('data', 'inference') weightsfile = join('weights', 'weights.pickle') with open(weightsfile, 'rb') as f: param_values = pickle.load(f) set_all_param_values(layers['trans'], param_values) pretrainfile = join('weights', 'vgg16.pkl') with open(pretrainfile, 'rb') as f: data = pickle.load(f) mean = data['mean value'] image_fpaths = [('Cars_013b.png', 'Cars_009b.png'), ('060_0071.png', '060_0000.png'), ('246_0052.png', '246_0042.png')] print('compiling theano functions for inference') num_infer_idx = (len(image_fpaths) + batch_size - 1) / batch_size infer_func = theano_funcs.create_infer_func(layers) infer_iter = utils.get_batch_idx(len(image_fpaths), batch_size) for i, idx in tqdm(infer_iter, total=num_infer_idx, leave=False): Xa, Xb = utils.prepare_batch(image_fpaths[idx], mean) M = infer_func(Xa, Xb) utils.plot_samples(Xa, Xb, M, mean, prefix=join(infer_dir, 'infer_%d' % i))
def update_model(trainer, batch): model.train() optimizer.zero_grad() # to gpu net_inputs, target = prepare_batch(args, batch) # ** : dictionary input to each argument # y_pred : dict {z_i, z_j, p_i, p_j} x_i = net_inputs['x_i'] z_i, p_i = model(x_i) del x_i x_j = net_inputs['x_j'] z_j, p_j = model(x_j) #y_pred = model(**net_inputs) del net_inputs, x_j y_pred = {'p_i': p_i, 'p_j': p_j, 'z_i': z_i, 'z_j': z_j} batch_size = target.shape[0] # N loss = loss_fn(y_pred) #loss = loss.mean() # ddp #with amp.scale_loss(loss, optimizer, loss_id=0) as scaled_loss: # scaled_loss.backward() loss.backward() optimizer.step() scheduler.step() return loss.item(), batch_size, y_pred.detach()
def _inference(evaluator, batch): model.eval() with torch.no_grad(): net_inputs, target = prepare_batch(args, batch, model.vocab) y_pred = model(**net_inputs) batch_size = y_pred.shape[0] loss, stats = loss_fn(y_pred, target) return loss.item(), stats, batch_size, y_pred, target # TODO: add false_answer metric
def decode(): # TODO: Config has to be taken from train config = OrderedDict(FLAGS.__flags.items()) ''' test_set = TextIterator(source=FLAGS.decode_input, batch_size=FLAGS.decode_batch_size, source_dict=source_vocabulary, maxlen=None, n_words_source=30000) ''' test_set = CorpusIterator(sourcepath='../text/validation', batch_size=FLAGS.decode_batch_size, source_vocabulary=source_vocabulary) # print 'CHECK', test_set.source_dict['motivazioni'] target_inverse_dict = utils.load_inverse_dict(target_vocabulary) with tf.Session(config=tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement, gpu_options=tf.GPUOptions(allow_growth=True))) as sess: model = load_model(sess, config) try: print 'Decoding {}..'.format(FLAGS.decode_input) fout = codecs.open(FLAGS.decode_output, 'w', 'utf8') for text in test_set: # For multiple text print 'Text: ', text.source for idx, source_seq in enumerate(text): source, source_len = prepare_batch(source_seq) predicted_ids = model.predict( sess, encoder_inputs=source, encoder_inputs_length=source_len) for seq in predicted_ids: # print utils.seq2words(seq, target_inverse_dict) # print len(utils.seq2words(seq, target_inverse_dict).split()) # TODO: Check if it writes the sentences in the correct order fout.write( utils.seq2words(seq, target_inverse_dict) + '\n') # if not FLAGS.write_n_best: # break print ' {}th line decoded'.format( (idx + 1) * FLAGS.decode_batch_size) print 'Decoding terminated' except IOError: pass finally: fout.close()
def update_model(trainer, batch): model.train() optimizer.zero_grad() net_inputs, target = prepare_batch(args, batch, model.vocab) y_pred = model(**net_inputs) batch_size = y_pred.shape[0] loss, stats = loss_fn(y_pred, target) loss.backward() optimizer.step() return loss.item(), stats, batch_size, y_pred.detach(), target.detach()
def _inference(evaluator, batch): model.eval() with torch.no_grad(): qids = batch["qid"] net_inputs, _ = prepare_batch(args, batch, model.vocab) y_pred = model(**net_inputs) y_pred = y_pred.argmax(dim=-1) # + 1 # 0~4 -> 1~5 for qid, ans_idx in zip(qids, y_pred): engine.answers[qid] = ans_idx.item() return
def _inference(evaluator, batch): model.eval() with torch.no_grad(): net_inputs, target = prepare_batch(args, batch, model.vocab) if net_inputs['subtitle'].nelement() == 0: import ipdb ipdb.set_trace() # XXX DEBUG y_pred = model(**net_inputs) batch_size = y_pred.shape[0] loss, stats = loss_fn(y_pred, target) return loss.item( ), stats, batch_size, y_pred, target # TODO: add false_answer metric
def _inference(evaluator, batch): model.eval() with torch.no_grad(): qids = batch["qid"] net_inputs, _ = prepare_batch(args, batch, model.module.vocab) y_pred, char_pred, mask_pred = model(**net_inputs) print("Before argmax:", y_pred.size()) y_pred = y_pred.argmax(dim=-1) # + 1 # 0~4 -> 1~5 print("After argmax:", y_pred.size()) for qid, ans in zip(qids, y_pred): engine.answers[qid] = ans.item() return
def _inference(evaluator, batch): nonlocal sample_count model.eval() with torch.no_grad(): net_inputs, target = prepare_batch(args, batch, model.vocab) if net_inputs['subtitle'].nelement() == 0: import ipdb ipdb.set_trace() # XXX DEBUG y_pred = model(**net_inputs) batch_size = y_pred.shape[0] loss, stats = loss_fn(y_pred, target) vocab = model.vocab ''' if sample_count < 100: print('Batch %d: data and prediction from %d to %d' % (sample_count // batch_size, sample_count, sample_count + batch_size - 1)) que = net_inputs['que'] answers = net_inputs['answers'] # visuals = net_inputs['visual'] script = net_inputs['filtered_sub'] _, pred_idx = y_pred.max(dim=1) for i in range(batch_size): targ = target[i].item() pred = pred_idx[i].item() ans = ['\tans %d: ' % j + ' '.join(indices_to_words(answers[i][j], vocab)) for j in range(5)] if targ != pred: ans[targ] = colored(ans[targ], 'green') ans[pred] = colored(ans[pred], 'red') print('QA', sample_count) print('\tque:', *indices_to_words(que[i], vocab)) print('script:', *indices_to_words(script[i], vocab)) print(*ans, sep='\n') print('\tcorrect_idx:', targ) print('\tprediction:', pred) # print('\tvisual:') # for vis in visuals[i]: # print('\t\tspeaker: %s, behavior: %s, emotion: %s' % visual_to_words(vis, vocab)) sample_count += 1 print() ''' return loss.item( ), stats, batch_size, y_pred, target # TODO: add false_answer metric
def _inference(evaluator, batch): model.eval() with torch.no_grad(): new_qids = batch["new_qid"] real_ans = batch["correct_idx"] net_inputs, _ = prepare_batch(args, batch, model.vocab) y_pred = model(**net_inputs) y_pred = y_pred.argmax(dim=-1) # + 1 # 0~4 -> 1~5 for new_qid, ans, r_ans in zip(new_qids, y_pred, real_ans): engine.answers[new_qid] = ans.item() if ans.item() == r_ans: engine.count += 1 return
def interactive(args): args, model, iters, vocab, ckpt_available = get_model_ckpt(args) vid = args.get('vid', 's02e09_04_153') question = args.get('question', 'What does Monica place on the tablo?') batch = get_interactive_batch(args, vid, question, iters) with torch.no_grad(): net_inputs, target = prepare_batch(args, batch, model.vocab) y_pred = model(**net_inputs) top_1 = y_pred.argmax(dim=-1).item() top_0_ans = net_inputs['answers'].squeeze(0)[top_1] ans = to_sent(vocab, top_0_ans) return ans
def train(self, epoch): for i in range(epoch): for step, batch in enumerate(self.data.train_iter): (x, lengths), l, l_ = prepare_batch(batch) gen_logit, dis_logit = self.model((x, lengths), l, l_) adv_loss = self._discriminator_step(dis_logit) recon_loss = self._generator_step(gen_logit, dis_logit, (x, lengths)) # QUESTION: view on loss - (recon+adv) or (gen+dis)? if step % 100 == 0: msg = 'loss at epoch {}, step {}: {:.2f}'.format( i, step,self.lambda_ * adv_loss + recon_loss) logger.info(msg) # TODO: implement evaluation(inference) if step % 1000 == 0: self.evaluate()
def evaluate(self): self.data.valid_iter.shuffle = True import random a = random.randint(0, len(self.data.valid_iter)) # temp test for i, batch in enumerate(self.data.valid_iter): if i != a: continue (x, lengths), l, l_ = prepare_batch(batch) generated = self.model((x, lengths), l, l_, is_gen=True) print('=' * 50) print('original \t\t -> \t\t changed') for idx in random.sample(range(lengths.size(0)), 5): ori = reverse(x, self.data.vocab)[idx] chg = reverse(generated[0], self.data.vocab)[idx] print(' '.join(ori)) print('\t\t->', ' '.join(chg)) print('=' * 50) return
def infer_batch(seqs, tfserver): """ Returns 3d then 8d """ seq_arr = prepare_batch(seqs) # reshape for request seq_arr = seq_arr.astype(int) seq_arr = seq_arr.tolist() seq_arr = [s for s in seq_arr] # send data to tf server payload = {"inputs": {"input_seq_batch": seq_arr}} r = requests.post(tfserver, json=payload) pred = json.loads(r.content.decode('utf-8')) pred = infer_sgi.generate(pred['outputs']) return pred
def check_dataloader(self, **kwargs): from dataloader.dataset_multichoice import modes from utils import prepare_batch from tqdm import tqdm args = self._default_args(**kwargs) iters, vocab = get_iterator(args) train_iter_test = next(iter(iters['train'])) for key, value in train_iter_test.items(): if isinstance(value, torch.Tensor): print(key, value.shape) else: print(key, value) for mode in modes: print('Test loading %s data' % mode) for batch in tqdm(iters[mode]): # import ipdb; ipdb.set_trace() # XXX DEBUG batch = prepare_batch(args, batch, vocab)
def run_rep_exp(path, model, loss, tasks, device, ways, shots, rep_params=default_params, features=None): rep_path = path + '/rep_exp' if os.path.exists(rep_path): ans = input('Overriding previous results! Are you sure? (y/n) ') if ans == 'n': exit(0) else: os.mkdir(rep_path) # Ignore labels sanity_batch, _ = tasks.sample() sanity_batch = sanity_batch.to(device) # An instance of the model before adaptation init_model = model.clone() adapt_model = model.clone() init_rep_sanity = get_rep_from_batch(init_model, sanity_batch) # column 0: adaptation results, column 1: init results acc_results = np.zeros((rep_params['n_tasks'], 2)) # Create a dictionary of layer : results for each metric (e.g cca_results["0"] = [0.3, 0.2, 0.1]) cca_results = {str(layer): [] for layer in rep_params['layers']} cka_l_results = {str(layer): [] for layer in rep_params['layers']} cka_k_results = {str(layer): [] for layer in rep_params['layers']} for task in tqdm(range(rep_params['n_tasks']), desc="Tasks"): batch = tasks.sample() adapt_d, adapt_l, eval_d, eval_l = prepare_batch( batch, shots, ways, device) # Adapt the model for step in range(rep_params['adapt_steps']): train_error = loss(adapt_model(adapt_d), adapt_l) train_error /= len(adapt_d) adapt_model.adapt(train_error) # Evaluate the adapted model a_predictions = adapt_model(eval_d) a_valid_acc = accuracy(a_predictions, eval_l) # Evaluate the init model i_predictions = init_model(eval_d) i_valid_acc = accuracy(i_predictions, eval_l) acc_results[task, 0] = a_valid_acc acc_results[task, 1] = i_valid_acc # Get their representations for every layer for layer in cca_results.keys(): adapted_rep_i = get_rep_from_batch(adapt_model, adapt_d, int(layer)) init_rep_i = get_rep_from_batch(init_model, adapt_d, int(layer)) cca_results[layer].append( get_cca_similarity(adapted_rep_i.T, init_rep_i.T, epsilon=1e-10)[1]) # NOTE: Currently CKA takes too long to compute so leave it out # cka_l_results[layer].append(get_linear_CKA(adapted_rep_i, init_rep_i)) # cka_k_results[layer].append(get_kernel_CKA(adapted_rep_i, init_rep_i)) # Average and calculate standard deviation cca_mean = [] cca_std = [] for layer, values in cca_results.items(): mean = statistics.mean(values) std = statistics.stdev(values) cca_mean.append(mean) cca_std.append(std) cca_plot = dict(title="CCA Evolution", x_legend="Inner loop steps", y_legend="CCA similarity", y_axis=cca_results, path=path + "/inner_CCA_evolution.png") cka_l_plot = dict(title="Linear CKA Evolution", x_legend="Inner loop steps", y_legend="CKA similarity", y_axis=cka_l_results, path=path + "/inner_Linear_CKA_evolution.png") cka_k_plot = dict(title="Kernel CKA Evolution", x_legend="Inner loop steps", y_legend="CKA similarity", y_axis=cka_k_results, path=path + "/inner_Kernel_CKA_evolution.png") # plot_dict(cca_plot, save=True) # plot_dict(cka_l_plot, save=True) # plot_dict(cka_k_plot, save=True) with open(rep_path + '/rep_params.json', 'w') as fp: json.dump(rep_params, fp, sort_keys=True, indent=4) with open(rep_path + '/cca_results.json', 'w') as fp: json.dump(cca_results, fp, sort_keys=True, indent=4) x_axis = [] for i in cca_results.keys(): if int(i) > 0: x_axis.append(f'Conv{i}') else: x_axis.append('Head') cca_plot_2 = dict(title="Layer-wise changes before / after adaptation", x_legend="Layer", y_legend="CCA similarity", x_axis=x_axis, y_axis=cca_mean, std=cca_std) cka_plot_2 = dict(title="CKA Evolution layer-wise", x_legend="Layer", y_legend="CKA similarity", x_axis=list(cka_l_results.keys()), y_axis=list(cka_l_results.values())) plot_dict_explicit(cca_plot_2) return cca_results
def main(): # parse options parser = TrainOptions() opts = parser.parse() # data loader print('--- load parameter ---') # outer_iter = opts.outer_iter # fade_iter = max(1.0, float(outer_iter / 2)) epochs = opts.epoch batchsize = opts.batchsize # datasize = opts.datasize # datarange = opts.datarange augementratio = opts.augementratio centercropratio = opts.centercropratio # model print('--- create model ---') tetGAN = TETGAN(gpu=(opts.gpu != 0)) if opts.gpu != 0: tetGAN.cuda() tetGAN.init_networks(weights_init) num_params = 0 for param in tetGAN.parameters(): num_params += param.numel() print('Total number of parameters in TET-GAN: %.3f M' % (num_params / 1e6)) print('--- training ---') texture_class = 'base_gray_texture' in opts.dataset_class or 'skeleton_gray_texture' in opts.dataset_class if texture_class: tetGAN.load_state_dict(torch.load(opts.model)) dataset_path = os.path.join(opts.train_path, opts.dataset_class, 'style') val_dataset_path = os.path.join(opts.train_path, opts.dataset_class, 'val') if 'base_gray_texture' in opts.dataset_class: few_size = 6 batchsize = 2 epochs = 1500 elif 'skeleton_gray_texture' in opts.dataset_class: few_size = 30 batchsize = 10 epochs = 300 fnames = load_trainset_batchfnames_dualnet(dataset_path, batchsize, few_size=few_size) val_fnames = sorted(os.listdir(val_dataset_path)) style_fnames = sorted(os.listdir(dataset_path)[:few_size]) else: dataset_path = os.path.join(opts.train_path, opts.dataset_class, 'train') fnames = load_trainset_batchfnames_dualnet(dataset_path, batchsize) tetGAN.train() train_size = os.listdir(dataset_path) print('List of %d styles:' % (len(train_size))) result_dir = os.path.join(opts.result_dir, opts.dataset_class) if not os.path.exists(result_dir): os.mkdir(result_dir) for epoch in range(epochs): for idx, fname in enumerate(fnames): x, y_real, y = prepare_batch(fname, 1, 1, centercropratio, augementratio, opts.gpu) losses = tetGAN.one_pass(x[0], None, y[0], None, y_real[0], None, 1, 0) if (idx + 1) % 100 == 0: print('Epoch [%d/%d], Iter [%d/%d]' % (epoch + 1, epochs, idx + 1, len(fnames))) print( 'Lrec: %.3f, Ldadv: %.3f, Ldesty: %.3f, Lsadv: %.3f, Lsty: %.3f' % (losses[0], losses[1], losses[2], losses[3], losses[4])) if texture_class and ((epoch + 1) % (epochs / 20)) == 0: outname = 'save/' + 'val_epoch' + str( epoch + 1) + '_' + opts.dataset_class + '_' + opts.save_model_name print('--- save model Epoch [%d/%d] ---' % (epoch + 1, epochs)) torch.save(tetGAN.state_dict(), outname) print('--- validating model [%d/%d] ---' % (epoch + 1, epochs)) for val_idx, val_fname in enumerate(val_fnames): v_fname = os.path.join(val_dataset_path, val_fname) random.shuffle(style_fnames) sty_fname = style_fnames[0] s_fname = os.path.join(dataset_path, sty_fname) with torch.no_grad(): val_content = load_image_dualnet(v_fname, load_type=1) val_sty = load_image_dualnet(s_fname, load_type=0) if opts.gpu != 0: val_content = val_content.cuda() val_sty = val_sty.cuda() result = tetGAN(val_content, val_sty) if opts.gpu != 0: result = to_data(result) result_filename = os.path.join( result_dir, str(epoch) + '_' + val_fname) print(result_filename) save_image(result[0], result_filename) elif not texture_class and ((epoch + 1) % 2) == 0: outname = 'save/' + 'epoch' + str(epoch + 1) + '_' + opts.save_model_name print('--- save model ---') torch.save(tetGAN.state_dict(), outname)
def main(): # parse options parser = FinetuneOptions() opts = parser.parse() # data loader print('--- load parameter ---') outer_iter = opts.outer_iter epochs = opts.epoch batchsize = opts.batchsize datasize = opts.datasize stylename = opts.style_name # model print('--- create model ---') tetGAN = TETGAN(gpu=(opts.gpu != 0)) if opts.gpu != 0: tetGAN.cuda() tetGAN.load_state_dict(torch.load(opts.load_model_name)) tetGAN.train() print('--- training ---') # supervised one shot learning if opts.supervise == 1: for i in range(outer_iter): fnames = load_oneshot_batchfnames(stylename, batchsize, datasize) for epoch in range(epochs): for fname in fnames: x, y_real, y = prepare_batch(fname, 3, 0, 0, 0, opts.gpu) losses = tetGAN.one_pass(x[0], None, y[0], None, y_real[0], None, 3, 0) print('Iter[%d/%d], Epoch [%d/%d]' % (i + 1, outer_iter, epoch + 1, epochs)) print( 'Lrec: %.3f, Ldadv: %.3f, Ldesty: %.3f, Lsadv: %.3f, Lsty: %.3f' % (losses[0], losses[1], losses[2], losses[3], losses[4])) # unsupervised one shot learning else: for i in range(outer_iter): fnames = load_oneshot_batchfnames(stylename, batchsize, datasize) for epoch in range(epochs): for fname in fnames: # no ground truth x provided _, y_real, _ = prepare_batch(fname, 3, 0, 0, 0, opts.gpu) Lsrec = tetGAN.update_style_autoencoder(y_real[0]) for fname in fnames: # no ground truth x provided _, y_real, y = prepare_batch(fname, 3, 0, 0, 0, opts.gpu) with torch.no_grad(): x_auxiliary = tetGAN.desty_forward(y_real[0]) losses = tetGAN.one_pass(x_auxiliary, None, y[0], None, y_real[0], None, 3, 0) print('Iter[%d/%d], Epoch [%d/%d]' % (i + 1, outer_iter, epoch + 1, epochs)) print( 'Lrec: %.3f, Ldadv: %.3f, Ldesty: %.3f, Lsadv: %.3f, Lsty: %.3f, Lsrec: %.3f' % (losses[0], losses[1], losses[2], losses[3], losses[4], Lsrec)) print('--- save ---') torch.save(tetGAN.state_dict(), opts.save_model_name)
def run_cl_exp(path, maml, loss, tasks, device, ways, shots, cl_params=default_params, features=None): cl_path = path + '/cl_exp' if os.path.exists(cl_path): ans = input('Overriding previous results! Are you sure? (y/n)') if ans == 'n': exit(0) else: os.mkdir(cl_path) # Randomly select some batches for training and evaluation tasks_pool = [] for task_i in range(cl_params['n_tasks']): batch = tasks.sample() adapt_d, adapt_l, eval_d, eval_l = prepare_batch(batch, shots, ways, device, features=features) task = {'adapt': (adapt_d, adapt_l)} if setting == 1: task['eval'] = (adapt_d, adapt_l) else: task['eval'] = (eval_d, eval_l) tasks_pool.append(task) # Matrix R NxN of accuracies in tasks j after trained on a tasks i (x_axis = test tasks, y_axis = train tasks) acc_matrix = np.zeros((cl_params['n_tasks'], cl_params['n_tasks'])) # Training loop for i, task_i in enumerate(tasks_pool): adapt_i_data, adapt_i_labels = task_i['adapt'] adapt_i_data, adapt_i_labels = adapt_i_data.to( device), adapt_i_labels.to(device) learner = maml.clone() # Adapt to task i for step in range(cl_params['adapt_steps']): train_error = loss(learner(adapt_i_data), adapt_i_labels) learner.adapt(train_error) # Evaluation loop for j, task_j in enumerate(tasks_pool): eval_j_data, eval_j_labels = task_j['eval'] eval_j_data, eval_j_labels = eval_j_data.to( device), eval_j_labels.to(device) predictions = learner(eval_j_data) valid_accuracy_j = accuracy(predictions, eval_j_labels) acc_matrix[ i, j] = valid_accuracy_j # Accuracy on task j after trained on task i cl_res = calc_cl_metrics(acc_matrix) save_acc_matrix(cl_path, acc_matrix) with open(cl_path + '/cl_params.json', 'w') as fp: json.dump(cl_params, fp, sort_keys=True, indent=4) with open(cl_path + '/cl_res.json', 'w') as fp: json.dump(cl_res, fp, sort_keys=True, indent=4) return acc_matrix, cl_res
def main(): # parse options parser = TrainOptions() opts = parser.parse() # data loader print('--- load parameter ---') outer_iter = opts.outer_iter fade_iter = max(1.0, float(outer_iter / 2)) epochs = opts.epoch batchsize = opts.batchsize datasize = opts.datasize datarange = opts.datarange augementratio = opts.augementratio centercropratio = opts.centercropratio # model print('--- create model ---') tetGAN = TETGAN(gpu=(opts.gpu != 0)) if opts.gpu != 0: tetGAN.cuda() tetGAN.init_networks(weights_init) tetGAN.train() print('--- training ---') stylenames = os.listdir(opts.train_path) print('List of %d styles:' % (len(stylenames)), *stylenames, sep=' ') if opts.progressive == 1: # proressive training. From level1 64*64, to level2 128*128, to level3 256*256 # level 1 for i in range(outer_iter): jitter = min(1.0, i / fade_iter) fnames = load_trainset_batchfnames(opts.train_path, batchsize * 4, datarange, datasize * 2) for epoch in range(epochs): for fname in fnames: x, y_real, y = prepare_batch(fname, 1, jitter, centercropratio, augementratio, opts.gpu) losses = tetGAN.one_pass(x[0], None, y[0], None, y_real[0], None, 1, None) print('Level1, Iter[%d/%d], Epoch [%d/%d]' % (i + 1, outer_iter, epoch + 1, epochs)) print( 'Lrec: %.3f, Ldadv: %.3f, Ldesty: %.3f, Lsadv: %.3f, Lsty: %.3f' % (losses[0], losses[1], losses[2], losses[3], losses[4])) # level 2 for i in range(outer_iter): w = max(0.0, 1 - i / fade_iter) fnames = load_trainset_batchfnames(opts.train_path, batchsize * 2, datarange, datasize * 2) for epoch in range(epochs): for fname in fnames: x, y_real, y = prepare_batch(fname, 2, 1, centercropratio, augementratio, opts.gpu) losses = tetGAN.one_pass(x[0], x[1], y[0], y[1], y_real[0], y_real[1], 2, w) print('Level2, Iter[%d/%d], Epoch [%d/%d]' % (i + 1, outer_iter, epoch + 1, epochs)) print( 'Lrec: %.3f, Ldadv: %.3f, Ldesty: %.3f, Lsadv: %.3f, Lsty: %.3f' % (losses[0], losses[1], losses[2], losses[3], losses[4])) # level 3 for i in range(outer_iter): w = max(0.0, 1 - i / fade_iter) fnames = load_trainset_batchfnames(opts.train_path, batchsize, datarange, datasize) for epoch in range(epochs): for fname in fnames: x, y_real, y = prepare_batch(fname, 3, 1, centercropratio, augementratio, opts.gpu) losses = tetGAN.one_pass(x[0], x[1], y[0], y[1], y_real[0], y_real[1], 3, w) print('Level3, Iter[%d/%d], Epoch [%d/%d]' % (i + 1, outer_iter, epoch + 1, epochs)) print( 'Lrec: %.3f, Ldadv: %.3f, Ldesty: %.3f, Lsadv: %.3f, Lsty: %.3f' % (losses[0], losses[1], losses[2], losses[3], losses[4])) else: # directly train on level3 256*256 for i in range(outer_iter): fnames = load_trainset_batchfnames(opts.train_path, batchsize, datarange, datasize) for epoch in range(epochs): for fname in fnames: x, y_real, y = prepare_batch(fname, 3, 1, centercropratio, augementratio, opts.gpu) losses = tetGAN.one_pass(x[0], None, y[0], None, y_real[0], None, 3, 0) print('Iter[%d/%d], Epoch [%d/%d]' % (i + 1, outer_iter, epoch + 1, epochs)) print( 'Lrec: %.3f, Ldadv: %.3f, Ldesty: %.3f, Lsadv: %.3f, Lsty: %.3f' % (losses[0], losses[1], losses[2], losses[3], losses[4])) print('--- save ---') torch.save(tetGAN.state_dict(), opts.save_model_name)
def _inference(evaluator, batch): nonlocal sample_count model.eval() with torch.no_grad(): net_inputs, target = prepare_batch(args, batch, model.module.vocab) #net_inputs, target = prepare_batch(args, batch, model.vocab) if net_inputs['subtitle'].nelement() == 0: import ipdb ipdb.set_trace() # XXX DEBUG y_pred, char_pred, mask_pred = model(**net_inputs) batch_size = y_pred.shape[0] """ # get person ground truth and compute character loss n_char = 21 visual_char = net_inputs['filtered_visual'].view(batch_size, -1, 3)[:,:,0] char_target = visual_char.unsqueeze(2).view(batch_size, -1) char_target = char_target.view(-1) char_pred = char_pred.view(-1, n_char) character_loss = nn.CrossEntropyLoss().cuda()(char_pred, char_target) # get ground truth labels and compute MLM loss vocab_size = mask_pred.size(-1) mask_target = net_inputs['labels'] mask_target = mask_target.view(-1) mask_pred = mask_pred.view(-1, vocab_size) mlm_loss = nn.CrossEntropyLoss(ignore_index=-1).cuda()(mask_pred, mask_target) """ # compute QA loss loss, stats = loss_fn(y_pred, target) # compute total loss #loss = loss + character_loss + mlm_loss #vocab = model.vocab ''' if sample_count < 100: print('Batch %d: data and prediction from %d to %d' % (sample_count // batch_size, sample_count, sample_count + batch_size - 1)) que = net_inputs['que'] answers = net_inputs['answers'] # visuals = net_inputs['visual'] script = net_inputs['filtered_sub'] _, pred_idx = y_pred.max(dim=1) for i in range(batch_size): targ = target[i].item() pred = pred_idx[i].item() ans = ['\tans %d: ' % j + ' '.join(indices_to_words(answers[i][j], vocab)) for j in range(5)] if targ != pred: ans[targ] = colored(ans[targ], 'green') ans[pred] = colored(ans[pred], 'red') print('QA', sample_count) print('\tque:', *indices_to_words(que[i], vocab)) print('script:', *indices_to_words(script[i], vocab)) print(*ans, sep='\n') print('\tcorrect_idx:', targ) print('\tprediction:', pred) # print('\tvisual:') # for vis in visuals[i]: # print('\t\tspeaker: %s, behavior: %s, emotion: %s' % visual_to_words(vis, vocab)) sample_count += 1 print() ''' return loss.item( ), stats, batch_size, y_pred, target # TODO: add false_answer metric
def train_loop( run_id, dataset_dir, ckpt_run_dir, output_dir, validation_only=False, use_cuda=False, light_target=False, ): """Train loop""" if torch.cuda.is_available(): torch.cuda.empty_cache() rank = dist.get_rank() world_size = dist.get_world_size() train_epochs = 8 train_min_len, train_max_len = 0, 75 val_min_len, val_max_len = 0, 150 math_mode = "fp16" # One of `fp16`, `fp32` lang = ("en", "de") # Training train_global_batch_size = 2048 # Global batch size max_bs = 128 # Max batch size for used hardware update_freq = int(max(1, train_global_batch_size // (max_bs * world_size))) train_batch_size = int(train_global_batch_size // (world_size * update_freq)) val_batch_size = 64 # Model attributes model_args = { "hidden_size": 1024, "num_layers": 4, "dropout": 0.2, "share_embedding": True, "fusion": True, } # Criterion criterion_args = {"smoothing": 0.1, "fast_xentropy": True} # Loss scaling loss_scaling = {"init_scale": 1024, "upscale_interval": 128} # Optimizer optimizer_args = { "lr": 2e-3, "grad_clip": 5.0, } # Scheduler scheduler_args = { "warmup_steps": 200, "remain_steps": 0.4, "decay_interval": 0.05, "decay_steps": 4, "decay_factor": 0.5, } # Translator translator_args = { "beam_size": 5, "len_norm_factor": 0.6, "cov_penalty_factor": 0.1, "len_norm_const": 5.0, "max_seq_len": 150, } # Build train/val datsets train_set = WMT16Dataset( dataset_dir, math_precision=math_mode, lang=lang, train=True, download=True, preprocessed=True, min_len=train_min_len, max_len=train_max_len, ) train_set.prepare() val_set = WMT16Dataset( dataset_dir, math_precision=math_mode, lang=lang, validation=True, download=False, min_len=val_min_len, max_len=val_max_len, sort=True, ) tokenizer = train_set.tokenizer # Build model model = GNMT(vocab_size=train_set.vocab_size, **model_args) # Build loss function criterion = LabelSmoothing(padding_idx=wmt16_config.PAD, **criterion_args) # Bilingual Evaluation Understudy Score metrics = [BLEUScore()] # Partition data train_set = partition_dataset_by_rank(train_set, rank, world_size) val_set = partition_dataset_by_rank(val_set, rank, world_size) collate_fn = build_collate_fn(sort=True) train_loader = DataLoader( train_set, batch_size=train_batch_size, collate_fn=collate_fn, num_workers=2, pin_memory=True, drop_last=False, shuffle=True, ) val_loader = DataLoader( val_set, batch_size=val_batch_size, collate_fn=collate_fn, num_workers=2, pin_memory=True, drop_last=False, ) validate_every = update_freq * round( len(train_loader) * 0.30 / update_freq ) # Validate every 30% # Build optimizer & scheduler total_train_iters = (len(train_loader) // update_freq) * train_epochs print("Number of batches per epoch {}".format(len(train_loader))) print("Train iterations per epoch {}".format(total_train_iters / train_epochs)) if use_cuda: model = model.cuda() criterion = criterion.cuda() use_horovod = math_mode == "fp16" and dist.get_backend() == dist.Backend.MPI if use_horovod: hvd.init() logger.info("Using horovod rank={}".format(hvd.rank())) tensor = torch.tensor([1]) res = hvd.allreduce(tensor, op=hvd.Sum) assert res[0] == world_size fp_optimizer, optimizer, model = build_optimizer( model=model, math=math_mode, loss_scaling=loss_scaling, use_cuda=use_cuda, use_horovod=use_horovod, **optimizer_args ) # Create a learning rate scheduler for an optimizer scheduler = ExponentialWarmupMultiStepLR( optimizer, total_train_iters, **scheduler_args ) # Translator translator = Translator(model=model, trg_tokenizer=tokenizer, **translator_args) checkpointer = Checkpointer( ckpt_run_dir=ckpt_run_dir, rank=rank, freq=CheckpointFreq.BEST ) if not validation_only: if light_target: goal = task4_time_to_bleu_goal(20) else: goal = task4_time_to_bleu_goal(24) num_batches_per_device_train = len(train_loader) tracker = Tracker(metrics, run_id, rank, goal=goal) dist.barrier() tracker.start() for epoch in range(0, train_epochs): if torch.cuda.is_available(): torch.cuda.empty_cache() model.train() tracker.train() for batch_idx, (data, target) in enumerate(train_loader): tracker.batch_start() data, target = prepare_batch(data, target, use_cuda=use_cuda) tracker.record_batch_load() is_last = batch_idx == len(train_loader) update = (batch_idx % update_freq) == update_freq - 1 init = (batch_idx % update_freq) == 0 # Clear gradients in the optimizer. if init: fp_optimizer.zero_grad() tracker.record_batch_init() # Compute the output output = compute_model_output(model, data, target) tracker.record_batch_fwd_pass() # Compute the loss loss, loss_per_token = compute_loss( data, target, output, criterion, update_freq ) tracker.record_batch_comp_loss() # Backprop fp_optimizer.backward_loss(loss) tracker.record_batch_backprop() # Opt step if update or is_last: # For this task, simply sum all gradients updated = fp_optimizer.step(tracker=tracker, denom=1) # Learning rate scheduler if updated: scheduler.step() tracker.batch_end() record_train_batch_stats( batch_idx=batch_idx, loss=loss_per_token, output=target[0], # Use target just for the size metric_results={}, tracker=tracker, num_batches_per_device_train=num_batches_per_device_train, ) # Validation during training if (batch_idx + 1) % validate_every == 0: if torch.cuda.is_available(): torch.cuda.empty_cache() metrics_values, loss = validation_round( val_loader, metrics, model, criterion, update_freq, translator, tracker=tracker, use_cuda=use_cuda, ) record_validation_stats(metrics_values, loss, tracker, rank) if tracker.goal_reached: break model.train() tracker.train() if torch.cuda.is_available(): torch.cuda.empty_cache() metrics_values, loss = validation_round( val_loader, metrics, model, criterion, update_freq, translator, use_cuda=use_cuda, ) is_best = record_validation_stats(metrics_values, loss, tracker, rank) checkpointer.save( tracker, model, fp_optimizer.optimizer, scheduler, tracker.current_epoch, is_best, ) tracker.epoch_end() if tracker.goal_reached: print("Goal Reached!") dist.barrier() time.sleep(10) return else: cecf = CheckpointsEvaluationControlFlow( ckpt_dir=ckpt_run_dir, rank=rank, world_size=world_size, checkpointer=checkpointer, model=model, epochs=train_epochs, loss_function=criterion, metrics=metrics, use_cuda=use_cuda, dtype="fp32", max_batch_per_epoch=None, ) train_stats = cecf.evaluate_by_epochs(train_loader) with open(os.path.join(output_dir, "train_stats.json"), "w") as f: json.dump(train_stats, f) val_stats = cecf.evaluate_by_epochs(val_loader) with open(os.path.join(output_dir, "val_stats.json"), "w") as f: json.dump(val_stats, f)
def optimiser(idx, shared_model, SIMULATOR, args, lock): try: writer = SummaryWriter('runs/{}/optimiser:{:02}'.format(datetime.now().strftime("%d|%m_%H|%M"), idx)) logging.basicConfig(filename='logs/optimiser:{:02}.log'.format(idx), filemode='w', format='%(message)s', level=logging.DEBUG) sgd = t.optim.SGD(params=shared_model.parameters(), lr=args.lr) # allocate a device n_gpu = t.cuda.device_count() if n_gpu > 0: Device.set_device(idx % n_gpu) q_network = deepcopy(shared_model) q_network.to(Device.get_device()) q_network.train() target_network = deepcopy(q_network) target_network.to(Device.get_device()) target_network.eval() buffer = deque(maxlen=args.buffer_size) simulator = SIMULATOR() for itr in tqdm(count(), position=idx, desc='optimiser:{:02}'.format(idx)): state = simulator.reset() episode_reward = 0 for e in count(): if np.random.RandomState().rand() < max(args.eps ** itr, args.min_eps): action = np.random.RandomState().randint(simulator.n_actions()) else: action = q_network(as_tensor(state)).argmax().item() next_state, reward, terminal = simulator.step(action) buffer.append(transition_to_tensor(state, action, reward, next_state, terminal)) episode_reward += reward state = next_state # Sample a data point from dataset batch = prepare_batch(buffer, args.batch_size) # Sync local model with shared model q_network.load_state_dict(shared_model.state_dict()) # Calculate loss for the batch loss = calculate_loss(q_network, target_network, batch, args) # Optimise for the batch loss = optimise_model(shared_model, q_network, loss, sgd, args, lock) # Log the results logging.debug('Batch loss: {:.2f}'.format(loss)) writer.add_scalar('batch/loss', loss, e) if terminal: break logging.debug('Episode reward: {:.2f}'.format(episode_reward)) writer.add_scalar('episode_reward', episode_reward, itr) writer.close() if itr % args.target_update_frequency == 0: target_network.load_state_dict(q_network.state_dict()) except KeyboardInterrupt: print('exiting optimiser:{:02}'.format(idx))