def __init__(self, config): self.params = config self.device = device self.word2index, self.index2word, self.embeddings = pickle.load(open(config.data_pickle, 'rb')) train_dataset = Loader(config, config.p_train_data) val_dataset = Loader(config, config.p_val_data) test_dataset = Loader(config, config.p_test_data) self.model = Model(config, self.embeddings).to(self.device) self.train_loader = DataLoader(dataset=train_dataset, batch_size=config.batch_size, shuffle=True) self.val_loader = DataLoader(dataset=val_dataset, batch_size=config.batch_size, shuffle=False) self.test_loader = DataLoader(dataset=test_dataset, batch_size=config.batch_size, shuffle=False) params = filter(lambda param: param.requires_grad, self.model.parameters()) self.optimizer = torch.optim.Adam(lr=config.learning_rate, betas=(config.beta1, config.beta2), eps=1e-7, weight_decay=3e-7, params=params) # lr = config.learning_rate # base_lr = 1.0 # params = filter(lambda param: param.requires_grad, slef.model.parameters()) # optimizer = torch.optim.Adam(lr=base_lr, betas=(config.beta1, config.beta2), eps=1e-7, weight_decay=3e-7, params=params) # cr = lr / math.log2(config.lr_warm_up_num) # scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, # lr_lambda=lambda ee: cr * math.log2(ee + 1) if ee < config.lr_warm_up_num else lr) self.model_path = os.path.join(self.params.cache_dir) if not os.path.exists(self.model_path): print('create path: ', self.model_path) os.makedirs(self.model_path) self.best_model = None self.lr_epoch = 0
def main(): parser = argparse.ArgumentParser() parser.add_argument("--pmb") parser.add_argument("--sick") parser.add_argument("--sick2pd") parser.add_argument("--out") args = parser.parse_args() # Final model data = Loader.load_data("../NLI2FOLI/SICK/SICK_train.txt") data = data.append(Loader.load_data("../NLI2FOLI/SICK/SICK_trial.txt")) test = Loader.load_data(args.sick) data["postags"] = FeatureExtractor.postag_tokenizer(data["tokens"]) test["postags"] = FeatureExtractor.postag_tokenizer(test["tokens"]) # The countvectorizer features with postagging appended to each token bag_of_words_plus_pos = ColumnTransformer([ ("POS", CountVectorizer(tokenizer=lambda x: x, preprocessor=lambda x: x), "postags") ]) m = Model(data) m.add_feature(bag_of_words_plus_pos, "Feature") m.train_model( RandomForestClassifier(n_estimators=900, criterion="entropy", max_depth=729)) with open('model.pkl', 'wb') as fid: pickle.dump(m, fid) with open('model.pkl', 'rb') as fid: m = pickle.load(fid) try: m.test_model(test, test["entailment_judgment"]) labels = m.model.classes_ cm = confusion_matrix(test["entailment_judgment"], m.prediction) print_cm(cm, labels) except: m.test_model(test) with open(args.out, "w") as file: for idx, pred in enumerate(m.prediction): pid = test.iloc[idx]['pair_ID'] file.write("{}:{}\n".format(pid, pred)) print("Successfully generated prediction on test data.")
def makecsv(file, model, loadfile): cuda = False kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {} data_loader = Loader(c.FILE_TRAIN_LABELED_AUG, c.FILE_TRAIN_UNLABELED, c.FILE_TEST, 'data/test-labeled.p', kwargs) test_loader = data_loader.getTest() test_actual = data_loader.getValidation() label_predict = np.array([]) mnist_model = model if loadfile: mnist_model = torch.load(model) correct = 0 for data, target in test_loader: mnist_model.eval() data, target = Variable(data, volatile=True), Variable(target) output = mnist_model(data) temp = output.data.max(1)[1] pred = output.data.max(1)[1] correct += pred.eq(target.data).cpu().sum() label_predict = np.concatenate((label_predict, temp.numpy().reshape(-1))) print('\nTest set: Accuracy: {}/{} ({:.0f}%)\n'.format(correct, len(test_loader.dataset), 100. * correct / len(test_loader.dataset))) predict_label = pd.DataFrame(label_predict, columns=['label'], dtype=int) predict_label.reset_index(inplace=True) predict_label.rename(columns={'index': 'ID'}, inplace=True) filename = 'predictions/' + file + "-labeled.csv" predict_label.to_csv(filename, index=False) label_predict = np.array([]) correct = 0 for data, target in test_actual: mnist_model.eval() data, target = Variable(data, volatile=True), Variable(target) output = mnist_model(data) temp = output.data.max(1)[1] pred = output.data.max(1)[1] # get the index of the max log-probability correct += pred.eq(target.data).cpu().sum() label_predict = np.concatenate((label_predict, temp.numpy().reshape(-1))) print('\nTest set: Accuracy: {}/{} ({:.0f}%)\n'.format(correct, len(test_loader.dataset), 100. * correct / len(test_loader.dataset))) predict_label = pd.DataFrame(label_predict, columns=['label'], dtype=int) predict_label.reset_index(inplace=True) predict_label.rename(columns={'index': 'ID'}, inplace=True) filename = 'predictions/' + file + "-unlabeled.csv" predict_label.to_csv(filename, index=False)
def getDataLoaderDict(idxModel): # idxModel is tuple print("\nStarted Preprocessing") trainDataLoaderDict = {} testDataLoaderDict = {} for idx in idxModel: print("-Preprocessing model{} dataloader".format(idx)) loader = Loader(idx) trainLoader, testLoader = loader.getLoader() trainDataLoaderDict[idx] = trainLoader testDataLoaderDict[idx] = testLoader print("-Completed Preprocessing model{} dataloader\n".format(idx)) print("All Completed Preprocessing\n") return trainDataLoaderDict, testDataLoaderDict
def main(_): if config.mode == "train": trainer = Trainer(config) trainer.train() elif config.mode == "preprocess": dataloader.preprocess(_) elif config.mode == "debug": trainer = Trainer(config) train_dataset = Loader(config, config.p_train_data) train_loader = DataLoader(dataset=train_dataset, batch_size=2, shuffle=False) data_iter = iter(train_loader) frame_vecs, frame_n, ques, ques_n, start_frame, end_frame = data_iter.next() frame_vecs = frame_vecs.to(trainer.device) ques = ques.to(trainer.device) # Forward pass p1, p2 = trainer.model(frame_vecs, ques) y1, y2 = start_frame.to(trainer.device), end_frame.to(trainer.device) print(p1.shape,p2.shape,y1.shape,y2.shape) loss1 = F.nll_loss(p1, y1, reduction='elementwise_mean') loss2 = F.nll_loss(p2, y2, reduction='elementwise_mean') loss = (loss1 + loss2) / 2 # Backward and optimize trainer.optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_value_(trainer.model.parameters(), 1) trainer.optimizer.step() print(loss.item()) elif config.mode == "test": trainer = Trainer(config) trainer.evaluate() else: print("Unknown mode") exit(0)
def main(_): print 'begin loading data...' loader = Loader.loader(FLAGS.mincount, FLAGS.maxlen) print 'finished loading data\n' print 'begin building model...' model = Model( loader, embedding_size = FLAGS.embedding_size, batch_size = FLAGS.batch_size, max_epoch = FLAGS.max_epoch, learning_rate = FLAGS.learning_rate, keep_prob = FLAGS.keep_prob ) print 'finished building model\n' if FLAGS.retrain: model.train() test_accuracy = model.test() else: try: model.load('save', 'best') test_accuracy = model.test() except tf.errors.NotFoundError: model.train() test_accuracy = model.test() except Exception, e: print 'Error may be caused by changing parameters without retraining. Try setting --retrain to True.' raise e
def popularity_ratio(pop_model: np.ndarray, pop_model_user: np.ndarray, dataset: Loader): """calculate the degree of the "long-tailness" for a distribution Args: pop_model (ndarray): the freq of items recommended by model pop_model_user (ndarray): (user X topk) the recommend list of users dataset (dataloader.Loader): the freq of items in dataset Returns: dict: {"I_ratio""float, "I_KL":float, "I_gini":float, "APT":[], "I_bin": float} """ pop_dataset, _ = dataset.popularity() assert len(pop_model) == len(pop_dataset) num_item = len(pop_dataset) num_interaction = pop_model.sum() metrics = {} prop_model = pop_model / num_interaction prop_uniform = 1. / num_item prop_dataset = pop_dataset / pop_dataset.sum() # print("dataset KL", # np.sum(prop_dataset * np.log(prop_dataset / prop_uniform + 1e-7))) metrics['I_KL'] = np.sum(prop_model * np.log(prop_model / prop_uniform + 1e-7)) mapping = map_item_three(dataset.popularity()[0]) mapping_N = map_item_N((dataset.popularity()[0]), [0.1, 0.1, 0.3, 0.3, 0.2]) metrics['APT'] = APT(pop_model_user, mapping) metrics['APT5'] = APT(pop_model_user, mapping_N) # print("dataset APT", APT(dataset.allPos, mapping)) # print("dataset bin", np.sum(pop_dataset/pop_dataset.max())) metrics['I_bin'] = np.sum(pop_model / np.max(pop_model)) return metrics
def main(_): print 'begin loading data...' loader = Loader(FLAGS.mincount, FLAGS.maxlen, full_train=False) print 'finished loading data\n' print 'begin building model...' # dataloader, embedding_size, max_epoch, learning_rate, keep_prob model = Model(loader, embedding_size=FLAGS.embedding_size, max_epoch=FLAGS.max_epoch, learning_rate=FLAGS.learning_rate, keep_prob=FLAGS.keep_prob) print 'finished building model\n' # model.train(full_train=True) # model.save_doc_vector() model.many_test()
def main(): parser = Parser() config = parser.config for param, value in config.__dict__.items(): print(param + '.' * (50 - len(param) - len(str(value))) + str(value)) print() # Load previous checkpoint if it exists checkpoint = load_latest(config) # Create model model = load_model(config, checkpoint) # print number of parameters in the model n_params = sum([param.view(-1).size()[0] for param in model.parameters()]) print('Total number of parameters: \33[91m{}\033[0m'.format(n_params)) # Load train and test data train_loader, valid_loader, test_loader = Loader(config) n_batches = int(len(train_loader.dataset.train_data) / config.batch_size) # save the configuration with open(os.path.join(config.save, 'log.txt'), 'w') as file: json.dump('json_stats: ' + str(config.__dict__), file) # Instantiate the criterion, optimizer and learning rate scheduler criterion = torch.nn.CrossEntropyLoss(size_average=True) optimizer = torch.optim.SGD(model.parameters(), lr=config.LR, momentum=config.momentum, weight_decay=config.weight_decay, nesterov=config.nesterov) start_time = 0 if checkpoint is not None: start_epoch = checkpoint['time'] + 1 optimizer.load_state_dict(checkpoint['optimizer']) if config.lr_shape == 'multistep': scheduler = MultiStepLR(optimizer, milestones=[81, 122], gamma=0.1) elif config.lr_shape == 'cosine': if checkpoint is not None: scheduler = checkpoint['scheduler'] else: scheduler = CosineAnnealingRestartsLR(optimizer, 1, config.T_e, T_mul=config.T_mul) # The trainer handles the training loop and evaluation on validation set trainer = Trainer(model, criterion, config, optimizer, scheduler) epoch = 1 while True: # Train for a single epoch train_top1, train_loss, stop_training = trainer.train( epoch, train_loader) # Run model on the validation and test set valid_top1 = trainer.evaluate(epoch, valid_loader, 'valid') test_top1 = trainer.evaluate(epoch, test_loader, 'test') current_time = time.time() results = { 'epoch': epoch, 'time': current_time, 'train_top1': train_top1, 'valid_top1': valid_top1, 'test_top1': test_top1, 'train_loss': float(train_loss.data), } with open(os.path.join(config.save, 'results.txt'), 'w') as file: json.dump(str(results), file) file.write('\n') print( '==> Finished epoch %d (budget %.3f): %7.3f (train) %7.3f (validation) %7.3f (test)' % (epoch, config.budget, train_top1, valid_top1, test_top1)) if stop_training: break epoch += 1 if start_time >= config.budget: trainer.evaluate(epoch, test_loader, 'test') else: save_checkpoint(int(config.budget), trainer.model, trainer.optimizer, trainer.scheduler, config)
def test(): data = Loader.load_data("../NLI2FOLI/SICK/SICK_train.txt") test = Loader.load_data("../NLI2FOLI/SICK/SICK_trial.txt") encoder = FeatureExtractor.generate_postag_onehot(data["tokens"]) data["postags"] = FeatureExtractor.postag_tokenizer(data["tokens"]) test["postags"] = FeatureExtractor.postag_tokenizer(test["tokens"]) data["antons"] = FeatureExtractor.antonym_relations(data["pair_ID"]) test["antons"] = FeatureExtractor.antonym_relations(test["pair_ID"]) data["synons"] = FeatureExtractor.synonym_relations( data["tokens"], data["pair_ID"]) test["synons"] = FeatureExtractor.synonym_relations( test["tokens"], test["pair_ID"]) # Features # You don't have to do anything here # These are all the features we have # The tfidf features with both sentences seperated bag_of_words = ColumnTransformer([("A", TfidfVectorizer(), "sentence_A"), ("B", TfidfVectorizer(), "sentence_B")]) # The countvectorizer features with postagging appended to each token bag_of_words_plus_pos = ColumnTransformer([ ("POS", CountVectorizer(tokenizer=lambda x: x, preprocessor=lambda x: x), "postags") ]) # The One-hot-encoded postag features for both sentences postags_A = POSTAGTransformer(encoder, "sentence_A", maxlen=800) postags_B = POSTAGTransformer(encoder, "sentence_B", maxlen=800) # The negation features for both sentences negation_A = NEGTransformer("sentence_A") negation_B = NEGTransformer("sentence_B") # The antonyms and synonyms features antons = DumbTransfromer("antons") synons = DumbTransfromer("synons") # classifiers # Every classifiers should be put in a tuple with its name on the right hand side # You may tweak the hyperparameters nb = (MultinomialNB(alpha=0.1), "Naive Bayes") knn = (KNeighborsClassifier(), "KNN") svm = (SVC(kernel="linear", C=0.7), "SVM") forest = (RandomForestClassifier(n_estimators=1000, max_depth=128), "Random Forest") mlp = (MLPClassifier(1000), "Multi layer Perceptrons") classifiers = [nb, knn, svm, forest, mlp] # for n in range(5, 10): # hyperforest = (RandomForestClassifier( # n_estimators=n*100, max_depth=n**3), f"Random Forest with {n*100} of estimators and a max depth of {n**3}") # classifiers.append(hyperforest) # Feature_combs # Feature combinations are a list of tuples # Give the combination a name on the right hand side combs = [([bag_of_words], "TFIDF"), ([bag_of_words_plus_pos], "Combined + Postagging"), ([bag_of_words, postags_A, postags_B], "TFIDF + One hot postags"), ([postags_A, postags_B], "OneHotPosTag only"), ([bag_of_words, negation_A, negation_B], "TFIDF + NEGATION"), ([negation_A, negation_B], "NEGATION_ONLY"), ([bag_of_words, antons, synons], "TFIDF + ANTONYMS + SYNONYMS"), ([bag_of_words, antons], "TFIDF + ANTONYMS"), ([bag_of_words, synons], "TFIDF + SYNONYMS"), ([ bag_of_words, postags_B, postags_A, negation_A, negation_B, antons, synons ], "All features")] # The seach function takes the combination, classifier list and the train test data. # The result will be printed and exported to a csv file called search result # It also returns the dictionary of all the accuracys search(combs, classifiers, data, test)
# read word2vec with open('/home/student/glove/glove.6B.100d.txt', 'r') as f: lines = f.readlines() dictionary = {} weight_matrix = [] for i, line in enumerate(lines): word = line.split() dictionary[word[0]] = i weight_matrix.append([float(word[i]) for i in range(1, len(word))]) weight_matrix = np.array(weight_matrix) # ------------------------------------------------------------------------------- # # read data train_data = Loader(0, dictionary) valid_data = Loader(1, dictionary) test_data = Loader(2, dictionary) train_data = DataLoader(train_data, batch_size=128, shuffle=True) valid_data = DataLoader(valid_data, batch_size=128, shuffle=False) test_data = DataLoader(test_data, batch_size=128, shuffle=False) # ------------------------------------------------------------------------------ # model = SentenceCompression(100, 200, 50, 2, weight_matrix, len(dictionary)).cuda() lr = 0.0005 num_epoch = 20 criterion = nn.CrossEntropyLoss().cuda() optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=lr,
batch_size = 2 device = '2' os.environ['CUDA_VISIBLE_DEVICES'] = device #main if __name__ == '__main__': #create model model = Model() model.cuda() optimizer = torch.optim.Adam(model.parameters(), lr) loss = torch.nn.MSELoss() visdom_server = Visdom(port=8097) #create dataloade train_set = Loader(mode='train') val_set = Loader(mode='test') data_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=1) val_data_loader = torch.utils.data.DataLoader(val_set, batch_size=batch_size, shuffle=True, num_workers=1) #training print(f'The number of training data {len(data_loader)}') print(f'The number of testing data {len(val_data_loader)}') for epoch in range(epochs):
for result in pre_results: results['recall'] += result['recall'] results['precision'] += result['precision'] results['ndcg'] += result['ndcg'] results['recall'] /= float(len(users)) results['precision'] /= float(len(users)) results['ndcg'] /= float(len(users)) # results['auc'] = np.mean(auc_record) if multicore == 1: pool.close() print(results) return results def cprint(words : str): print(f"\033[0;30;43m{words}\033[0m") if __name__ == '__main__': train_dataset = Loader(path=world.dataset) Recmodel = NGCF(world.config, train_dataset) Neg_k = 1 bpr=BPRLoss(Recmodel, world.config) f = open (f'train_logger_{world.dataset}.txt','w') f_test = open (f'test_logger_{world.dataset}.txt','w') for epoch in range(world.TRAIN_epochs): if epoch %10 == 0: cprint("[TEST]") result = Test(train_dataset, Recmodel, epoch, world.config['multicore']) print(epoch, result, file=f_test, flush=True) output_information = BPR_train_original(train_dataset, Recmodel, bpr, epoch, neg_k=Neg_k,w=None) log_output = f'EPOCH[{epoch+1}/{world.TRAIN_epochs}] {output_information}' print(f'EPOCH[{epoch+1}/{world.TRAIN_epochs}] {output_information}') print(log_output, file=f, flush=True) f.close()
import scipy.sparse as sp import numpy as np from dataloader import Loader from scipy.sparse import csr_matrix loader = Loader() R = loader.UserItemNet.tolil() twohop = 0.004 adj_user = R.T.todok() print("generating 2") rowsum_user = np.array(adj_user.sum(axis=0)) D_user = np.power(rowsum_user, -0.5).flatten() D_user[np.isinf(D_user)] = 0 Dmat_user = sp.diags(D_user) print("generating 3") adj_item = R.todok() print("generating 4") rowsum_item = np.array(adj_item.sum(axis=0)) D_item = np.power(rowsum_item, -0.5).flatten() D_item[np.isinf(D_item)] = 0 Dmat_item = sp.diags(D_item) print("generating 5") norm_user = Dmat_item.dot(adj_user).dot(Dmat_user) norm_item = Dmat_user.dot(adj_item).dot(Dmat_item) def sparsify_propagation(adj, hop_thres): adj_valid = (adj > hop_thres)
from yamlparams.utils import Hparam import sys from metrics import mean_average_presision_k, hitrate_k, novelty, coverage from dataloader import Loader from preprocessor import Preprocessor from model import TopPopular # Read config if len(sys.argv) < 2: raise AttributeError('Use config name to define model config') cfg_path = sys.argv[1] #'books_big_setting.yml' print('Using config ' + cfg_path) config = Hparam(cfg_path) loader = Loader(config.path) preprocessor = Preprocessor() print('Reading data') df = loader.get_views() print('Preprocessing') # train_df = preprocessor.filter_zeros(train_df) df = preprocessor.filter_lazy_users(df, 0) train_df, test_df = loader.split_train_test(df, config.min_user_views, config.testing.samples) train_df, test_df = preprocessor.filter_not_in_test_items(train_df, test_df) preprocessor.build_mappers(train_df.append(test_df)) train_df.user_id = train_df.user_id.apply(preprocessor.get_user_ix)
"word2vec": "data/word2vec.bin", "is_origin_dataset": True, "train_json": "data/activity-net/train.json", "val_json": "data/activity-net/val_1.json", "test_json": "data/activity-net/val_2.json", "train_data": "data/activity-net/train_data.json", "val_data": "data/activity-net/val_data.json", "test_data": "data/activity-net/test_data.json", "feature_path": "data/activity-c3d", "feature_path_tsn": "data/tsn_score" } device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') word2vec = KeyedVectors.load_word2vec_format(config["word2vec"], binary=True) train_dataset = Loader(config, config['train_data'], word2vec, flag=True) val_dataset = Loader(config, config['val_data'], word2vec) test_dataset = Loader(config, config['test_data'], word2vec) train_loader = DataLoader(dataset=train_dataset, batch_size=64, shuffle=True) val_loader = DataLoader(dataset=val_dataset, batch_size=64, shuffle=False) test_loader = DataLoader(dataset=test_dataset, batch_size=64, shuffle=False) d_model = config['d_model'] d_ff = config['d_ff'] num_heads = config['num_heads']
class cyclegan(object): def __init__(self, sess): self.sess = sess def read_data(self): self.loader = Loader() self.loader.read_data() def load_batch(self, num, type): return self.loader.load_batch(num, type) def load_data_tr(self): return self.loader.load_dataset_tr() def load_data_tst(self): return self.loader.load_data_tst() def set_hyperparam(self, args): self.batch_size = args.batch_size self.image_size = args.fine_size self.input_c_dim = args.input_nc self.output_c_dim = args.output_nc self.L1_lambda = args.L1_lambda self.dataset_dir = args.dataset_dir if args.discrim_model == 'disc': self.discriminator = disc elif args.discrim_model == 'disc_my': self.discriminator = disc_my # if args.use_resnet: # self.generator = generator_resnet if args.gen_model == 'res': self.generator = generator_res elif args.gen_model == 'res_2': self.generator = generator_res_2 else: self.generator = generator_unet if args.use_lsgan: self.criterionGAN = mae_criterion else: self.criterionGAN = sce_criterion OPTIONS = namedtuple( 'OPTIONS', 'batch_size image_size gf_dim df_dim output_c_dim is_training') self.options = OPTIONS._make( (args.batch_size, args.fine_size, args.ngf, args.ndf, args.output_nc, args.phase == 'train')) self._build_model() self.saver = tf.train.Saver() self.pool = ImagePool(args.max_size) def _build_model(self): self.real_data = tf.placeholder(tf.float32, [ None, self.image_size, self.image_size, self.image_size, self.input_c_dim + self.input_c_dim ], name='real_A_and_B_images') print('real data shape : ', self.real_data.shape) self.real_A = self.real_data[:, :, :, :, :self.input_c_dim] self.real_B = self.real_data[:, :, :, :, self.input_c_dim:self.input_c_dim + self.output_c_dim] print('real data A and B shape : ', self.real_A.shape, self.real_B.shape) self.fake_B = self.generator(self.real_A, self.options, False, name="generatorA2B") self.fake_A_ = self.generator(self.fake_B, self.options, False, name="generatorB2A") self.fake_A = self.generator(self.real_B, self.options, True, name="generatorB2A") self.fake_B_ = self.generator(self.fake_A, self.options, True, name="generatorA2B") print('fake data A and B shape : ', self.fake_A.shape, self.fake_B.shape) print('fake data A_ and B_ shape : ', self.fake_A_.shape, self.fake_B_.shape) self.DB_fake = self.discriminator(self.fake_B, self.options, reuse=False, name="discriminatorB") self.DA_fake = self.discriminator(self.fake_A, self.options, reuse=False, name="discriminatorA") self.g_loss_a2b = self.criterionGAN(self.DB_fake, tf.ones_like(self.DB_fake)) \ + self.L1_lambda * abs_criterion(self.real_A, self.fake_A_) \ + self.L1_lambda * abs_criterion(self.real_B, self.fake_B_) self.g_loss_b2a = self.criterionGAN(self.DA_fake, tf.ones_like(self.DA_fake)) \ + self.L1_lambda * abs_criterion(self.real_A, self.fake_A_) \ + self.L1_lambda * abs_criterion(self.real_B, self.fake_B_) self.g_loss = self.criterionGAN(self.DA_fake, tf.ones_like(self.DA_fake)) \ + self.criterionGAN(self.DB_fake, tf.ones_like(self.DB_fake)) \ + self.L1_lambda * abs_criterion(self.real_A, self.fake_A_) \ + self.L1_lambda * abs_criterion(self.real_B, self.fake_B_) print('g loss is builded') # why sample is bellow ??? self.fake_A_sample = tf.placeholder(tf.float32, [ None, self.image_size, self.image_size, self.image_size, self.input_c_dim ], name='fake_A_sample') self.fake_B_sample = tf.placeholder(tf.float32, [ None, self.image_size, self.image_size, self.image_size, self.output_c_dim ], name='fake_B_sample') self.DB_real = self.discriminator(self.real_B, self.options, reuse=True, name="discriminatorB") self.DA_real = self.discriminator(self.real_A, self.options, reuse=True, name="discriminatorA") self.DB_fake_sample = self.discriminator(self.fake_B_sample, self.options, reuse=True, name="discriminatorB") self.DA_fake_sample = self.discriminator(self.fake_A_sample, self.options, reuse=True, name="discriminatorA") print('discriminating sample is builded') self.db_loss_real = self.criterionGAN(self.DB_real, tf.ones_like(self.DB_real)) self.db_loss_fake = self.criterionGAN( self.DB_fake_sample, tf.zeros_like(self.DB_fake_sample)) self.db_loss = (self.db_loss_real + self.db_loss_fake) / 2 self.da_loss_real = self.criterionGAN(self.DA_real, tf.ones_like(self.DA_real)) self.da_loss_fake = self.criterionGAN( self.DA_fake_sample, tf.zeros_like(self.DA_fake_sample)) self.da_loss = (self.da_loss_real + self.da_loss_fake) / 2 self.d_loss = self.da_loss + self.db_loss self.g_loss_a2b_sum = tf.summary.scalar("g_loss_a2b", self.g_loss_a2b) self.g_loss_b2a_sum = tf.summary.scalar("g_loss_b2a", self.g_loss_b2a) self.g_loss_sum = tf.summary.scalar("g_loss", self.g_loss) self.g_sum = tf.summary.merge( [self.g_loss_a2b_sum, self.g_loss_b2a_sum, self.g_loss_sum]) self.db_loss_sum = tf.summary.scalar("db_loss", self.db_loss) self.da_loss_sum = tf.summary.scalar("da_loss", self.da_loss) self.d_loss_sum = tf.summary.scalar("d_loss", self.d_loss) self.db_loss_real_sum = tf.summary.scalar("db_loss_real", self.db_loss_real) self.db_loss_fake_sum = tf.summary.scalar("db_loss_fake", self.db_loss_fake) self.da_loss_real_sum = tf.summary.scalar("da_loss_real", self.da_loss_real) self.da_loss_fake_sum = tf.summary.scalar("da_loss_fake", self.da_loss_fake) self.d_sum = tf.summary.merge([ self.da_loss_sum, self.da_loss_real_sum, self.da_loss_fake_sum, self.db_loss_sum, self.db_loss_real_sum, self.db_loss_fake_sum, self.d_loss_sum ]) self.test_A = tf.placeholder(tf.float32, [ None, self.image_size, self.image_size, self.image_size, self.input_c_dim ], name='test_A') self.test_B = tf.placeholder(tf.float32, [ None, self.image_size, self.image_size, self.image_size, self.output_c_dim ], name='test_B') self.testB = self.generator(self.test_A, self.options, True, name="generatorA2B") self.testA = self.generator(self.test_B, self.options, True, name="generatorB2A") t_vars = tf.trainable_variables() self.d_vars = [var for var in t_vars if 'discriminator' in var.name] self.g_vars = [var for var in t_vars if 'generator' in var.name] for var in t_vars: print(var.name) def train(self, args): """Train cyclegan""" self.lr = tf.placeholder(tf.float32, None, name='learning_rate') self.d_optim = tf.train.AdamOptimizer( self.lr, beta1=args.beta1).minimize(self.d_loss, var_list=self.d_vars) self.g_optim = tf.train.AdamOptimizer( self.lr, beta1=args.beta1).minimize(self.g_loss, var_list=self.g_vars) init_op = tf.global_variables_initializer() self.sess.run(init_op) self.writer = tf.summary.FileWriter("./logs", self.sess.graph) counter = 1 start_time = time.time() if args.continue_train: if self.load(args.checkpoint_dir): print(" [*] Load SUCCESS") else: print(" [!] Load failed...") # dataset = self.load_data_tr() for epoch in range(args.epoch): # dataA, dataB = self.load_data_tr() # dataset = [dataA, dataB] # np.random.shuffle(dataset) # batch_idxs = min(len(dataset), args.train_size) // self.batch_size lr = args.lr if epoch < args.epoch_step else args.lr * ( args.epoch - epoch) / (args.epoch - args.epoch_step) batch, meta = self.load_batch(self.batch_size, type='train') batch_idxs = len(batch) for idx in range(0, batch_idxs): # batch_images = dataset[idx*self.batch_size:(idx+1)*self.batch_size] batch_images = batch # batch_files = list(zip(dataA[idx * self.batch_size:(idx + 1) * self.batch_size], # dataB[idx * self.batch_size:(idx + 1) * self.batch_size])) # batch_images = [load_train_data(batch_file, args.load_size, args.fine_size) for batch_file in batch_files] # batch_images = np.array(batch_images).astype(np.float32) # batch_images = batch_files[batch_idxs] # batch_images = np.array(batch_images).astype(np.float32) # Update G network and record fake outputs fake_A, fake_B, _, summary_str, g_loss = self.sess.run( [ self.fake_A, self.fake_B, self.g_optim, self.g_sum, self.g_loss ], feed_dict={ self.real_data: batch_images, self.lr: lr }) self.writer.add_summary(summary_str, counter) [fake_A, fake_B] = self.pool([fake_A, fake_B]) # Update D network _, summary_str, d_loss = self.sess.run( [self.d_optim, self.d_sum, self.d_loss], feed_dict={ self.real_data: batch_images, self.fake_A_sample: fake_A, self.fake_B_sample: fake_B, self.lr: lr }) self.writer.add_summary(summary_str, counter) counter += 1 print( "Epoch: [%2d] [%4d/%4d] time: %4.4f | g loss : %8d | d loss : %4d" % (epoch, idx, batch_idxs, time.time() - start_time, g_loss, d_loss)) if np.mod(counter, args.print_freq) == args.print_freq - 1: print('save sample model at {}'.format((counter))) self.sample_model(args.sample_dir, epoch, idx) if np.mod(counter, args.save_freq) == args.save_freq - 1: print('save in checkpoint directory at {}'.format(counter)) self.save(args.checkpoint_dir, counter) def save(self, checkpoint_dir, step): model_name = "cyclegan.model" model_dir = "%s_%s" % (self.dataset_dir, self.image_size) checkpoint_dir = os.path.join(checkpoint_dir, model_dir) if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) self.saver.save(self.sess, os.path.join(checkpoint_dir, model_name), global_step=step) def load(self, checkpoint_dir): print(" [*] Reading checkpoint...") model_dir = "%s_%s" % (self.dataset_dir, self.image_size) checkpoint_dir = os.path.join(checkpoint_dir, model_dir) ckpt = tf.train.get_checkpoint_state(checkpoint_dir) if ckpt and ckpt.model_checkpoint_path: ckpt_name = os.path.basename(ckpt.model_checkpoint_path) self.saver.restore(self.sess, os.path.join(checkpoint_dir, ckpt_name)) return True else: return False def sample_model(self, sample_dir, epoch, idx): batch, meta = self.load_batch(self.batch_size, type='train') sample_image = batch[0] # batch_files = list(zip(dataA[:self.batch_size], dataB[:self.batch_size])) # sample_images = [load_train_data(batch_file, is_testing=True) for batch_file in batch_files] # sample_images = np.array(sample_images).astype(np.float32) # sample_image = np.reshape(sample_image, [1,256,256,256,2]) sample_image = np.expand_dims(sample_image, axis=0) #[1,256,256,256,2] print('sample image shape: {}'.format(sample_image.shape)) fake_A, fake_B = self.sess.run( [self.fake_A, self.fake_B], feed_dict={self.real_data: sample_image}) real_A = sample_image[:, :, :, :, :self.input_c_dim] real_B = sample_image[:, :, :, :, self.input_c_dim:self.input_c_dim + self.input_c_dim] print('real image shape: {}'.format(real_A.shape)) print('fake image shape: {}'.format(fake_A.shape)) save_images(real_A, './{}/A_real{:02d}_{:04d}'.format(sample_dir, epoch, idx)) save_images(fake_A, './{}/A_fake{:02d}_{:04d}'.format(sample_dir, epoch, idx)) save_images(fake_B, './{}/B_fake{:02d}_{:04d}'.format(sample_dir, epoch, idx)) def test(self, args): """Test cyclegan""" init_op = tf.global_variables_initializer() self.sess.run(init_op) batch, meta = self.load_batch(self.batch_size, type='test') batch_idxs = len(batch) if self.load(args.checkpoint_dir): print(" [*] Load SUCCESS") else: print(" [!] Load failed...") # write html for visual comparison index_path = os.path.join( args.test_dir, '{0}_index.html'.format(args.which_direction)) index = open(index_path, "w") index.write("<html><body><table><tr>") index.write("<th>name</th><th>input</th><th>output</th></tr>") for idx in range(batch_idxs): sample_file = meta[idx].get_name() if args.which_direction == 'MtoP': out_var, in_var = (self.testB, self.test_A) sample_image = batch[idx][:, :, :, :self.input_c_dim] else: out_var, in_var = (self.testA, self.test_B) sample_image = batch[idx][:, :, :, :self. input_c_dim:self.input_c_dim + self.input_c_dim] sample_image = np.reshape(sample_image, [1, 256, 256, 256, 1]) print('Processing image: ' + sample_file) # sample_image = [load_test_data(sample_file, args.fine_size)] sample_image = np.array(sample_image).astype(np.float32) image_path = os.path.join( args.test_dir, '{0}_{1}'.format(args.which_direction, os.path.basename(sample_file))) fake_img = self.sess.run(out_var, feed_dict={in_var: sample_image}) save_images(fake_img, image_path) index.write("<td>%s</td>" % os.path.basename(image_path)) index.write("<td><img src='%s'></td>" % (sample_file if os.path.isabs(sample_file) else ('..' + os.path.sep + sample_file))) index.write("<td><img src='%s'></td>" % (image_path if os.path.isabs(image_path) else ('..' + os.path.sep + image_path))) index.write("</tr>") index.close()
histogram_freq=1, write_graph=True, write_images=False) RP = ReduceLROnPlateau( monitor='val_loss', factor=0.1, patience=3, verbose=0, mode='auto', ) callbacks = [ES, TB, RP] #create data train_set = Loader(mode='train') data_loader = torch.utils.data.DataLoader(train_set, batch_size=300, shuffle=False, num_workers=1) for i, (x, y) in enumerate(data_loader): x = x.data.numpy() y = y.data.numpy() #main if __name__ == '__main__': model = simple_s2s() model.fit(x, y, epochs=2000, batch_size=1,
'num_workers': args.num_workers, 'pin_memory': True } if args.cuda else {} if args.cuda: torch.cuda.manual_seed(args.seed) captioning_dataset = json.load(open(args.captioning_dataset_path, "rb")) all_arts = pickle.load(open(args.articles_metadata, 'rb')) art2id = {} # ipdb.set_trace() for i, art in enumerate(all_arts): art2id[art] = i p = open(args.fake_articles, 'r') fake_articles = [json.loads(l) for i, l in enumerate(p)] train_loader = DataLoader(Loader(args, 'train', captioning_dataset, art2id, fake_articles), batch_size=args.batch_size, shuffle=True, collate_fn=collate) val_loader = DataLoader(Loader(args, 'valid', captioning_dataset, art2id, fake_articles), batch_size=args.batch_size, shuffle=False, collate_fn=collate, **kwargs) test_loader = DataLoader(Loader(args, 'test', captioning_dataset, art2id, fake_articles), batch_size=args.batch_size, shuffle=False, collate_fn=collate, **kwargs)
for line in ss_file: line = line.split("|") nr = line[0] synsets = line[1].rstrip().split(",")[:-1] wn_synsets = [] for ss in synsets: ss = wn.synset(ss) wn_synsets.append(ss) wordnet_ss[nr] = wn_synsets return wordnet_ss if __name__ == "__main__": data = Loader.load_data("../NLI2FOLI/SICK/SICK_train.txt") test = Loader.load_data("../NLI2FOLI/SICK/SICK_trial.txt") model = Model(data) encoder = FeatureExtractor.generate_postag_onehot(data["tokens"]) data["postags"] = FeatureExtractor.postag_tokenizer(data["tokens"]) test["postags"] = FeatureExtractor.postag_tokenizer(test["tokens"]) data["antons"] = FeatureExtractor.antonym_relations(data["pair_ID"]) test["antons"] = FeatureExtractor.antonym_relations(test["pair_ID"]) data["synons"] = FeatureExtractor.synonym_relations( data["tokens"], data["pair_ID"]) test["synons"] = FeatureExtractor.synonym_relations(
from train import * from eval import * from predict import predict import paddle.distributed as dist from paddle.distributed import fleet def cprint(words: str): print(f"\033[0;30;43m{words}\033[0m") if __name__ == '__main__': print(config.config) train_dataset = Loader(path=config.dataset) name_datasets = config.dataset.split('/')[-1] Recmodel = NGCF(config.config, train_dataset) if config.config['multigpu']: print('using fleet multigpu training', Recmodel) dist.init_parallel_env() Recmodel = paddle.DataParallel(Recmodel) if config.config['multicpu']: fleet.init(is_collective=True) optimizer = fleet.distributed_optimizer(optimizer) Recmodel = fleet.distributed_model(Recmodel) print('using fleet multicpu training', Recmodel) Neg_k = 1 bpr = BPRLoss(Recmodel, config.config) f = open(f'logger/train_logger_{name_datasets}.txt', 'w') f_test = open(f'logger/test_logger_{name_datasets}.txt', 'w')
default=100, metavar='N', help='how many batches to wait before logging training status') args = parser.parse_args() args.cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) if args.cuda: torch.cuda.manual_seed(args.seed) print(args) kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {} data_loader = Loader('data/train_labeled_aug.p', c.FILE_TRAIN_UNLABELED, c.FILE_VALIDATION, c.FILE_TEST, kwargs) train_loader = data_loader.getLabeledtrain() unlabeled_train_loader = data_loader.getUnlabeledtrain() valid_loader = data_loader.getValidation() model = Ladder() if args.cuda: model.cuda() l2loss = torch.nn.BCELoss( ) #torch.nn.L1Loss() # BCELoss : Pass through sigmoid #l2_2 = torch.nn.L1Loss() nllloss = nn.NLLLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)
transforms = T.Compose([M.Numpy2Tensor()]) target_transforms = T.Compose([M.Numpy2Tensor()]) # image processing RGBD2DivRGBD = IP.RGBD2DivRGBD(img_size=img_size, div_num=output_size, depth_max=depth_max, device=device) # 学習を開始した日付 dt_now = datetime.datetime.now() dt_str = dt_now.strftime('%Y-%m-%d') # Dataset object dataset_full = Loader.ExYOLOMakeDatasetObject(train_data_dir + '/color', train_data_dir + '/depth', train_data_dir + '/AABB_3D', img_size=img_size, output_size=output_size, output_channels=output_channels) def train(dataset_train): # __Initialize__ # model.train() running_loss = 0.0 iou_scores = 0.0 data_count = 0 dataloader_train = data.DataLoader(dataset=dataset_train, batch_size=batch_size, shuffle=True) for step, (rgbds,
def setUp(self): #Wipe DB apiproxy_stub_map.apiproxy.GetStub('datastore_v3').Clear() #load default data l = Loader() l.loadDatabase()
help="available datasets: [lastfm, gowalla, yelp2018, amazon-book]") parser.add_argument( '--path', type=str, default="./checkpoints", help="path to save weights") parser.add_argument( '--topks', nargs='?', default="[20]", help="@k test list") parser.add_argument('--epochs', type=int, default=1000) parser.add_argument('--seed', type=int, default=2020, help='random seed') return parser.parse_args() if __name__ == '__main__': args = parse_args() dataset = Loader(args, path=args.dataset) model = NGCF(args, dataset) neg = 1 optim = paddle.optimizer.Adam( parameters=model.parameters(), learning_rate=args.lr) for epoch in range(args.epochs): if epoch % 10 == 0: results = test(dataset, model, epoch, args) print(results) log = train(dataset, model, epoch, optim, args, neg_k=neg, w=None) print(f'EPOCH[{epoch+1}/{args.epochs}] {log}')
def read_data(self): self.loader = Loader() self.loader.read_data()
import torch import pickle import numpy as np import pandas as pd import torch.nn as nn import constants as c from dataloader import Loader from torch.autograd import Variable cuda = torch.cuda.is_available() kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {} data_loader = Loader(c.FILE_TRAIN_LABELED_AUG, c.FILE_TRAIN_UNLABELED, c.FILE_TEST, 'data/test-labeled.p', kwargs) test_loader = data_loader.getTest() test_actual = data_loader.getValidation() label_predict = np.array([]) def callval(mnist_model, test_loader, test_actual, model, file): label_predict = np.array([]) loadfile = True if loadfile: mnist = torch.load(model) mnist_model.load_state_dict(mnist) correct = 0 if torch.cuda.is_available(): mnist_model.cuda()