def __init__(self): #Build dataloaders, vocabulary, and numericalize texts self.databunch = TextClasDataBunch.from_csv(args.data, bs = 10, csv_name='data.csv', pad_first=True, pad_idx = 1) ''' Build word_to_idx and idx_to_word dictionaries for the dataset's vocabulary ''' def build_word_to_idx(idx_to_word): word_to_idx = {} for i in range(len(idx_to_word)): word_to_idx[idx_to_word[i]] = i return word_to_idx idx_to_word = self.databunch.vocab.itos word_to_idx = build_word_to_idx(idx_to_word) models = {} models['LSTM'] = LSTM(vocab_size = len(idx_to_word), embedding_dim = 300, hidden_size = 300, word_to_idx = word_to_idx, glove_path = args.embedding) models['GloVe'] = Word_Vector_Model(vocab_size = len(idx_to_word), embedding_dim = 300, word_to_idx = word_to_idx, glove_path = args.embedding) models['GRU'] = GRU(vocab_size=len(idx_to_word), embedding_dim = 300, hidden_size = 300, word_to_idx = word_to_idx, glove_path = args.embedding) self.model = models[args.model] #self.model = nn.DataParallel(self.model) self.device = torch.device("cuda:0") self.model.to(self.device) self.train_dataloader = self.databunch.train_dl self.valid_dataloader = self.databunch.valid_dl self.epochs = 20 self.learning_rate = 0.0001 self.optimizer = optim.Adam(self.model.parameters(), lr=self.learning_rate) self.loss_function = nn.CrossEntropyLoss()
def init_model(self): ''' pooling, rnn, lstm, bilstm, cnn, multi_cnn, gru :return: ''' if self.opts.model == 'pooling': self.model = Pooling(opts=self.opts, vocab=self.vocab, label_vocab=self.label_vocab) elif self.opts.model == 'cnn': self.model = CNN(opts=self.opts, vocab=self.vocab, label_vocab=self.label_vocab) elif self.opts.model == 'multi_channel_cnn': self.model = Multi_Channel_CNN(opts=self.opts, vocab=self.vocab, label_vocab=self.label_vocab) elif self.opts.model == 'multi_layer_cnn': self.model = Multi_Layer_CNN(opts=self.opts, vocab=self.vocab, label_vocab=self.label_vocab) elif self.opts.model == 'char_cnn': self.char = True self.model = Char_CNN(opts=self.opts, vocab=self.vocab, char_vocab=self.char_vocab, label_vocab=self.label_vocab) elif self.opts.model == 'lstm': self.model = LSTM(opts=self.opts, vocab=self.vocab, label_vocab=self.label_vocab) elif self.opts.model == 'gru': self.model = GRU(opts=self.opts, vocab=self.vocab, label_vocab=self.label_vocab) elif self.opts.model == 'lstm_cnn': self.model = LSTM_CNN(opts=self.opts, vocab=self.vocab, label_vocab=self.label_vocab) elif self.opts.model == 'treelstm': self.tree = True self.model = BatchChildSumTreeLSTM(opts=self.opts, vocab=self.vocab, label_vocab=self.label_vocab) elif self.opts.model == 'cnn_treelstm': self.tree = True self.model = CNN_TreeLSTM(opts=self.opts, vocab=self.vocab, label_vocab=self.label_vocab) elif self.opts.model == 'lstm_treelstm': self.tree = True self.model = LSTM_TreeLSTM(opts=self.opts, vocab=self.vocab, label_vocab=self.label_vocab) else: raise RuntimeError('please choose your model first!') if self.opts.use_cuda: self.model = self.model.cuda()
def main(): config = set_up_experience('configs/config.json') with tf.Session() as sess: # create an instance of the model you want model = Model(config) #load model if exists model.load(sess) # create your data generator # set generate_tfrecords to true in config file to write new tfrecords data = DataLoader(config) #data.write_train_and_test_sets(train_ratio=0.8) # create trainer and pass all the previous components to it trainer = Trainer(sess, model, data, config) # here you train your model trainer.train()
def __init__(self, embedding_dim, output_dim, hidden_size, num_layers, bidirectional, dropout, pretrained_embeddings): super(TextRNN, self).__init__() self.embedding = nn.Embedding.from_pretrained(pretrained_embeddings, freeze=False) self.rnn = LSTM(embedding_dim, hidden_size, num_layers, bidirectional, dropout) self.fc = Linear(hidden_size * 2, output_dim) self.dropout = nn.Dropout(dropout)
def __init__(self, vocab_size, embedding_dim, output_dim, hidden_size, num_layers, bidirectional, dropout, pad_idx): super(TextRNN, self).__init__() self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx=pad_idx) self.rnn = LSTM(embedding_dim, hidden_size, num_layers, bidirectional, dropout) self.fc = nn.Linear(hidden_size * 2, output_dim) self.dropout = nn.Dropout(dropout)
def get_model(model): """ Get Model instance """ assert model in ['CHAR', 'WIDE', 'VDCNN', 'LSTM', 'MULTI_LSTM'] if model == 'CHAR': return Char_CNN(config, conv_layers, fc_layers) elif model == 'WIDE': return Wide_CNN(config, wconv_layers) elif model == 'VDCNN': return VDCNN2(config) elif model == 'LSTM': return LSTM(config, fc_layers) elif model == 'MULTI_LSTM': return MULTI_LSTM(config, fc_layers, rnn_layers)
def initialize_model_and_trainer(model_properties, training_properties, datasetloader, device): logger.info("Model type is %s", training_properties["learner"]) if training_properties["learner"] == "text_cnn": model = TextCnn(model_properties).to(device) trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader, device) elif training_properties["learner"] == "gru": model = GRU(model_properties).to(device) trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader, device) elif training_properties["learner"] == "lstm": model = LSTM(model_properties).to(device) trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader, device) elif training_properties["learner"] == "char_cnn": model = CharCNN(model_properties).to(device) trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader, device) elif training_properties["learner"] == "vdcnn": model = VDCNN(model_properties).to(device) trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader, device) elif training_properties["learner"] == "conv_deconv_cnn": model = ConvDeconvCNN(model_properties) trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader, device) elif training_properties["learner"] == "transformer_google": model = TransformerGoogle(model_properties).model.to(device) trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader, device) elif training_properties["learner"] == "lstmcrf": assert training_properties["task"] == "ner" model = LSTMCRF(model_properties).to(device) trainer = Trainer.trainer_factory("single_model_ner_trainer", training_properties, datasetloader, device) else: raise ValueError( "Model is not defined! Available learner values are : 'text_cnn', 'char_cnn', 'vdcnn', 'gru', " "'lstm', 'conv_deconv_cnn' and 'transformer_google'") return model, trainer
def __init__(self, embedding_dim, output_dim, hidden_size, num_layers, bidirectional, dropout, pretrained_embeddings): super(LSTMATT, self).__init__() self.embedding = nn.Embedding.from_pretrained( pretrained_embeddings, freeze=False) self.rnn = LSTM(embedding_dim, hidden_size, num_layers, bidirectional, dropout) self.fc = nn.Linear(hidden_size * 2, output_dim) self.dropout = nn.Dropout(dropout) self.W_w = nn.Parameter(torch.Tensor(hidden_size * 2, hidden_size * 2)) self.u_w = nn.Parameter(torch.Tensor(hidden_size * 2, 1)) nn.init.uniform_(self.W_w, -0.1, 0.1) nn.init.uniform_(self.u_w, -0.1, 0.1)
def __init__(self, word_dim, char_dim, output_dim, hidden_size, num_layers, bidirectional, dropout, word_emb, char_emb, highway_layers): super(TextRNNHighway, self).__init__() self.char_embedding = nn.Embedding.from_pretrained(char_emb, freeze=False) self.word_embedding = nn.Embedding.from_pretrained(word_emb, freeze=False) self.text_embedding = Embedding(highway_layers, word_dim, char_dim) self.rnn = LSTM(word_dim + char_dim, hidden_size, num_layers, bidirectional, dropout) self.fc = Linear(hidden_size * 2, output_dim) self.dropout = nn.Dropout(dropout)
def initialize_model_and_trainer(model_properties, training_properties, datasetloader, device): logger.info("Model type is %s", training_properties["learner"]) if training_properties["learner"] == "text_cnn": model = TextCnn(model_properties).to(device) trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter, datasetloader.val_iter, datasetloader.test_iter, device) elif training_properties["learner"] == "gru": model = GRU(model_properties).to(device) trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter, datasetloader.val_iter, datasetloader.test_iter, device) elif training_properties["learner"] == "lstm": model = LSTM(model_properties).to(device) trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter, datasetloader.val_iter, datasetloader.test_iter, device) elif training_properties["learner"] == "char_cnn": model = CharCNN(model_properties).to(device) trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter, datasetloader.val_iter, datasetloader.test_iter, device) elif training_properties["learner"] == "vdcnn": model = VDCNN(model_properties).to(device) trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter, datasetloader.val_iter, datasetloader.test_iter, device) elif training_properties["learner"] == "conv_deconv_cnn": convDeconveCNN = ConvDeconvCNN(model_properties) encoderCNN = convDeconveCNN.encoder.to(device) decoderCNN = convDeconveCNN.decoder.to(device) classifier = convDeconveCNN.classifier.to(device) trainer = Trainer.trainer_factory("multiple_model_trainer", training_properties, datasetloader.train_iter, datasetloader.val_iter, datasetloader.test_iter, device) model = [encoderCNN, decoderCNN, classifier] elif training_properties["learner"] == "transformer_google": model = TransformerGoogle(model_properties).model.to(device) trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter, datasetloader.val_iter, datasetloader.test_iter, device) elif training_properties["learner"] == "crf": model = ConditionalRandomField().to(device) else: raise ValueError("Model is not defined! Available learner values are : 'text_cnn', 'char_cnn', 'vdcnn', 'gru', " "'lstm', 'conv_deconv_cnn' and 'transformer_google'") return model, trainer
def __init__(self, word_dim, char_dim, output_dim, hidden_size, num_layers, bidirectional, dropout, word_emb, char_emb, highway_layers): super(LSTMATTHighway, self).__init__() self.char_embedding = nn.Embedding.from_pretrained(char_emb, freeze=False) self.word_embedding = nn.Embedding.from_pretrained(word_emb, freeze=False) self.text_embedding = Embedding(highway_layers, word_dim, char_dim) self.rnn = LSTM(word_dim + char_dim, hidden_size, num_layers, bidirectional, dropout) self.fc = nn.Linear(hidden_size * 2, output_dim) self.dropout = nn.Dropout(dropout) self.W_w = nn.Parameter(torch.Tensor(hidden_size * 2, hidden_size * 2)) self.u_w = nn.Parameter(torch.Tensor(hidden_size * 2, 1)) nn.init.uniform_(self.W_w, -0.1, 0.1) nn.init.uniform_(self.u_w, -0.1, 0.1)
sess = tf.Session(config=config) tf.keras.backend.set_session(sess) # # save np.load # np_load_old = np.load # # modify the default parameters of np.load # np.load = lambda *a,**k: np_load_old(*a, allow_pickle=True, **k) num_words = 20000 (x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=num_words) epoch = 5 batch_size = 64 word_to_index = imdb.get_word_index() word_to_index = {key:(value+3) for key,value in word_to_index.items()} word_to_index["<PAD>"] = 0 word_to_index["<START>"] = 1 word_to_index["<UNK>"] = 2 index_to_word = {value:key for key,value in word_to_index.items()} def print_sentence(id_list): print(' '.join([index_to_word[id] for id in id_list if id != 0])) print("Train-set size: ", len(x_train)) print("Test-set size: ", len(x_test)) x_train, x_test, max_tokens = utils.pad_sequences(train_sequences=x_train, test_sequences=x_test) lstm = LSTM(num_words, max_tokens) lstm.build() lstm.train(epochs=epoch, x_train=x_train, y_train=y_train, batch_size=batch_size)
def main(): parser = argparse.ArgumentParser() # parameters parser.add_argument("--epoch", default=100, type=int, help="the number of epoches needed to train") parser.add_argument("--lr", default=2e-5, type=float, help="the learning rate") parser.add_argument("--classifier", default='lstm', type=str, help="the classifier, such as LSTM, CNN ...") parser.add_argument("--hidden_size", default=64, type=int, help="the hidden size") parser.add_argument("--output_size", default=1, type=int, help="the output label size") parser.add_argument("--early_stopping", default=15, type=int, help="Tolerance for early stopping (# of epochs).") parser.add_argument("--load_model", default=None, type=str, help="load pretrained model for testing") parser.add_argument('--n_models', nargs='+') args = parser.parse_args() path = 'D:/MyDocument/Project/CarSalesPrediction/' T = 12 data = dataset.read_data(path) df, feature_list, y_list = dataset.get_feature(data, T) train_x, train_y, valid_x, valid_y, test_x, model2index = dataset.get_Xy( df, T, feature_list, y_list) X_train, X_val, X_test = dataset.normalization(train_x, valid_x, test_x) input_size = X_train.shape[2] print("feature_dim: ", input_size) model = LSTM(args.output_size, [args.hidden_size], input_size) X_train = torch.from_numpy(X_train.astype(np.float32)) y_train = torch.from_numpy(train_y.astype(np.float32)) X_val = torch.from_numpy(X_val.astype(np.float32)) y_val = torch.from_numpy(valid_y.astype(np.float32)) X_test = torch.from_numpy(X_test.astype(np.float32)) y_mean = [] train_y = [] valid_y = [] for step in range(4): yy = y_train[:, step].view(-1, 1) train_y.append(yy) mean_y = yy.mean() y_mean.append(mean_y) valid_y.append(y_val[:, step]) # print (X_val) # print(X_val) if args.load_model: evaluation_public = pd.read_csv( path + 'test2_dataset/evaluation_public.csv')[[ 'id', 'regMonth', 'forecastVolum' ]] id_list = evaluation_public['id'].values evaluation_result = DataFrame({'id': id_list}) # forecastVolum = [] for ii, model_name in enumerate(args.n_models): print(model_name) model.load_state_dict( torch.load(args.load_model + '/' + model_name)) model.eval() val_prediction = model(X_val) score = get_score( np.expm1(val_prediction.detach().numpy() + y_mean[ii].numpy()), np.expm1(valid_y[ii].numpy()), model2index) logger.info('Valid Score:%.4f', score) test_prediction = model(X_test) test_prediction = np.expm1(test_prediction.detach().numpy() + y_mean[ii].numpy()).reshape( -1).tolist() evaluation_public.loc[(evaluation_public.regMonth == ii + 1), 'forecastVolum'] = test_prediction # forecastVolum.extend(test_prediction) del model model = LSTM(args.output_size, [args.hidden_size], input_size) # evaluation_result['forecastVolum'] = forecastVolum evaluation_public[['id', 'forecastVolum']].round().astype(int).to_csv( 'evaluation_public.csv', index=False) exit() for i in range(4): train_model(model, X_train, train_y[i], y_mean[i], X_val, valid_y[i], args.epoch, model2index, args.lr, i)
rnn_name = 'lstm' if args.use_gru == 1: rnn_name = 'gru' emb_name = args.pretrained_emb.split("/")[-1][:-4] model_name = rnn_name + "_" + str(args.emb_dim) + "embDim_" + \ str(args.dropout) + "dropout_" + \ str(args.lstm_hidden_size) + "LSTMhidden_" + \ str(args.mlp_hidden_size) + "MLPhidden_" + \ str(args.num_epochs) + "epochs" + "_" + emb_name model_name += "_run" + str(run_num + 1) print("Model name:", model_name) log_file = open(save_path + model_name + "_logs.txt", 'w') model = LSTM(lstm_args=lstm_args, mlp_args=mlp_args).to(device) print(model) optimizer = optim.Adam(model.parameters(), lr=0.001) loss_fn = nn.BCELoss() train(model=model, optimizer=optimizer, num_epochs=num_epochs, criterion=loss_fn, eval_every=args.step_size, train_loader=train_loader, valid_loader=valid_loader, save_path=save_path, model_name=model_name)
args.embed_num = len(text_field.vocab) args.class_num = len( label_field.vocab) - 1 # 4, positive, negative, neutral, conflict args.text_field = text_field args.cuda = (not args.no_cuda) and torch.cuda.is_available() del args.no_cuda print("\nParameters:") for attr, value in sorted(args.__dict__.items()): print("\t{}={}".format(attr.upper(), value)) models = [ TD_LSTM(args), TC_LSTM(args), LSTM(args), AE_LSTM(args), ATAE_LSTM(args), IAN(args), RAM(args) ] models_name = [ "TD_LSTM", "TC_LSTM", "LSTM", "AE_LSTM", "ATAE_LSTM", "IAN_LSTM", "RAM" ] if args.cuda: torch.cuda.set_device(args.device) for i in range(len(models)): models[i] = models[i].cuda() # train and test for all models
log_file = open(save_path + model_name + "_test_logs.txt", 'a+') # prepare test loader for the test set test_file = args.data_path + args.test_file test_data = ArticlesDataset(csv_file=test_file, vocab=vocab, label2id=label2id) test_loader = DataLoader(test_data, batch_size=args.batch_size, shuffle=False) scores_dict = {'f1': [], 'recall': [], 'precision': [], 'confidence': []} for run_num in range(args.num_runs): model_run_name = model_name + "_run" + str(run_num + 1) print("-" * 10, "Run", run_num + 1, "-" * 10) print("Model name:", model_run_name) print("Loading model from", save_path + model_run_name + ".pt") best_model = LSTM(lstm_args=lstm_args, mlp_args=mlp_args).to(device) optimizer = torch.optim.Adam(best_model.parameters(), lr=0.005) load_checkpoint(save_path + model_run_name + ".pt", best_model, optimizer, device, log_file) results = evaluate(best_model, test_loader) scores_dict['f1'].append(results['f1']) scores_dict['recall'].append(results['recall']) scores_dict['precision'].append(results['precision']) # if args.save_confidence is True: # scores_dict['confidence'].append(results['confidence']) # scores_dict['labels'].append(results['labels']) # scores_dict['content'].append(results['content']) # sentence_encodings = results['sentence_encodings']
def train(self, trainFile, devFile, testFile): readerTrain = Reader.reader(trainFile) readerDev = Reader.reader(devFile) readerTest = Reader.reader(testFile) sentsTrain = readerTrain.getWholeText() sentsDev = readerDev.getWholeText() sentsTest = readerTest.getWholeText() sentsTrain = self.cutSentFromText(sentsTrain) sentsDev = self.cutSentFromText(sentsDev) sentsTest = self.cutSentFromText(sentsTest) self.HyperParams.trainLen = len(sentsTrain) self.HyperParams.devLen = len(sentsDev) self.HyperParams.testLen = len(sentsTest) self.createAlphabet(sentsTrain + sentsDev) self.HyperParams.topicSize = len(self.topics) args = self.HyperParams.args() LearningRate = self.HyperParams.learningRate Steps = self.HyperParams.Steps model = biLSTM.Model(self.HyperParams) Optimizer = oprim.Adam(model.parameters(), lr=LearningRate) def accuracy(model, sents): pred_right_num_idx = 0 pred_num_idx = 1 gold_num_idx = 2 evalList = [[0, 0, 0] for _ in range(self.HyperParams.labelSize)] # for sent in sents: topic, text, label = self.processingRawStanceData(sents) text = self.seq2id(text) label = self.label2id(label) topic = Variable(torch.LongTensor(topic)) text = Variable(torch.LongTensor(text)) label = Variable(torch.LongTensor(label)) Y = model(topic, text) C = (torch.max(Y, 1)[1].view(label.size()).data == label.data).sum() pred_list = torch.max(Y, 1)[1].view(label.size()).data.tolist() label_list = label.data.tolist() for i in range(len(evalList)): for j in range(len(label_list)): if label_list[j] == i: evalList[i][gold_num_idx] += 1 if label_list[j] == pred_list[j]: evalList[i][pred_right_num_idx] += 1 if pred_list[j] == i: evalList[i][pred_num_idx] += 1 P_R_F1_list = [ Eval(pred_right_num=evalList[i][pred_right_num_idx], pred_num=evalList[i][pred_num_idx], gold_num=evalList[i][gold_num_idx]).P_R_F1 for i in range(len(evalList)) ] return float(C) / len(sents) * 100, C, len(sents), P_R_F1_list def getTextBatchList(text, batch): textBatchlist = [] textBatchNum = len(text) // batch if len(text) % batch != 0: textBatchNum += 1 if textBatchNum - 1 < 0: print("wrong: func getTextBatchList's text's length is 0!!!") return [] end = 0 for i in range(textBatchNum - 1): begin = end end += batch textBatchlist.append(text[begin:end]) textBatchlist.append(text[end:len(text)]) return textBatchlist file = open(self.HyperParams.writeFileName, 'a+') file.write(args) file.close() sentsTrain = sentsTrain sentsDev = sentsDev sentsTest = sentsTest batchSize = self.HyperParams.batchSize for step in range(Steps): file = open(self.HyperParams.writeFileName, 'a+') totalLoss = torch.Tensor([0]) cnt = 0 trainCorrect = 0 random.shuffle(sentsTrain) textBatchList = getTextBatchList(sentsTrain, batchSize) for batch in textBatchList: # print(batch.size()) model.train() Optimizer.zero_grad() topic, text, label = self.processingRawStanceData(batch) text = self.seq2id(text) label = self.label2id(label) topic = Variable(torch.LongTensor(topic)) text = Variable(torch.LongTensor(text)) label = Variable(torch.LongTensor(label)) Y = model(topic, text) Loss = F.cross_entropy(Y, label) Loss.backward() #torch.nn.utils.clip_grad_norm(model.parameters(), 10) Optimizer.step() cnt += 1 if cnt % 500 == 0: print(cnt) totalLoss += Loss.data trainCorrect += (torch.max(Y, 1)[1].view( label.size()).data == label.data).sum() totalLoss /= len(sentsTrain) TrainAcc = float(trainCorrect) / len(sentsTrain) * 100 FAVOR_index = self.HyperParams.labelAlpha.string2id["favor"] AGAINST_index = self.HyperParams.labelAlpha.string2id["against"] DevAcc, DevCorrect, DevNum, P_R_F1_dev_list = accuracy( model, sentsDev) TestAcc, TestCorrect, TestNum, P_R_F1_test_list = accuracy( model, sentsTest) dev_mean_F1 = (P_R_F1_dev_list[FAVOR_index][2] + P_R_F1_dev_list[AGAINST_index][2]) / 2 test_mean_F1 = (P_R_F1_test_list[FAVOR_index][2] + P_R_F1_test_list[AGAINST_index][2]) / 2 output = "Step: {} - loss: {:.6f} Train acc: {:.4f}%{}/{} Dev acc: {:.4f}%{}/{} Test acc: {:.4f}%{}/{} F1={:.4f}".format( step, totalLoss.numpy()[0], TrainAcc, trainCorrect, len(sentsTrain), DevAcc, DevCorrect, int(DevNum), TestAcc, TestCorrect, int(TestNum), test_mean_F1) print(output) file.write(output + "\n") file.close()
hidden1 = 512 drop_p = 0.0 # Train with Skeleton+LSTM if __name__ == '__main__': # Load data transform = None # TODO train_set = CSL_Skeleton(data_path=data_path, label_path=label_path, frames=sample_duration, num_classes=num_classes, selected_joints=selected_joints, train=True, transform=transform) val_set = CSL_Skeleton(data_path=data_path, label_path=label_path, frames=sample_duration, num_classes=num_classes, selected_joints=selected_joints, train=False, transform=transform) logger.info("Dataset samples: {}".format(len(train_set)+len(val_set))) train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=4, pin_memory=True) val_loader = DataLoader(val_set, batch_size=batch_size, shuffle=True, num_workers=4, pin_memory=True) # Create model model = LSTM(lstm_input_size=lstm_input_size, lstm_hidden_size=lstm_hidden_size, lstm_num_layers=lstm_num_layers, num_classes=num_classes, hidden1=hidden1, drop_p=drop_p).to(device) # Run the model parallelly if torch.cuda.device_count() > 1: logger.info("Using {} GPUs".format(torch.cuda.device_count())) model = nn.DataParallel(model) # Create loss criterion & optimizer criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=learning_rate) # Start training logger.info("Training Started".center(60, '#')) for epoch in range(epochs): # Train the model train_epoch(model, criterion, optimizer, train_loader, device, epoch, logger, log_interval, writer) # Validate the model
#method = "train" method = "read" cnn = CNN() cnn.data(train_data, train_labels, test_data, test_labels) cnn.createModel() cnn.findBestWeights(method) cnn.testModel() print() #method = "train" method = "read" lstm = LSTM() lstm.data(train_data, train_labels, test_data, test_labels) lstm.createModel() lstm.findBestWeights(method) lstm.testModel()
def main(args): best_er1 = 0 # Check if CUDA is enabled args.cuda = not args.no_cuda and torch.cuda.is_available() # Load data root = args.datasetPath files = [] train_ids = [] test_ids = [] if os.path.isfile('file_list.txt'): print("File exists") else: print("File does not exist") print("Prepare files") files = [ f for f in os.listdir(root) if os.path.isfile(os.path.join(root, f)) ] idx = np.random.permutation(len(files)) idx = idx.tolist() files = [files[i] for i in idx[:]] with open('file_list.txt', "w") as myfile: for f in files: myfile.write("%s\n" % f) file2 = open("file_list.txt") files = [line[:-1] for line in file2] file2.close() chunk = int(len(files) / 10) train_ids = [] test_ids = [] print(len(files)) for i in range(10): if i == int(args.fold): test_ids = files[i * chunk:i * chunk + chunk] print("test: " + str(i * chunk) + ":" + str(i * chunk + chunk)) continue train_ids += files[i * chunk:i * chunk + chunk] print("train: " + str(i * chunk) + ":" + str(i * chunk + chunk)) all_data = utils.Qm9( root, files, edge_transform=datasets.qm9_edges, e_representation="raw_distance", ) data_train = utils.Qm9( root, train_ids, edge_transform=datasets.qm9_edges, e_representation="raw_distance", ) data_test = utils.Qm9( root, test_ids, edge_transform=datasets.qm9_edges, e_representation="raw_distance", ) # Define model and optimizer print("Define model") # Select one graph g_tuple, l = data_train[1] g, h_t, e = g_tuple print("\tStatistics") stat_dict = datasets.get_graph_stats(all_data, ["target_mean", "target_std"]) # Identify atoms in all files data_train.set_target_transform(lambda x: datasets.normalize_data( x, stat_dict["target_mean"], stat_dict["target_std"])) data_test.set_target_transform(lambda x: datasets.normalize_data( x, stat_dict["target_mean"], stat_dict["target_std"])) # Data Loader train_loader = torch.utils.data.DataLoader( data_train, batch_size=args.batch_size, shuffle=True, collate_fn=datasets.collate_g, num_workers=args.prefetch, pin_memory=True, ) test_loader = torch.utils.data.DataLoader( data_test, batch_size=args.batch_size, collate_fn=datasets.collate_g, num_workers=args.prefetch, pin_memory=True, ) print("\tCreate model") in_n = [len(h_t[0]), len(list(e.values())[0])] hidden_state_size = 73 message_size = 73 n_layers = 3 l_target = len(l) type = "regression" if args.model == "MPNNv2": model = MPNNv2(in_n, [5, 15, 15], [10, 20, 20], l_target, type=type) elif args.model == "MPNNv3": model = MPNNv3([1, 2, 3, 4], in_n, [5, 15, 15], 30, l_target, type=type) elif args.model == "LSTM": model = LSTM(in_n, hidden_state_size, message_size, n_layers, l_target, type=type) else: model = MPNN(in_n, hidden_state_size, message_size, n_layers, l_target, type=type) del in_n, hidden_state_size, message_size, n_layers, l_target, type print("Optimizer") optimizer = optim.Adam(model.parameters(), lr=args.lr) criterion = nn.MSELoss() evaluation = lambda output, target: torch.mean( torch.abs(output - target) / torch.abs(target)) print("Logger") logger = Logger(args.logPath) lr_step = (args.lr - args.lr * args.lr_decay) / ( args.epochs * args.schedule[1] - args.epochs * args.schedule[0]) # get the best checkpoint if available without training if args.resume: checkpoint_dir = args.resume best_model_file = os.path.join(checkpoint_dir, "model_best.pth") if not os.path.isdir(checkpoint_dir): os.makedirs(checkpoint_dir) if os.path.isfile(best_model_file): print("=> loading best model '{}'".format(best_model_file)) checkpoint = torch.load(best_model_file) args.start_epoch = checkpoint["epoch"] best_acc1 = checkpoint["best_er1"] model.load_state_dict(checkpoint["state_dict"]) optimizer.load_state_dict(checkpoint["optimizer"]) print("=> loaded best model '{}' (epoch {})".format( best_model_file, checkpoint["epoch"])) else: print("=> no best model found at '{}'".format(best_model_file)) print("Check cuda") if args.cuda: print("\t* Cuda") model = model.cuda() criterion = criterion.cuda() # Epoch for loop for epoch in range(0, args.epochs): if (epoch > args.epochs * args.schedule[0] and epoch < args.epochs * args.schedule[1]): args.lr -= lr_step for param_group in optimizer.param_groups: param_group["lr"] = args.lr # train for one epoch train(train_loader, model, criterion, optimizer, epoch, evaluation, logger) # evaluate on test set #er1 = validate(valid_loader, model, criterion, evaluation, logger) #is_best = er1 > best_er1 #best_er1 = min(er1, best_er1) is_best = True best_er1 = 1 datasets.save_checkpoint( { "epoch": epoch + 1, "state_dict": model.state_dict(), "best_er1": best_er1, "optimizer": optimizer.state_dict(), }, is_best=is_best, directory=args.resume, ) # Logger step logger.log_value("learning_rate", args.lr).step() # get the best checkpoint and test it with test set if args.resume: checkpoint_dir = args.resume best_model_file = os.path.join(checkpoint_dir, "model_best.pth") if not os.path.isdir(checkpoint_dir): os.makedirs(checkpoint_dir) if os.path.isfile(best_model_file): print("=> loading best model '{}'".format(best_model_file)) checkpoint = torch.load(best_model_file) args.start_epoch = checkpoint["epoch"] best_acc1 = checkpoint["best_er1"] model.load_state_dict(checkpoint["state_dict"]) if args.cuda: model.cuda() optimizer.load_state_dict(checkpoint["optimizer"]) print("=> loaded best model '{}' (epoch {})".format( best_model_file, checkpoint["epoch"])) else: print("=> no best model found at '{}'".format(best_model_file)) # For testing validate(test_loader, model, criterion, evaluation)