def model_fn(model_dir): """ Loads the PyTorch model from the `model_dir` directory. :param model_dir: model directory :return: model created """ print("Loading model.") model_info = {} model_info_path = os.path.join(model_dir, 'model_info.pth') with open(model_info_path, 'rb') as f: model_info = torch.load(f) print("model_info: {}".format(model_info)) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = Regression(model_info['input_features'], model_info['hidden_dim1'], model_info['hidden_dim2'], model_info['output_dim']) model_path = os.path.join(model_dir, 'model.pth') with open(model_path, 'rb') as f: model.load_state_dict(torch.load(f)) model.to(device).eval() print("Done loading model.") return model
def __call__(self, number_of_iterations=2, learning_rate=0.005, embedding_size=300): print("Starting 'Image Retrieval' in 'Regression' mode with '" + self.difficulty + "' data") self.model_full_path = self.model_path + "/" + self.model_name + "_" + self.timestamp + "_" + str( learning_rate) + "_" + str(embedding_size) + ".pty" self.output_file_name = self.output_path + "/" + self.model_name + "_" + self.timestamp + "_" + str( learning_rate) + "_" + str(embedding_size) + ".csv" self.number_of_iterations = number_of_iterations self.learning_rate = learning_rate self.embedding_size = embedding_size self.model = Regression(self.nwords, self.embedding_size, self.image_feature_size, self.output_vector_size) self.criterion = nn.MSELoss() self.evaluate = Evaluate(self.model, self.img_features, self.minibatch, self.preprocess, self.image_feature_size) print(self.model) self.optimizer = optim.Adam(self.model.parameters(), lr=self.learning_rate) self.train_loss_values = [] self.dev_loss_values = [] self.test_loss_values = [] self.magic() self.save_model() self.save_data()
def __init__(self, args, trainRegressionDataLoader, trainRegressionClassificationLoader, testDataLoader, trainRainFallLoader, means, std): self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') self.trainRegressionDataLoader = trainRegressionDataLoader self.trainRegressionClassificationLoader = trainRegressionClassificationLoader self.testDataLoader = testDataLoader self.classificationLoader = trainRainFallLoader self.run_datetime = datetime.datetime.now() self.out_path = args.out self.sigma = args.sigma self.beta = args.beta self.earlyStop = args.earlyStop self.nClass = args.nClass self.noiseMean = torch.zeros(args.batch_size, args.featureNums, 17, 17) self.noiseStd = 1e-3 self.model = AutoencoderBN(self.noiseMean, self.noiseStd).to(self.device) self.regressionModel = Regression(self.nClass).to(self.device) self.classificationModel = regressionClassification(self.nClass).to(self.device) self.rainFallClassifierModel = rainFallClassification().to(self.device) self.meanStdNormalizer = MeanVarianceNormalizer(means, std).to(self.device) self.meanvarLoss = MeanVarLoss(self.nClass).to(self.device) self.normaliedLoss = NormalizerLoss(std).to(self.device) self.focalLoss = FocalLoss(self.nClass, alpha=0.25, gamma=2).to(self.device) self.rainFocalLoss = FocalLoss(2, alpha=0.25, gamma=2).to(self.device) self.regressionOptim = torch.optim.Adam([ {'params': self.regressionModel.parameters(), 'lr': args.lr, 'weight_decay': args.weight_decay}, {'params': self.model.parameters(), 'lr': args.lr, 'weight_decay': args.weight_decay}, ], lr=args.lr * 10, weight_decay=args.weight_decay * 10) self.classificationOptim = torch.optim.Adam(self.classificationModel.parameters(), lr=args.lr * 100) self.rainFallOptim = torch.optim.Adam(self.rainFallClassifierModel.parameters(), lr=args.lr * 10) # self.reconstructOptim = torch.optim.Adam(self.model.parameters(), lr=args.lr, weight_decay=args.weight_decay) self.scheduler = torch.optim.lr_scheduler.StepLR(self.regressionOptim, step_size=750 * 2) self.criterion = nn.MSELoss() self.classificationCriterion = nn.CrossEntropyLoss() if not os.path.exists(self.out_path): os.makedirs(self.out_path) self.logger = Logger(self.out_path) with open(os.path.join(self.out_path, "para.json"), "w") as f: json.dump(args.__dict__, f) self.epoch = 0 self.iteration = 0 self.classificationIteration = 0 self.rainfallclassificationIteration = 0 self.test_step = 0 self.max_epoch = args.epochs self.val_interval = args.interval self.res = 0 self.bestConstructLoss = 1e7 self.bestConstructEpoch = 0 self.best_error = 1e7; self.best_res_epoch = 0
def regression(): # ********************* load the dataset and divide to X&y *********************** from sklearn.datasets import make_blobs X, Y = make_blobs(cluster_std=0.9, random_state=20, n_samples=1000, centers=10, n_features=10) from Algorithms.ML_.helper.data_helper import split_train_val_test X, Xv, y, Yv, Xt, Yt = split_train_val_test(X, Y) print(X.shape, y.shape, Xv.shape, Yv.shape, Xt.shape, Yt.shape) # ********************* build model *********************** from model import Regression from layer import Layer, Dense from activation import Activation, Softmax, Sigmoid, ReLU from regularization import Regularization, L1, L2, L12 from optimizer import Vanilla model = Regression() input_size = X.shape[1] hidden_size = 50 num_classes = 10 learning_rate, reg_rate = 1e-3, 0.5 model = Regression([ Dense(hidden_size, input_shape=(input_size, ), activation=ReLU(), alpha=learning_rate, lambda_=reg_rate), ]) model += Dense(num_classes, activation=Softmax(), alpha=learning_rate, lambda_=reg_rate) # add layer with += model.compile() model.describe() # ********************* train *********************** loss_train, loss_val = model.train(X, y, val=(Xv, Yv), iter_=5000, batch=32, return_loss=True, verbose=True) import matplotlib.pyplot as plt plt.plot(range(len(loss_train)), loss_train) plt.plot(range(len(loss_val)), loss_val) plt.legend(['train', 'val']) plt.xlabel('Iteration') plt.ylabel('Training loss') plt.title('Training Loss history') plt.show() # ********************* predict *********************** pred_train = model.predict(X) pred_val = model.predict(Xv) pred_test = model.predict(Xt) import metrics print('train accuracy=', metrics.accuracy(y, pred_train)) print('val accuracy=', metrics.accuracy(Yv, pred_val)) print('test accuracy=', metrics.accuracy(Yt, pred_test)) print('null accuracy=', metrics.null_accuracy(y)) import metrics metrics.print_metrics(Yt, pred_test)
def training(): # Load data. print('Loading data...') try: with gfile.Open(MODEL_DIR + '/data', 'rb') as f: x_data, y_data = pickle.loads(f.read()) print(' Old data found in {}.'.format(MODEL_DIR + '/data')) except: print(' Creation of a new set of data.') x_data, y_data = zip(*du.load_labels_data(DATA_DIRECTORY)) with gfile.Open(MODEL_DIR + '/data', 'wb') as f: f.write(pickle.dumps((x_data, y_data))) # Load and save vocabulary. print('Loading vocabulary...') try: vocab_processor = learn.preprocessing.VocabularyProcessor.restore( MODEL_DIR + '/vocab') print(" Old vocabulary found in {}.".format(MODEL_DIR + '/vocab')) except: print(" Creation of a new vocabulary.") max_document_length = max([len(x.split(" ")) for x in y_data]) vocab_processor = learn.preprocessing.VocabularyProcessor( max_document_length) vocab_processor.fit(y_data) vocab_processor_x = learn.preprocessing.VocabularyProcessor( 4, vocabulary=vocab_processor.vocabulary_) vocab_processor.save(MODEL_DIR + '/vocab') print(" Vocabulary Size: {:d}".format(len(vocab_processor.vocabulary_))) # Write correspondance 'word ID' to 'word'. with open(MODEL_DIR + '/correspondance.tsv', 'w') as f: f.write('Word ID\tWord\n') for word, word_id in vocab_processor.vocabulary_._mapping.iteritems(): f.write('{}\t{}\n'.format(str(word_id), word)) with tf.Graph().as_default() as graph: #sess = tf_debug.LocalCLIDebugWrapperSession(sess) # Create model. print('Creating model...') model = Regression(number_of_words=len(x_data[0]), sequence_length=LENGTH_MAX, vocab_size=len(vocab_processor.vocabulary_), embedding_size=EMBEDDING_SIZE) # Define Training procedure. global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(LEARNING_RATE) grads_and_vars = optimizer.compute_gradients(model.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Checkpoint directory. checkpoint_path = MODEL_DIR + "/checkpoint.ckpt" saver = tf.train.Saver(tf.global_variables(), max_to_keep=3) with tf.Session(graph=graph) as sess: # Initialize. print('Initializing...') sess.run(tf.global_variables_initializer()) # Maybe restore model parameters. ckpt = tf.train.get_checkpoint_state(MODEL_DIR) if ckpt and tf.gfile.Exists(ckpt.model_checkpoint_path + '.index'): print("Restoring model parameters from %s." % ckpt.model_checkpoint_path) saver.restore(sess, ckpt.model_checkpoint_path) else: print("Fresh parameters for this model.") # Tensorboard. dir_summary = MODEL_DIR + '/summary/' + datetime.datetime.now( ).isoformat() train_writer = tf.summary.FileWriter(dir_summary, sess.graph) merged_summary = tf.summary.merge_all() def train_step(x_batch, y_batch): """ A single training step. """ feed_dict = {model.input_x: x_batch, model.input_y: y_batch} summary, _, step, loss = sess.run( [merged_summary, train_op, global_step, model.loss], feed_dict) train_writer.add_summary(summary, step) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {}".format(time_str, step, loss)) # Generate batches. batch_generator = du.batch_iter(DATA_DIRECTORY, BATCH_SIZE, 200000) # Training loops. while True: x_text, y_text = zip(*batch_generator.next()) x_batch = [" ".join(four_words) for four_words in x_text] x_batch = vocab_processor_x.transform( x_batch ) # list of token sequence = [[1,2,3,4], [5,6,7,8], [7,8,9,10]] y_batch = vocab_processor.transform( y_text ) # list of tokens sequences = [[1,3 2 5 6], [7,8,9,10,12,15,16]] x_batch = np.array([x for x in x_batch]) y_batch = np.array([y for y in y_batch]) # Pad sentences of variable lengths. y_batch = np.concatenate( (y_batch, np.zeros( (len(y_batch), LENGTH_MAX - len(y_batch[1])))), 1) train_step(x_batch, y_batch) current_step = tf.train.global_step(sess, global_step) if current_step % SAVE_EVERY == 0: path = saver.save(sess, checkpoint_path, global_step=current_step) print("Saved model checkpoint to {}\n".format(path))
def using(four_words_in_a_tuple): # Load data. print('Loading data...') try: ## TODO: change try-except with is_file.. with gfile.Open(MODEL_DIR + '/data', 'rb') as f: x_data, y_data = pickle.loads(f.read()) print(' Old data found in {}.'.format(MODEL_DIR + '/data')) except: print("I cannot continue: no data has been found in {}.".format( MODEL_DIR + '/data')) return # Load and save vocabulary. print('Loading vocabulary...') try: vocab_processor = learn.preprocessing.VocabularyProcessor.restore( MODEL_DIR + '/vocab') print(" Old vocabulary found in {}.".format(MODEL_DIR + '/vocab')) except: print("I cannot continue: no vocabulary has been found in {}.".format( MODEL_DIR + '/vocab')) return vocab_processor_x = learn.preprocessing.VocabularyProcessor( 4, vocabulary=vocab_processor.vocabulary_) with tf.Graph().as_default() as graph: #sess = tf_debug.LocalCLIDebugWrapperSession(sess) # Create model. print('Creating model...') model = Regression(number_of_words=len(x_data[0]), sequence_length=LENGTH_MAX, vocab_size=len(vocab_processor.vocabulary_), embedding_size=EMBEDDING_SIZE) # Checkpoint directory. saver = tf.train.Saver(tf.global_variables(), max_to_keep=1) with tf.Session(graph=graph) as sess: # Initialize. print('Initializing...') sess.run(tf.global_variables_initializer()) # Maybe restore model parameters. ckpt = tf.train.get_checkpoint_state(MODEL_DIR) if ckpt and tf.gfile.Exists(ckpt.model_checkpoint_path + '.index'): print("Restoring model parameters from %s." % ckpt.model_checkpoint_path) saver.restore(sess, ckpt.model_checkpoint_path) else: print("I cannot continue: no checkpoint has been found in {}.". format(ckpt.model_checkpoint_path)) return def test_step(x_batch, y_batch): """ A single training step. """ feed_dict = {model.input_x: x_batch, model.input_y: y_batch} scores = sess.run([model.scores], feed_dict) return scores x_text, y_text = zip( *[[four_words_in_a_tuple, 'help <<EOS>> help <<EOS>> help']]) x_batch = [" ".join(four_words) for four_words in x_text] x_batch = vocab_processor_x.transform( x_batch ) # list of token sequence = [[1,2,3,4], [5,6,7,8], [7,8,9,10]] y_batch = vocab_processor.transform( y_text ) # list of tokens sequences = [[1,3 2 5 6], [7,8,9,10,12,15,16]] x_batch = np.array([x for x in x_batch]) y_batch = np.array([y for y in y_batch]) # Padding y_batch = np.concatenate( (y_batch, np.zeros( (len(y_batch), LENGTH_MAX - len(y_batch[0])))), 1) scores = test_step(x_batch, y_batch) return scores
def testing(): tf.reset_default_graph() with tf.Session() as sess: #sess = tf_debug.LocalCLIDebugWrapperSession(sess) # Definition of x_data, y_data for the definition of the model. x_data = [['i'] * 4] * 4 y_data = [ 'man eat dog <<EOS>> help <<EOS>> pie', 'man eat dog <<EOS>> fit <<EOS>> pile', 'man eat dog <<EOS>> form <<EOS>> lip', 'man eat dog god <<EOS>> bye <<EOS>> plot' ] # Creation of the vocabulary max_document_length = max([len(x.split(" ")) for x in y_data]) vocab_processor = learn.preprocessing.VocabularyProcessor( max_document_length) vocab_processor.fit(y_data) vocab_processor_x = learn.preprocessing.VocabularyProcessor( 4, vocabulary=vocab_processor.vocabulary_) print("Vocabulary Size: {:d}".format(len(vocab_processor.vocabulary_))) #print(vocab_processor.vocabulary_._mapping) # print all vocabulary # Definition model # Create model. print('Creating model...') model = Regression(number_of_words=len(x_data[0]), sequence_length=LENGTH_MAX, vocab_size=len(vocab_processor.vocabulary_), embedding_size=3) # Define Training procedure. print('training procedure') global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(0.001) grads_and_vars = optimizer.compute_gradients(model.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Initialize. print('Initialize...') sess.run(tf.global_variables_initializer()) print('End of initialization.') def train_step(x_batch, y_batch): """ A single training step. """ feed_dict = { model.input_x: x_batch, model.input_y: y_batch, } _, step, loss = sess.run([train_op, global_step, model.loss], feed_dict) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {}".format(time_str, step, loss)) # Training loops while True: x_text = (('man', 'dog', 'eat', 'pie'), ('man', 'dog', 'eat', 'pile'), ('man', 'dog', 'eat', 'lip'), ('man', 'dog', 'eat', 'plot')) y_text = ('man eat dog <<EOS>> help <<EOS>> pie', 'man eat dog <<EOS>> fit <<EOS>> pile', 'man eat dog <<EOS>> form <<EOS>> lip', 'man eat dog god <<EOS>> bye <<EOS>> plot') x_batch = [" ".join(four_words) for four_words in x_text] x_batch = vocab_processor_x.transform( x_batch ) # list of token sequence = [[1,2,3,4], [5,6,7,8], [7,8,9,10]] y_batch = vocab_processor.transform( y_text ) # list of tokens sequences = [[1,3 2 5 6], [7,8,9,10,12,15,16]] x_batch = np.array([x for x in x_batch]) y_batch = np.array([y for y in y_batch]) # Padding y_batch = np.concatenate( (y_batch, np.zeros( (len(y_batch), LENGTH_MAX - len(y_batch[1])))), 1) train_step(x_batch, y_batch)
metavar='OUT', help='output dim of model (default: 1)') parser.add_argument('--lr', type=float, default=0.001, metavar='LR', help='learning rate (default: 0.001)') args = parser.parse_args() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print("Using device {}.".format(device)) train_loader = _get_train_data_loader(args.batch_size, args.data_dir) model = Regression(args.input_features, args.hidden_dim1, args.hidden_dim2, args.output_dim).to(device) optimizer = optim.Adam(model.parameters(), lr=args.lr) criterion = RMSELoss train(model, train_loader, args.epochs, criterion, optimizer, device) model_info_path = os.path.join(args.model_dir, 'model_info.pth') with open(model_info_path, 'wb') as f: model_info = { 'input_features': args.input_features, 'hidden_dim1': args.hidden_dim1, 'hidden_dim2': args.hidden_dim2, 'output_dim': args.output_dim, } torch.save(model_info, f)