예제 #1
0
def model_fn(model_dir):
    """
    Loads the PyTorch model from the `model_dir` directory.
    
    :param model_dir: model directory
    :return: model created
    """
    print("Loading model.")

    model_info = {}
    model_info_path = os.path.join(model_dir, 'model_info.pth')
    with open(model_info_path, 'rb') as f:
        model_info = torch.load(f)

    print("model_info: {}".format(model_info))

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model = Regression(model_info['input_features'], model_info['hidden_dim1'],
                       model_info['hidden_dim2'], model_info['output_dim'])

    model_path = os.path.join(model_dir, 'model.pth')
    with open(model_path, 'rb') as f:
        model.load_state_dict(torch.load(f))

    model.to(device).eval()

    print("Done loading model.")
    return model
예제 #2
0
    def __call__(self,
                 number_of_iterations=2,
                 learning_rate=0.005,
                 embedding_size=300):
        print("Starting 'Image Retrieval' in 'Regression' mode with '" +
              self.difficulty + "' data")

        self.model_full_path = self.model_path + "/" + self.model_name + "_" + self.timestamp + "_" + str(
            learning_rate) + "_" + str(embedding_size) + ".pty"
        self.output_file_name = self.output_path + "/" + self.model_name + "_" + self.timestamp + "_" + str(
            learning_rate) + "_" + str(embedding_size) + ".csv"

        self.number_of_iterations = number_of_iterations
        self.learning_rate = learning_rate
        self.embedding_size = embedding_size

        self.model = Regression(self.nwords, self.embedding_size,
                                self.image_feature_size,
                                self.output_vector_size)
        self.criterion = nn.MSELoss()

        self.evaluate = Evaluate(self.model, self.img_features, self.minibatch,
                                 self.preprocess, self.image_feature_size)
        print(self.model)

        self.optimizer = optim.Adam(self.model.parameters(),
                                    lr=self.learning_rate)

        self.train_loss_values = []
        self.dev_loss_values = []
        self.test_loss_values = []

        self.magic()

        self.save_model()

        self.save_data()
예제 #3
0
    def __init__(self, args,
                 trainRegressionDataLoader, trainRegressionClassificationLoader,
                 testDataLoader, trainRainFallLoader,
                 means, std):

        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
        self.trainRegressionDataLoader = trainRegressionDataLoader
        self.trainRegressionClassificationLoader = trainRegressionClassificationLoader
        self.testDataLoader = testDataLoader
        self.classificationLoader = trainRainFallLoader

        self.run_datetime = datetime.datetime.now()

        self.out_path = args.out
        self.sigma = args.sigma
        self.beta = args.beta
        self.earlyStop = args.earlyStop
        self.nClass = args.nClass

        self.noiseMean = torch.zeros(args.batch_size, args.featureNums, 17, 17)
        self.noiseStd = 1e-3

        self.model = AutoencoderBN(self.noiseMean, self.noiseStd).to(self.device)
        self.regressionModel = Regression(self.nClass).to(self.device)
        self.classificationModel = regressionClassification(self.nClass).to(self.device)

        self.rainFallClassifierModel = rainFallClassification().to(self.device)
        self.meanStdNormalizer = MeanVarianceNormalizer(means, std).to(self.device)

        self.meanvarLoss = MeanVarLoss(self.nClass).to(self.device)
        self.normaliedLoss = NormalizerLoss(std).to(self.device)
        self.focalLoss = FocalLoss(self.nClass, alpha=0.25, gamma=2).to(self.device)
        self.rainFocalLoss = FocalLoss(2, alpha=0.25, gamma=2).to(self.device)

        self.regressionOptim = torch.optim.Adam([
            {'params': self.regressionModel.parameters(), 'lr': args.lr,
             'weight_decay': args.weight_decay},
            {'params': self.model.parameters(), 'lr': args.lr,
             'weight_decay': args.weight_decay},
        ],
            lr=args.lr * 10, weight_decay=args.weight_decay * 10)

        self.classificationOptim = torch.optim.Adam(self.classificationModel.parameters(), lr=args.lr * 100)

        self.rainFallOptim = torch.optim.Adam(self.rainFallClassifierModel.parameters(), lr=args.lr * 10)

        # self.reconstructOptim = torch.optim.Adam(self.model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
        self.scheduler = torch.optim.lr_scheduler.StepLR(self.regressionOptim, step_size=750 * 2)

        self.criterion = nn.MSELoss()

        self.classificationCriterion = nn.CrossEntropyLoss()

        if not os.path.exists(self.out_path):
            os.makedirs(self.out_path)

        self.logger = Logger(self.out_path)

        with open(os.path.join(self.out_path, "para.json"), "w") as f:
            json.dump(args.__dict__, f)

        self.epoch = 0
        self.iteration = 0
        self.classificationIteration = 0
        self.rainfallclassificationIteration = 0
        self.test_step = 0
        self.max_epoch = args.epochs
        self.val_interval = args.interval
        self.res = 0
        self.bestConstructLoss = 1e7
        self.bestConstructEpoch = 0
        self.best_error = 1e7;
        self.best_res_epoch = 0
예제 #4
0
def regression():
    # *********************    load the dataset and divide to X&y   ***********************
    from sklearn.datasets import make_blobs
    X, Y = make_blobs(cluster_std=0.9,
                      random_state=20,
                      n_samples=1000,
                      centers=10,
                      n_features=10)

    from Algorithms.ML_.helper.data_helper import split_train_val_test
    X, Xv, y, Yv, Xt, Yt = split_train_val_test(X, Y)
    print(X.shape, y.shape, Xv.shape, Yv.shape, Xt.shape, Yt.shape)

    # *********************   build model    ***********************
    from model import Regression
    from layer import Layer, Dense
    from activation import Activation, Softmax, Sigmoid, ReLU
    from regularization import Regularization, L1, L2, L12
    from optimizer import Vanilla
    model = Regression()
    input_size = X.shape[1]
    hidden_size = 50
    num_classes = 10
    learning_rate, reg_rate = 1e-3, 0.5
    model = Regression([
        Dense(hidden_size,
              input_shape=(input_size, ),
              activation=ReLU(),
              alpha=learning_rate,
              lambda_=reg_rate),
    ])
    model += Dense(num_classes,
                   activation=Softmax(),
                   alpha=learning_rate,
                   lambda_=reg_rate)  # add layer with +=
    model.compile()
    model.describe()
    # *********************    train   ***********************
    loss_train, loss_val = model.train(X,
                                       y,
                                       val=(Xv, Yv),
                                       iter_=5000,
                                       batch=32,
                                       return_loss=True,
                                       verbose=True)

    import matplotlib.pyplot as plt
    plt.plot(range(len(loss_train)), loss_train)
    plt.plot(range(len(loss_val)), loss_val)
    plt.legend(['train', 'val'])
    plt.xlabel('Iteration')
    plt.ylabel('Training loss')
    plt.title('Training Loss history')
    plt.show()
    # *********************    predict   ***********************
    pred_train = model.predict(X)
    pred_val = model.predict(Xv)
    pred_test = model.predict(Xt)

    import metrics
    print('train accuracy=', metrics.accuracy(y, pred_train))
    print('val accuracy=', metrics.accuracy(Yv, pred_val))
    print('test accuracy=', metrics.accuracy(Yt, pred_test))
    print('null accuracy=', metrics.null_accuracy(y))
    import metrics
    metrics.print_metrics(Yt, pred_test)
예제 #5
0
def training():

    # Load data.
    print('Loading data...')
    try:
        with gfile.Open(MODEL_DIR + '/data', 'rb') as f:
            x_data, y_data = pickle.loads(f.read())
        print('  Old data found in {}.'.format(MODEL_DIR + '/data'))
    except:
        print('  Creation of a new set of data.')
        x_data, y_data = zip(*du.load_labels_data(DATA_DIRECTORY))
        with gfile.Open(MODEL_DIR + '/data', 'wb') as f:
            f.write(pickle.dumps((x_data, y_data)))

    # Load and save vocabulary.
    print('Loading vocabulary...')
    try:
        vocab_processor = learn.preprocessing.VocabularyProcessor.restore(
            MODEL_DIR + '/vocab')
        print("  Old vocabulary found in {}.".format(MODEL_DIR + '/vocab'))
    except:
        print("  Creation of a new vocabulary.")
        max_document_length = max([len(x.split(" ")) for x in y_data])
        vocab_processor = learn.preprocessing.VocabularyProcessor(
            max_document_length)
        vocab_processor.fit(y_data)
    vocab_processor_x = learn.preprocessing.VocabularyProcessor(
        4, vocabulary=vocab_processor.vocabulary_)
    vocab_processor.save(MODEL_DIR + '/vocab')
    print("  Vocabulary Size: {:d}".format(len(vocab_processor.vocabulary_)))

    # Write correspondance 'word ID' to 'word'.
    with open(MODEL_DIR + '/correspondance.tsv', 'w') as f:
        f.write('Word ID\tWord\n')
        for word, word_id in vocab_processor.vocabulary_._mapping.iteritems():
            f.write('{}\t{}\n'.format(str(word_id), word))

    with tf.Graph().as_default() as graph:
        #sess = tf_debug.LocalCLIDebugWrapperSession(sess)

        # Create model.
        print('Creating model...')
        model = Regression(number_of_words=len(x_data[0]),
                           sequence_length=LENGTH_MAX,
                           vocab_size=len(vocab_processor.vocabulary_),
                           embedding_size=EMBEDDING_SIZE)

        # Define Training procedure.
        global_step = tf.Variable(0, name="global_step", trainable=False)
        optimizer = tf.train.AdamOptimizer(LEARNING_RATE)
        grads_and_vars = optimizer.compute_gradients(model.loss)
        train_op = optimizer.apply_gradients(grads_and_vars,
                                             global_step=global_step)

        # Checkpoint directory.
        checkpoint_path = MODEL_DIR + "/checkpoint.ckpt"
        saver = tf.train.Saver(tf.global_variables(), max_to_keep=3)

    with tf.Session(graph=graph) as sess:

        # Initialize.
        print('Initializing...')
        sess.run(tf.global_variables_initializer())

        # Maybe restore model parameters.
        ckpt = tf.train.get_checkpoint_state(MODEL_DIR)
        if ckpt and tf.gfile.Exists(ckpt.model_checkpoint_path + '.index'):
            print("Restoring model parameters from %s." %
                  ckpt.model_checkpoint_path)
            saver.restore(sess, ckpt.model_checkpoint_path)
        else:
            print("Fresh parameters for this model.")

        # Tensorboard.
        dir_summary = MODEL_DIR + '/summary/' + datetime.datetime.now(
        ).isoformat()
        train_writer = tf.summary.FileWriter(dir_summary, sess.graph)
        merged_summary = tf.summary.merge_all()

        def train_step(x_batch, y_batch):
            """
            A single training step.
            """
            feed_dict = {model.input_x: x_batch, model.input_y: y_batch}

            summary, _, step, loss = sess.run(
                [merged_summary, train_op, global_step, model.loss], feed_dict)

            train_writer.add_summary(summary, step)
            time_str = datetime.datetime.now().isoformat()
            print("{}: step {}, loss {}".format(time_str, step, loss))

        # Generate batches.
        batch_generator = du.batch_iter(DATA_DIRECTORY, BATCH_SIZE, 200000)

        # Training loops.
        while True:
            x_text, y_text = zip(*batch_generator.next())

            x_batch = [" ".join(four_words) for four_words in x_text]
            x_batch = vocab_processor_x.transform(
                x_batch
            )  # list of token sequence = [[1,2,3,4], [5,6,7,8], [7,8,9,10]]
            y_batch = vocab_processor.transform(
                y_text
            )  # list of tokens sequences = [[1,3 2 5 6], [7,8,9,10,12,15,16]]

            x_batch = np.array([x for x in x_batch])
            y_batch = np.array([y for y in y_batch])

            # Pad sentences of variable lengths.
            y_batch = np.concatenate(
                (y_batch, np.zeros(
                    (len(y_batch), LENGTH_MAX - len(y_batch[1])))), 1)

            train_step(x_batch, y_batch)
            current_step = tf.train.global_step(sess, global_step)
            if current_step % SAVE_EVERY == 0:
                path = saver.save(sess,
                                  checkpoint_path,
                                  global_step=current_step)
                print("Saved model checkpoint to {}\n".format(path))
예제 #6
0
def using(four_words_in_a_tuple):

    # Load data.
    print('Loading data...')
    try:  ## TODO: change try-except with is_file..
        with gfile.Open(MODEL_DIR + '/data', 'rb') as f:
            x_data, y_data = pickle.loads(f.read())
        print('  Old data found in {}.'.format(MODEL_DIR + '/data'))
    except:
        print("I cannot continue: no data has been found in {}.".format(
            MODEL_DIR + '/data'))
        return

    # Load and save vocabulary.
    print('Loading vocabulary...')
    try:
        vocab_processor = learn.preprocessing.VocabularyProcessor.restore(
            MODEL_DIR + '/vocab')
        print("  Old vocabulary found in {}.".format(MODEL_DIR + '/vocab'))
    except:
        print("I cannot continue: no vocabulary has been found in {}.".format(
            MODEL_DIR + '/vocab'))
        return
    vocab_processor_x = learn.preprocessing.VocabularyProcessor(
        4, vocabulary=vocab_processor.vocabulary_)

    with tf.Graph().as_default() as graph:
        #sess = tf_debug.LocalCLIDebugWrapperSession(sess)

        # Create model.
        print('Creating model...')
        model = Regression(number_of_words=len(x_data[0]),
                           sequence_length=LENGTH_MAX,
                           vocab_size=len(vocab_processor.vocabulary_),
                           embedding_size=EMBEDDING_SIZE)

        # Checkpoint directory.
        saver = tf.train.Saver(tf.global_variables(), max_to_keep=1)

    with tf.Session(graph=graph) as sess:

        # Initialize.
        print('Initializing...')
        sess.run(tf.global_variables_initializer())

        # Maybe restore model parameters.
        ckpt = tf.train.get_checkpoint_state(MODEL_DIR)
        if ckpt and tf.gfile.Exists(ckpt.model_checkpoint_path + '.index'):
            print("Restoring model parameters from %s." %
                  ckpt.model_checkpoint_path)
            saver.restore(sess, ckpt.model_checkpoint_path)
        else:
            print("I cannot continue: no checkpoint has been found in {}.".
                  format(ckpt.model_checkpoint_path))
            return

        def test_step(x_batch, y_batch):
            """
            A single training step.
            """
            feed_dict = {model.input_x: x_batch, model.input_y: y_batch}

            scores = sess.run([model.scores], feed_dict)
            return scores

        x_text, y_text = zip(
            *[[four_words_in_a_tuple, 'help <<EOS>> help <<EOS>> help']])

        x_batch = [" ".join(four_words) for four_words in x_text]
        x_batch = vocab_processor_x.transform(
            x_batch
        )  # list of token sequence = [[1,2,3,4], [5,6,7,8], [7,8,9,10]]
        y_batch = vocab_processor.transform(
            y_text
        )  # list of tokens sequences = [[1,3 2 5 6], [7,8,9,10,12,15,16]]

        x_batch = np.array([x for x in x_batch])
        y_batch = np.array([y for y in y_batch])

        # Padding
        y_batch = np.concatenate(
            (y_batch, np.zeros(
                (len(y_batch), LENGTH_MAX - len(y_batch[0])))), 1)
        scores = test_step(x_batch, y_batch)

        return scores
예제 #7
0
def testing():
    tf.reset_default_graph()
    with tf.Session() as sess:
        #sess = tf_debug.LocalCLIDebugWrapperSession(sess)

        # Definition of x_data, y_data for the definition of the model.
        x_data = [['i'] * 4] * 4
        y_data = [
            'man eat dog <<EOS>> help <<EOS>> pie',
            'man eat dog <<EOS>> fit <<EOS>> pile',
            'man eat dog <<EOS>> form <<EOS>> lip',
            'man eat dog god <<EOS>> bye <<EOS>> plot'
        ]

        # Creation of the vocabulary
        max_document_length = max([len(x.split(" ")) for x in y_data])
        vocab_processor = learn.preprocessing.VocabularyProcessor(
            max_document_length)
        vocab_processor.fit(y_data)
        vocab_processor_x = learn.preprocessing.VocabularyProcessor(
            4, vocabulary=vocab_processor.vocabulary_)
        print("Vocabulary Size: {:d}".format(len(vocab_processor.vocabulary_)))
        #print(vocab_processor.vocabulary_._mapping) # print all vocabulary

        # Definition model
        # Create model.
        print('Creating model...')
        model = Regression(number_of_words=len(x_data[0]),
                           sequence_length=LENGTH_MAX,
                           vocab_size=len(vocab_processor.vocabulary_),
                           embedding_size=3)

        # Define Training procedure.
        print('training procedure')
        global_step = tf.Variable(0, name="global_step", trainable=False)
        optimizer = tf.train.AdamOptimizer(0.001)
        grads_and_vars = optimizer.compute_gradients(model.loss)
        train_op = optimizer.apply_gradients(grads_and_vars,
                                             global_step=global_step)

        # Initialize.
        print('Initialize...')
        sess.run(tf.global_variables_initializer())
        print('End of initialization.')

        def train_step(x_batch, y_batch):
            """
            A single training step.
            """
            feed_dict = {
                model.input_x: x_batch,
                model.input_y: y_batch,
            }
            _, step, loss = sess.run([train_op, global_step, model.loss],
                                     feed_dict)
            time_str = datetime.datetime.now().isoformat()
            print("{}: step {}, loss {}".format(time_str, step, loss))

        # Training loops
        while True:
            x_text = (('man', 'dog', 'eat', 'pie'), ('man', 'dog', 'eat',
                                                     'pile'),
                      ('man', 'dog', 'eat', 'lip'), ('man', 'dog', 'eat',
                                                     'plot'))

            y_text = ('man eat dog <<EOS>> help <<EOS>> pie',
                      'man eat dog <<EOS>> fit <<EOS>> pile',
                      'man eat dog <<EOS>> form <<EOS>> lip',
                      'man eat dog god <<EOS>> bye <<EOS>> plot')

            x_batch = [" ".join(four_words) for four_words in x_text]
            x_batch = vocab_processor_x.transform(
                x_batch
            )  # list of token sequence = [[1,2,3,4], [5,6,7,8], [7,8,9,10]]
            y_batch = vocab_processor.transform(
                y_text
            )  # list of tokens sequences = [[1,3 2 5 6], [7,8,9,10,12,15,16]]

            x_batch = np.array([x for x in x_batch])
            y_batch = np.array([y for y in y_batch])

            # Padding
            y_batch = np.concatenate(
                (y_batch, np.zeros(
                    (len(y_batch), LENGTH_MAX - len(y_batch[1])))), 1)

            train_step(x_batch, y_batch)
예제 #8
0
                        metavar='OUT',
                        help='output dim of model (default: 1)')
    parser.add_argument('--lr',
                        type=float,
                        default=0.001,
                        metavar='LR',
                        help='learning rate (default: 0.001)')

    args = parser.parse_args()

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    print("Using device {}.".format(device))

    train_loader = _get_train_data_loader(args.batch_size, args.data_dir)

    model = Regression(args.input_features, args.hidden_dim1, args.hidden_dim2,
                       args.output_dim).to(device)

    optimizer = optim.Adam(model.parameters(), lr=args.lr)
    criterion = RMSELoss

    train(model, train_loader, args.epochs, criterion, optimizer, device)

    model_info_path = os.path.join(args.model_dir, 'model_info.pth')
    with open(model_info_path, 'wb') as f:
        model_info = {
            'input_features': args.input_features,
            'hidden_dim1': args.hidden_dim1,
            'hidden_dim2': args.hidden_dim2,
            'output_dim': args.output_dim,
        }
        torch.save(model_info, f)