Exemplo n.º 1
0
def watcher(event, send_sms, log):
    print "Debug: Watcher started ..."
    try:
        ## Full code for main parent
        if event['type'] == 'send_sms':
            data = event['data']
            res, payloadArr = load_data(data)
            if not res:
              raise Exception

            log(data['id'], len(payloadArr) - 1)              # The last one is the sentinel
            print "Payload array size", len(payloadArr) - 1

            for payload in payloadArr:
                send_sms(payload)
        elif event['type'] == 'external_setup':
            data = event['data']
            result = work_external_data(data)
            print "DEBUG, external event, loaded = "+str(result)
            if not result:
                updateAction(data['id'], 'Data Load Failed', event.get('oid'))
        elif event['type'] == 'update_action':
            data = event['data']
            updateAction(data['id'], data['action'], data.get('oid'))

        return True
    except Exception:
        cLogger.exception("handler recovered over the following")
        return False
    finally:
        pass     # cleanup
Exemplo n.º 2
0
def main(unused_argv):
    # Load training and eval data
    (train_data, train_labels), (eval_data, eval_labels) = data_loader.load_data()

    # Create the Estimator
    mnist_classifier = get_classifier()

    # Set up logging for predictions
    # Log the values in the "Softmax" tensor with label "probabilities"
    tensors_to_log = {"probabilities": "softmax_tensor"}
    logging_hook = tf.train.LoggingTensorHook(
        tensors=tensors_to_log, every_n_iter=500)

    # Train the model
    train_input_fn = tf.estimator.inputs.numpy_input_fn(
        x={"x": train_data},
        y=train_labels,
        batch_size=100,
        num_epochs=None,
        shuffle=False)
    mnist_classifier.train(
        input_fn=train_input_fn,
        steps=10000,
        hooks=[logging_hook])

    # Evaluate the model and print results
    eval_input_fn = tf.estimator.inputs.numpy_input_fn(
        x={"x": eval_data},
        y=eval_labels,
        num_epochs=1,
        shuffle=False)
    eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn)
    print('\nTest set accuracy: {accuracy:0.3f}\n'.format(**eval_results))
Exemplo n.º 3
0
    def train(self, epochs, batch_size=128, sample_interval=50):

        # Load the dataset
        # (X_train, _), (_, _) = mnist.load_data()
        generator = load_data(batch_size, self.img_rows, self.img_cols)

        # Rescale -1 to 1
        # X_train = (X_train.astype(np.float32) - 127.5) / 127.5
        # X_train = np.expand_dims(X_train, axis=3)

        # Adversarial ground truths
        valid = -np.ones((batch_size, 1))
        fake = np.ones((batch_size, 1))

        for epoch in range(epochs):

            for _ in range(self.n_critic):

                # ---------------------
                #  Train Discriminator
                # ---------------------

                # Select a random batch of images
                # idx = np.random.randint(0, X_train.shape[0], batch_size)
                # imgs = X_train[idx]
                imgs, _ = next(generator)
                imgs = imgs / 127.5 - 1.
                if imgs.shape[0] != batch_size:
                    continue

                # Sample noise as generator input
                noise = np.random.normal(0, 1, (batch_size, self.latent_dim))

                # Generate a batch of new images
                gen_imgs = self.generator.predict(noise)

                # Train the critic
                d_loss_real = self.critic.train_on_batch(imgs, valid)
                d_loss_fake = self.critic.train_on_batch(gen_imgs, fake)
                d_loss = 0.5 * np.add(d_loss_fake, d_loss_real)

                # Clip critic weights
                for l in self.critic.layers:
                    weights = l.get_weights()
                    weights = [np.clip(w, -self.clip_value, self.clip_value) for w in weights]
                    l.set_weights(weights)

            # ---------------------
            #  Train Generator
            # ---------------------

            g_loss = self.combined.train_on_batch(noise, valid)

            # Plot the progress
            print("%d [D loss: %f] [G loss: %f]" % (epoch, 1 - d_loss[0], 1 - g_loss[0]))

            # If at save interval => save generated image samples
            if epoch % sample_interval == 0:
                self.sample_images(epoch)
                self._save_weight(epoch, g_loss[0])
Exemplo n.º 4
0
    def train(self, epochs, batch_size=128, save_interval=50):

        # Load the dataset
        # (X_train, _), (_, _) = mnist.load_data()
        generator = load_data(batch_size, self.img_rows, self.img_cols)

        # Rescale -1 to 1
        # X_train = X_train / 127.5 - 1.
        # X_train = np.expand_dims(X_train, axis=3)

        # Adversarial ground truths
        valid = np.ones((batch_size, 1))
        fake = np.zeros((batch_size, 1))

        for epoch in range(epochs):

            # ---------------------
            #  Train Discriminator
            # ---------------------

            # Select a random half of images
            # idx = np.random.randint(0, X_train.shape[0], batch_size)
            # imgs = X_train[idx]
            imgs, _ = next(generator)
            imgs = imgs / 127.5 - 1.
            if imgs.shape[0] != batch_size:
                continue

            # Sample noise and generate a batch of new images
            noise = np.random.uniform(-1, 1, (batch_size, self.latent_dim))
            gen_imgs = self.generator.predict(noise)

            # Train the discriminator (real classified as ones and generated as zeros)
            # X = np.concatenate((imgs, gen_imgs))
            # y = np.concatenate((valid, fake))
            # d_loss = self.discriminator.train_on_batch(X, y)
            d_loss_real = self.discriminator.train_on_batch(imgs, valid)
            d_loss_fake = self.discriminator.train_on_batch(gen_imgs, fake)
            d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)

            # ---------------------
            #  Train Generator
            # ---------------------

            # if epoch < 200:
            #     print('Train %d, d_loss: %f' % (epoch, d_loss[0]))
            # else:
            # Train the generator (wants discriminator to mistake images as real)
            self.discriminator.trainable = False
            g_loss = self.combined.train_on_batch(noise, valid)
            self.discriminator.trainable = True

            # Plot the progress
            print("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100 * d_loss[1], g_loss))

            # If at save interval => save generated image samples
            if epoch % save_interval == 0:
                self.save_imgs(epoch)
                self._save_weight(epoch, d_loss[0])
Exemplo n.º 5
0
def main():
    test_batch, train_batch = data_loader.load_data()
    data_manipulator.categorize(train_batch, test_batch)

    model = neural_net.get_trained_model(train_batches=train_batch,
                                         test_batch=test_batch,
                                         weights_in='weights/1024_1024_256_64_epochs_45',
                                         weights_out='weights/1024_1024_256_64_epochs_50')

    predictions = neural_net.get_predictions(model, test_batch)
    data_saver.save_results("results/result.csv", predictions)
Exemplo n.º 6
0
def main():
    # parse the command line arguments
    parser = NeonArgparser(__doc__)
    parser.add_argument('--output_path', required=True,
                        help='Output path used when training model')
    parser.add_argument('--w2v_path', required=False, default=None,
                        help='Path to GoogleNews w2v file for voab expansion.')
    parser.add_argument('--eval_data_path', required=False, default='./SICK_data',
                        help='Path to the SICK dataset for evaluating semantic relateness')
    parser.add_argument('--max_vocab_size', required=False, default=1000000,
                        help='Limit the vocabulary expansion to fit in GPU memory')
    parser.add_argument('--subset_pct', required=False, default=100,
                        help='subset of training dataset to use (use to retreive \
                        preprocessed data from training)')
    args = parser.parse_args(gen_be=True)

    # load vocab file from training
    _, vocab_file = load_data(args.data_dir, output_path=args.output_path,
                              subset_pct=float(args.subset_pct))
    vocab, _, _ = load_obj(vocab_file)

    vocab_size = len(vocab)
    neon_logger.display("\nVocab size from the dataset is: {}".format(vocab_size))

    index_from = 2  # 0: padding 1: oov
    vocab_size_layer = vocab_size + index_from
    max_len = 30

    # load trained model
    model_dict = load_obj(args.model_file)

    # Vocabulary expansion trick needs to pass the correct vocab set to evaluate (for tokenization)
    if args.w2v_path:
        neon_logger.display("Performing Vocabulary Expansion... Loading W2V...")
        w2v_vocab, w2v_vocab_size = get_w2v_vocab(args.w2v_path,
                                                  int(args.max_vocab_size), cache=True)

        vocab_size_layer = w2v_vocab_size + index_from
        model = load_sent_encoder(model_dict, expand_vocab=True, orig_vocab=vocab,
                                  w2v_vocab=w2v_vocab, w2v_path=args.w2v_path, use_recur_last=True)
        vocab = w2v_vocab
    else:
        # otherwise stick with original vocab size used to train the model
        model = load_sent_encoder(model_dict, use_recur_last=True)

    model.initialize(dataset=(max_len, 1))

    evaluate(model, vocab=vocab, data_path=args.eval_data_path, evaltest=True,
             vocab_size_layer=vocab_size_layer)
Exemplo n.º 7
0
    def train(self, epochs, batch_size, sample_interval=50):

        # Load the dataset
        # (X_train, _), (_, _) = mnist.load_data()
        # generator = load_data(batch_size, self.img_rows, self.img_cols)
        generator = load_data(batch_size, self.img_rows, self.img_cols, imgaug=True, path=ANIME_PATH)

        # Rescale -1 to 1
        # X_train = (X_train.astype(np.float32) - 127.5) / 127.5
        # X_train = np.expand_dims(X_train, axis=3)

        # Adversarial ground truths
        valid = -np.ones((batch_size, 1))
        fake = np.ones((batch_size, 1))
        dummy = np.zeros((batch_size, 1))  # Dummy gt for gradient penalty
        for epoch in range(epochs):

            for _ in range(self.n_critic):
                # ---------------------
                #  Train Discriminator
                # ---------------------

                # Select a random batch of images
                # idx = np.random.randint(0, X_train.shape[0], batch_size)
                # imgs = X_train[idx]
                imgs, _ = next(generator)
                imgs = imgs / 127.5 - 1.
                if imgs.shape[0] != batch_size:
                    continue
                # Sample generator input
                noise = np.random.normal(0, 1, (batch_size, self.latent_dim))
                # Train the critic
                d_loss = self.critic_model.train_on_batch([imgs, noise],
                                                          [valid, fake, dummy])

            # ---------------------
            #  Train Generator
            # ---------------------

            g_loss = self.generator_model.train_on_batch(noise, valid)

            # Plot the progress
            print("%d [D loss: %f] [G loss: %f]" % (epoch, d_loss[0], g_loss))

            # If at save interval => save generated image samples
            if epoch % sample_interval == 0:
                self.sample_images(epoch)
                self._save_weight(epoch, g_loss)
Exemplo n.º 8
0
 def load_data(self):
     if (self.dummy):
         return
     data = data_loader.load_data(config)
     sm = data['sensor.matrix']
     self.coordinate_matrix = data['coordinate.matrix']
     splitData = True
     if (splitData):
         trainingSize = len(sm) / 2
         testSize = len(sm) - trainingSize
         testDataOffset = trainingSize
         trainingData = sm[0:trainingSize]
         testData = sm[testDataOffset:(testSize + trainingSize)]
     else:
         testDataOffset = 0
         trainingData = sm
         testData = sm
     self.training_data = trainingData
     self.test_data = testData
     self.input_dimensions = len(sm[0])
Exemplo n.º 9
0
'''
from scipy.io.matlab.mio import loadmat
from scipy.signal.signaltools import resample
import matplotlib.pyplot as plt
import plotter
import numpy as np
import time
import data_loader
from place_cell_utilities import get_place_cell_activation
from analyzer import Analyzer
from config import config
import place_cell_reliability

if __name__ == '__main__':
    data = data_loader.load_data(config)
    sm = data['sensor.matrix']
    coordm = data['coordinate.matrix']
    sourceDescription = data['description']

    # these have time along the x axis
    #sensorData = np.array(sm)
    #sensorData = np.transpose(sm[1:1000, :]);
    
    inputDims = len(sm[0])
    
    print "inputDims=", inputDims, ", rows=", len(sm)
    
    analyzer = Analyzer(config, inputDims)
        
    #train the flow
Exemplo n.º 10
0
def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000,
             dataset='mnist.pkl.gz', batch_size=20, n_hidden=500,
             activation="tanh",
             rng=numpy.random.RandomState(1234)):

    datasets = data_loader.load_data(dataset)

    train_set_x, train_set_y = datasets["data"][0]
    valid_set_x, valid_set_y = datasets["data"][1]
    test_set_x, test_set_y = datasets["data"][2]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
    n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
    n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size

    ######################
    # BUILD ACTUAL MODEL #
    ######################
    print '... building the model'

    # allocate symbolic variables for the data
    index = T.lscalar()  # index to a [mini]batch
    x = T.matrix('x')  # the data is presented as rasterized images
    y = T.ivector('y')  # the labels are presented as 1D vector of
                        # 

    # construct the MLP class
    classifier = MLP(rng=rng, input=x, n_in=datasets["info"]["xdim"],
                     n_hidden=n_hidden,
                     n_out=datasets["info"]["y_num_categories"],
                     activations=activation)

    cost = (
        classifier.negative_log_likelihood(y)
        + L1_reg * classifier.L1
        + L2_reg * classifier.L2_sqr
    )

    # compiling a Theano function that computes the mistakes that are made
    # by the model on a minibatch
    test_model = theano.function(
        inputs=[index],
        outputs=classifier.errors(y),
        givens={
            x: test_set_x[index * batch_size:(index + 1) * batch_size],
            y: test_set_y[index * batch_size:(index + 1) * batch_size]
        }
    )

    validate_model = theano.function(
        inputs=[index],
        outputs=classifier.errors(y),
        givens={
            x: valid_set_x[index * batch_size:(index + 1) * batch_size],
            y: valid_set_y[index * batch_size:(index + 1) * batch_size]
        }
    )

    # compute the gradient of cost with respect to theta (stored in params)
    # the resulting gradients will be stored in a list gparams
    gparams = [T.grad(cost, param) for param in classifier.params]

    updates = [
        (param, param - learning_rate * gparam)
        for param, gparam in zip(classifier.params, gparams)
    ]

    # compiling a Theano function `train_model` that returns the cost, but
    # at the same time updates the parameters of the model based on the rules
    # defined in `updates`
    train_model = theano.function(
        inputs=[index],
        outputs=cost,
        updates=updates,
        givens={
            x: train_set_x[index * batch_size: (index + 1) * batch_size],
            y: train_set_y[index * batch_size: (index + 1) * batch_size]
        }
    )

    ###############
    # TRAIN MODEL #
    ###############
    print '... training'

    # early-stopping parameters
    patience = 10000  # look at this many examples regardless
    patience_increase = 2  # wait this much longer when a new best is
                           # found
    improvement_threshold = 0.995  # a relative improvement of this much is
                                   # considered significant
    validation_frequency = min(n_train_batches, patience / 2)
                                  # go through this many
                                  # minibatches before checking the network
                                  # on the validation set; in this case we
                                  # check every epoch

    best_validation_loss = numpy.inf
    best_iter = 0
    test_score = 0.
    start_time = timeit.default_timer()

    epoch = 0
    done_looping = False

    while (epoch < n_epochs) and (not done_looping):
        epoch = epoch + 1
        for minibatch_index in xrange(n_train_batches):

            minibatch_avg_cost = train_model(minibatch_index)
            # iteration number
            iter = (epoch - 1) * n_train_batches + minibatch_index

            if (iter + 1) % validation_frequency == 0:
                # compute zero-one loss on validation set
                validation_losses = [validate_model(i) for i
                                     in xrange(n_valid_batches)]
                this_validation_loss = numpy.mean(validation_losses)

                print(
                    'epoch %i, minibatch %i/%i, validation error %f %%' %
                    (
                        epoch,
                        minibatch_index + 1,
                        n_train_batches,
                        this_validation_loss * 100.
                    )
                )

                # if we got the best validation score until now
                if this_validation_loss < best_validation_loss:
                    #improve patience if loss improvement is good enough
                    if (
                        this_validation_loss < best_validation_loss *
                        improvement_threshold
                    ):
                        patience = max(patience, iter * patience_increase)

                    best_validation_loss = this_validation_loss
                    best_iter = iter

                    # test it on the test set
                    test_losses = [test_model(i) for i
                                   in xrange(n_test_batches)]
                    test_score = numpy.mean(test_losses)

                    print(('     epoch %i, minibatch %i/%i, test error of '
                           'best model %f %%') %
                          (epoch, minibatch_index + 1, n_train_batches,
                           test_score * 100.))

            if patience <= iter:
                done_looping = True
                break

    end_time = timeit.default_timer()

    # Store experiment information
    ex.info["run_time"] = end_time - start_time
    ex.info["validation_perf"] = best_validation_loss * 100
    ex.info["num_epochs"] = epoch
    print(('Optimization complete. Best validation score of %f %% '
           'obtained at iteration %i, with test performance %f %%') %
          (best_validation_loss * 100., best_iter + 1, test_score * 100.))
Exemplo n.º 11
0
from data_loader import load_data
from train import train

parser = argparse.ArgumentParser()
parser.add_argument('--train_file', type=str, default='news/train.txt', help='path to the training file')
parser.add_argument('--test_file', type=str, default='news/test.txt', help='path to the test file')
parser.add_argument('--transform', type=bool, default=True, help='whether to transform entity embeddings')
parser.add_argument('--use_context', type=bool, default=False, help='whether to use context embeddings')
parser.add_argument('--max_click_history', type=int, default=30, help='number of sampled click history for each user')
parser.add_argument('--n_filters', type=int, default=128, help='number of filters for each size in KCNN')
parser.add_argument('--filter_sizes', type=int, default=[1, 2], nargs='+',
                    help='list of filter sizes, e.g., --filter_sizes 2 3')
parser.add_argument('--l2_weight', type=float, default=0.01, help='weight of l2 regularization')
parser.add_argument('--lr', type=float, default=0.001, help='learning rate')
parser.add_argument('--batch_size', type=int, default=128, help='number of samples in one batch')
parser.add_argument('--n_epochs', type=int, default=10, help='number of training epochs')
parser.add_argument('--KGE', type=str, default='TransE',
                    help='knowledge graph embedding method, please ensure that the specified input file exists')
parser.add_argument('--entity_dim', type=int, default=50,
                    help='dimension of entity embeddings, please ensure that the specified input file exists')
parser.add_argument('--word_dim', type=int, default=50,
                    help='dimension of word embeddings, please ensure that the specified input file exists')
parser.add_argument('--max_title_length', type=int, default=10,
                    help='maximum length of news titles, should be in accordance with the input datasets')
args = parser.parse_args()


train_data, test_data = load_data(args)
train(args, train_data, test_data)

Exemplo n.º 12
0
def main():
    global nameBit
    names = ["sims_01_bifurcation_noninformative"]
    flag_informative = False
    flag_adapt = False
    best_error = True
    for namecounter in range(len(names)):
        nameNow = names[namecounter]
        (tsim, XK, YK, mu0, P0, Ns, dt, tf) = data_loader.load_data(nameNow, "../sim_data/")

        """
		tsim = tsim[0:5]
		XK = XK[0:5,:]
		YK = YK[0:5,:]
		tf = tsim[4]
		"""
        Ns = 100

        nameBit = int(nameNow[5:7], 2)
        # parse the name
        if nameBit == 1:
            # noise levels for the ENKF with white noise forcing
            Qk = np.array([[10.0]])
            Rk = np.array([[0.01]])
        if nameBit == 2:
            # noise levels for the UKF with cosine forcing
            Qk = np.array([[3.16 / dt]])
            Rk = np.array([[0.1]])
            # number of steps in each simulation
        nSteps = len(tsim)
        nees_history = np.zeros((nSteps, Ns))
        Nf_history = np.zeros((nSteps, Ns))
        e_sims = np.zeros((Ns * nSteps, 2))
        for counter in range(Ns):
            xk = XK[:, (2 * counter) : (2 * counter + 2)]
            yk = YK[:, counter]

            (Xf, Pf, Idx, Xp) = enkf_test(dt, tf, mu0, P0, yk, Qk, Rk, flag_adapt, flag_informative)
            print("enkf_clustering case %d/%d" % (counter + 1, Ns))

            if Ns == 1:
                fig = []
                for k in range(nSteps):
                    fig.append(plt.figure())
                    ax = fig[k].add_subplot(
                        1, 1, 1, title="t = %f" % (tsim[k]), xlim=(-25, 25), ylim=(-20, 20), ylabel="x2", xlabel="x1"
                    )
                    # compute the number of active means
                    meansIdx = Idx[k, :].copy()
                    activeMeans = 1
                    if np.any(meansIdx > 0):
                        activeMeans = 2
                    for jk in range(activeMeans):
                        idx = np.nonzero(meansIdx == jk)
                        idx = idx[0]
                        mux = np.mean(Xf[k, :, idx], axis=0)
                        Pxx = np.zeros((2, 2))
                        for kj in idx:
                            Pxx = Pxx + 1.0 / (float(len(idx)) - 1.0) * np.outer(Xf[k, :, kj] - mux, Xf[k, :, kj] - mux)
                        mux0 = np.mean(Xp[k, :, idx], axis=0)
                        Pxx0 = np.zeros((2, 2))
                        for kj in idx:
                            Pxx0 = Pxx0 + 1.0 / (float(len(idx)) - 1.0) * np.outer(
                                Xp[k, :, kj] - mux0, Xp[k, :, kj] - mux0
                            )
                        if jk == 0:
                            ax.plot(Xf[k, 0, idx], Xf[k, 1, idx], "mo")
                            ax.plot(Xp[k, 0, idx], Xp[k, 1, idx], "bd")
                        else:
                            ax.plot(Xf[k, 0, idx], Xf[k, 1, idx], "yo")
                            ax.plot(Xp[k, 0, idx], Xp[k, 1, idx], "rd")
                            # plot the single-mean covariance ellipsoid
                            # draw points on a unit circle
                        thetap = np.linspace(0, 2 * math.pi, 20)
                        circlP = np.zeros((20, 2))
                        circlP[:, 0] = 3.0 * np.cos(thetap)
                        circlP[:, 1] = 3.0 * np.sin(thetap)
                        # transform the points circlP through P^(1/2)*circlP + mu
                        Phalf = np.real(scipy.linalg.sqrtm(Pxx))
                        ellipsP = np.zeros(circlP.shape)
                        for kj in range(circlP.shape[0]):
                            ellipsP[kj, :] = np.dot(Phalf, circlP[kj, :]) + mux
                        if jk == 0:
                            ax.plot(ellipsP[:, 0], ellipsP[:, 1], "m--")
                        else:
                            ax.plot(ellipsP[:, 0], ellipsP[:, 1], "y--")
                            # transform the points circlP through P^(1/2)*circlP + mu
                        Phalf = np.real(scipy.linalg.sqrtm(Pxx0))
                        ellipsP = np.zeros(circlP.shape)
                        for kj in range(circlP.shape[0]):
                            ellipsP[kj, :] = np.dot(Phalf, circlP[kj, :]) + mux
                        if jk == 0:
                            ax.plot(ellipsP[:, 0], ellipsP[:, 1], "b--")
                        else:
                            ax.plot(ellipsP[:, 0], ellipsP[:, 1], "r--")
                            # plot the truth state
                        ax.plot(xk[k, 0], xk[k, 1], "cs")
                    ax.grid()
                    fig[k].show()
                raw_input("Return to quit")
                for k in range(nSteps):
                    fig[k].savefig("stepByStep/enkf_" + str(Xf.shape[2]) + "_" + str(k) + ".png")
                    plt.close(fig[k])

            (e1, chi2, mx, Pk) = cluster_processing.singleSimErrors(Xf, Idx, xk, yk, best_error)
            nees_history[:, counter] = chi2.copy()
            mean_nees = np.sum(chi2) / float(nSteps)
            print(mean_nees)
            # mean NEES
            mse = np.sum(np.power(e1, 2.0), axis=0) / float(nSteps)
            e_sims[(counter * nSteps) : (counter * nSteps + nSteps), :] = e1.copy()

            print("MSE: %f,%f" % (mse[0], mse[1]))

        if Ns < 2:
            # plot the mean trajectories and error
            fig1 = plt.figure()
            ax = []
            for k in range(4):
                if k < 2:
                    nam = "x" + str(k + 1)
                else:
                    nam = "e" + str(k - 1)
                ax.append(fig1.add_subplot(2, 2, k + 1, ylabel=nam))
                if k < 2:
                    ax[k].plot(tsim, xk[:, k], "b-")
                    ax[k].plot(tsim, mx[:, k], "m--")
                    """
					if k == 0:
						ax[k].plot(tsim,yk,'r--')
					"""
                else:
                    ax[k].plot(tsim, e1[:, k - 2])
                    ax[k].plot(tsim, 3.0 * np.sqrt(Pk[:, k - 2, k - 2]), "r--")
                    ax[k].plot(tsim, -3.0 * np.sqrt(Pk[:, k - 2, k - 2]), "r--")
                ax[k].grid()
            fig1.show()
        else:
            if best_error:
                mse_tot = np.mean(np.power(e_sims, 2.0), axis=0)
                print("mse_tot: %f,%f" % (mse_tot[0], mse_tot[1]))

                # get the mean NEES value versus simulation time across all sims
                nees_mean = np.sum(nees_history, axis=1) / Ns
                # get the mean number of particles in time
                Nf_mean = np.sum(Nf_history, axis=1) / Ns
                # get 95% confidence bounds for chi-sqaured... the df is the number of sims times the dimension of the state
                chiUpper = stats.chi2.ppf(0.975, 2.0 * Ns) / float(Ns)
                chiLower = stats.chi2.ppf(0.025, 2.0 * Ns) / float(Ns)
                # find fraction of inliers
                l1 = (nees_mean < chiUpper).nonzero()[0]
                l2 = (nees_mean > chiLower).nonzero()[0]
                # get number of inliers
                len_in = len(set(l1).intersection(l2))
                # get number of super (above) liers (sic)
                len_super = len((nees_mean > chiUpper).nonzero()[0])
                # get number of sub-liers (below)
                len_sub = len((nees_mean < chiLower).nonzero()[0])

                print("Conservative (below 95%% bounds): %f" % (float(len_sub) / float(nSteps)))
                print("Optimistic (above 95%% bounds): %f" % (float(len_super) / float(nSteps)))

                # save metrics
                FID = open("bestErrors_enkf_" + str(Xf.shape[2]) + "_" + nameNow + ".txt", "w")
                FID.write("mse1,mse2,nees_below95,nees_above95\n")
                FID.write(
                    "%f,%f,%f,%f\n"
                    % (mse_tot[0], mse_tot[1], float(len_sub) / float(nSteps), float(len_super) / float(nSteps))
                )
                FID.close()
            else:
                print("Passing to exit")
                pass
                mse_tot = np.mean(np.power(e_sims, 2.0), axis=0)
                print("mse_tot: %f,%f" % (mse_tot[0], mse_tot[1]))

                # get the mean NEES value versus simulation time across all sims
                nees_mean = np.sum(nees_history, axis=1) / Ns
                # get the mean number of particles in time
                Nf_mean = np.sum(Nf_history, axis=1) / Ns
                # get 95% confidence bounds for chi-sqaured... the df is the number of sims times the dimension of the state
                chiUpper = stats.chi2.ppf(0.975, 2.0 * Ns) / float(Ns)
                chiLower = stats.chi2.ppf(0.025, 2.0 * Ns) / float(Ns)

                # plot the mean NEES with the 95% confidence bounds
                fig2 = plt.figure(figsize=(6.0, 3.37))  # figsize tuple is width, height
                tilt = "ENKF, Ts = %.2f, %d sims, " % (dt, Ns)
                if nameBit == 0:
                    tilt = tilt + "unforced"
                if nameBit == 1:
                    # white-noise only
                    tilt = tilt + "white-noise forcing"
                if nameBit == 2:
                    tilt = tilt + "cosine forcing"
                if nameBit == 3:
                    # white-noise and cosine forcing
                    tilt = tilt + "white-noise and cosine forcing"
                ax = fig2.add_subplot(111, ylabel="mean NEES", title=tilt)
                ax.plot(tsim, chiUpper * np.ones(nSteps), "r--")
                ax.plot(tsim, chiLower * np.ones(nSteps), "r--")
                ax.plot(tsim, nees_mean, "b-")
                ax.grid()
                fig2.show()
                # save the figure
                fig2.savefig("nees_enkf2_" + str(Xf.shape[2]) + "_" + nameNow + ".png")
                # find fraction of inliers
                l1 = (nees_mean < chiUpper).nonzero()[0]
                l2 = (nees_mean > chiLower).nonzero()[0]
                # get number of inliers
                len_in = len(set(l1).intersection(l2))
                # get number of super (above) liers (sic)
                len_super = len((nees_mean > chiUpper).nonzero()[0])
                # get number of sub-liers (below)
                len_sub = len((nees_mean < chiLower).nonzero()[0])

                print("Conservative (below 95%% bounds): %f" % (float(len_sub) / float(nSteps)))
                print("Optimistic (above 95%% bounds): %f" % (float(len_super) / float(nSteps)))

                # save metrics
                FID = open("metrics_enkf2_" + str(Xf.shape[2]) + "_" + nameNow + ".txt", "w")
                FID.write("mse1,mse2,nees_below95,nees_above95\n")
                FID.write(
                    "%f,%f,%f,%f\n"
                    % (mse_tot[0], mse_tot[1], float(len_sub) / float(nSteps), float(len_super) / float(nSteps))
                )
                FID.close()

    raw_input("Return to exit")
    return
Exemplo n.º 13
0
parser.add_argument('--n_epoch', type=int, default=10, help='the number of epochs')
parser.add_argument('--n_memory', type=int, default=32, help='size of ripple set for each hop')
parser.add_argument('--item_update_mode', type=str, default='plus_transform',
                    help='how to update item at the end of each hop')
parser.add_argument('--using_all_hops', type=bool, default=True,
                    help='whether using outputs of all hops or just the last hop when making prediction')

'''
# default settings for Book-Crossing
parser = argparse.ArgumentParser()
parser.add_argument('--dataset', type=str, default='book', help='which dataset to use')
parser.add_argument('--dim', type=int, default=4, help='dimension of entity and relation embeddings')
parser.add_argument('--n_hop', type=int, default=2, help='maximum hops')
parser.add_argument('--kge_weight', type=float, default=1e-2, help='weight of the KGE term')
parser.add_argument('--l2_weight', type=float, default=1e-5, help='weight of the l2 regularization term')
parser.add_argument('--lr', type=float, default=1e-3, help='learning rate')
parser.add_argument('--batch_size', type=int, default=1024, help='batch size')
parser.add_argument('--n_epoch', type=int, default=10, help='the number of epochs')
parser.add_argument('--n_memory', type=int, default=32, help='size of ripple set for each hop')
parser.add_argument('--item_update_mode', type=str, default='plus_transform',
                    help='how to update item at the end of each hop')
parser.add_argument('--using_all_hops', type=bool, default=True,
                    help='whether using outputs of all hops or just the last hop when making prediction')
'''

args = parser.parse_args()

show_loss = False
data_info = load_data(args)
train(args, data_info, show_loss)
Exemplo n.º 14
0
    model.add(MaxPooling2D((2, 2), strides=(2, 2)))

    model.add(Flatten())
    model.add(Dense(512, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(512, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(8, activation='softmax'))

    if weights_path:
        model.load_weights(weights_path)

    return model

if __name__ == '__main__':
    import data_loader
    train, valid, test = data_loader.load_data()
    model = VGG_16_deep(shape = (1,64,64))
    sgd = SGD(lr=0.005, decay=1e-6, momentum=0.9, nesterov=True)
    model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])
    X_train = np.array(train[0]).reshape(len(train[0]),1,64,64)
    X_valid =  np.array(valid[0]).reshape(len(valid[0]),1,64,64)
    X_test = np.array(test[0]).reshape(len(test[0]),1,64,64)
    from keras.utils.np_utils import to_categorical
    Y_train = to_categorical(np.array(train[1]))
    Y_valid = to_categorical(np.array(valid[1]))
    Y_test = to_categorical(np.array(test[1]))
    train_out = model.fit(X_train,Y_train,nb_epoch=100,batch_size=30,validation_data=(X_valid,Y_valid))
    model.save_weights('my_model_weights.h5')
    test_out = model.evaluate(X_test, Y_test, batch_size=30, verbose=1, sample_weight=None)
Exemplo n.º 15
0
parser.add_argument('--vector_name', required=True,
                    help='the cached data file for all the sentence vectors')
parser.add_argument('--output_dir', required=True,
                    help='directory to save/load the saved datasets')
args = parser.parse_args(gen_be=False)

# hyperparameters from the reference
args.batch_size = 1
valid_split = None

# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))

# load the documents by giving the path and what extension the files are
data_file, vocab_file = load_data(args.data_dir,
                                  valid_split=valid_split,
                                  output_path=args.output_dir)
vocab, rev_vocab, word_count = load_obj(vocab_file)

vocab_size = len(vocab)
neon_logger.display("\nVocab size from the dataset is: {}".format(vocab_size))

index_from = 2  # 0: padding 1: oov
oov = 1
vocab_size_layer = vocab_size + index_from
max_len = 30

# load trained model
model_dict = load_obj(args.model_file)
model = load_sent_encoder(model_dict)
Exemplo n.º 16
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--split', type=str, default='train', help='train/val')
    parser.add_argument('--model_path',
                        type=str,
                        default='data/vgg16.tfmodel',
                        help='Pre-trained VGG16 Model')
    parser.add_argument('--data_dir',
                        type=str,
                        default='data',
                        help='Data directory')
    parser.add_argument('--batch_size',
                        type=int,
                        default=10,
                        help='Batch Size')

    args = parser.parse_args()

    vgg_file = open(args.model_path)
    vgg16raw = vgg_file.read()
    vgg_file.close()

    graph_def = tf.GraphDef()
    graph_def.ParseFromString(vgg16raw)

    images = tf.placeholder("float", [None, 224, 224, 3])
    tf.import_graph_def(graph_def, input_map={"images": images})

    graph = tf.get_default_graph()

    for opn in graph.get_operations():
        print("Name", opn.name, opn.values())

    all_data = data_loader.load_data()
    print(all_data)
    if args.split == "train":
        qa_data = all_data['training']
    else:
        qa_data = all_data['validation']

    image_ids = {}
    for qa in qa_data:
        image_ids[qa['image_id']] = 1

    image_id_list = [img_id for img_id in image_ids]
    print("Total Images", len(image_id_list))

    sess = tf.Session()
    fc7 = np.ndarray((len(image_id_list), 4096))
    idx = 0

    while idx < len(image_id_list):
        start = time.clock()
        image_batch = np.ndarray((args.batch_size, 224, 224, 3))

        count = 0
        for i in range(0, args.batch_size):
            if idx >= len(image_id_list):
                break
            image_file = join(args.data_dir,
                              'Group6/%d.jpg' % image_id_list[idx])
            image_batch[i, :, :, :] = utils.load_image_array(image_file)
            idx += 1
            count += 1

        feed_dict = {images: image_batch[0:count, :, :, :]}
        fc7_tensor = graph.get_tensor_by_name("import/Relu_1:0")
        fc7_batch = sess.run(fc7_tensor, feed_dict=feed_dict)
        fc7[(idx - count):idx, :] = fc7_batch[0:count, :]
        end = time.clock()
        print("Time for batch 10 photos", end - start)
        print("Hours For Whole Dataset",
              (len(image_id_list) * 1.0) * (end - start) / 60.0 / 60.0 / 10.0)

        print("Images Processed", idx)

    print("Saving fc7 features")
    h5f_fc7 = h5py.File(join(args.data_dir, args.split + '_fc7.h5'), 'w')
    h5f_fc7.create_dataset('fc7_features', data=fc7)
    h5f_fc7.close()

    print("Saving image id list")
    h5f_image_id_list = h5py.File(
        join(args.data_dir, args.split + '_image_id_list.h5'), 'w')
    h5f_image_id_list.create_dataset('image_id_list', data=image_id_list)
    h5f_image_id_list.close()
    print("Done!")
Exemplo n.º 17
0
            return "No rules generated"
        else:
            for k, v in self.rules.items():
                out += "-------------Rules for class " + k + "-------------\n"
                for i, r in enumerate(v):
                    out += str(i + 1) + ". "
                    for ii, c in enumerate(sorted(r, key=len)):
                        if ii < len(r) - 1:
                            out += c + " and "
                        else:
                            out += c + "\n"
                            out += "Cov: " + str(
                                round(
                                    v[r]['coverage'] / self.predicted_samples,
                                    4))
                            if v[r]['coverage'] > 0:
                                out += "    Acc: " + str(
                                    v[r]['correct'] / v[r]['coverage']) + "\n"
                            else:
                                out += "\n"
                out += "\n\n"
            return out


if __name__ == '__main__':
    X, y, names = load_data('divorce.csv')
    p = Prism()
    p.fit(X[:-10, :], y[:-10], names)
    print(p)
    p = p.predict(X[-10:, :])
Exemplo n.º 18
0
cnn_w = [1, 2, 3, 4, 5]
cnn_h = [25, 50, 75, 100, 125]
sum_h = sum(cnn_h)

# Training Parameters
batch_size = 20
num_epochs = 30
learning_rate = 0.001
momentum = (0.9, 0.999)
evaluate_every = 3

# Data Preparation
# ===========================================================
# Load data
print("Loading data...")
id2cult, id2comp, train_cult, train_comp, train_comp_len, test_cult, test_comp, test_comp_len, max_comp_cnt = data_loader.load_data(
    train_file)

print("Train/Test/Cult/Comp: {:d}/{:d}/{:d}/{:d}".format(
    len(train_cult), len(test_cult), len(id2cult), len(id2comp)))
print(
    "=================================================================================="
)


class ConvModule(nn.Module):
    def __init__(self, input_size, kernel_sizes, comp_cnt):
        super(ConvModule, self).__init__()

        # attributes:
        self.maxlen = max_comp_cnt
        self.in_channels = input_size
Exemplo n.º 19
0
    parser.add_argument('weak_learner', help='chosen weak learner')
    parser.add_argument(
        'trials',
        help=
        'number of trials (each with different shuffling of the data); defaults to 1',
        type=int,
        default=1,
        nargs='?')
    parser.add_argument('--record',
                        action='store_const',
                        const=True,
                        default=False,
                        help='export the results in file')
    args = parser.parse_args()

    X, y = data_loader.load_data(args.dataset)

    algorithms = {
        "ogboost": ogboost.OGBoost,
        "osboost": osboost.OSBoost,
        "ocpboost": ocpboost.OCPBoost,
        "expboost": expboost.EXPBoost,
        "smoothboost": smoothboost.SmoothBoost,
        "ozaboost": ozaboost.OzaBoostClassifier
    }

    weak_learners = {
        "sk_perceptron": sk_perceptron.PerceptronClassifier,
        "sk_nb": sk_nb.NaiveBayes,
        "gaussian_nb": nb_gaussian.NaiveBayes,
        "nb": nb.NaiveBayes,
Exemplo n.º 20
0
def main():
    file_path = 'spam.csv'
    data = load_data(file_path)
    preprocess_data(data)

    X_pyth = []
    y_pyth = []
    for d in data:
        text = d['text']
        entry = (features.currency_count(text), features.url_count(text),
                 features.word_count(text),
                 features.longest_numerical_string(text),
                 features.average_word_length(text),
                 features.num_win_occurences(text),
                 features.num_free_occurences(text))

        X_pyth.append(entry)

        if d['category'] == 'spam':
            y_pyth.append([1, 0])
        else:
            y_pyth.append([0, 1])

    X = np.array(X_pyth)
    y = np.array(y_pyth)

    # Randomly shuffle data
    p = np.random.permutation(len(y_pyth))
    X = X[p]
    y = y[p]

    # Split into training and testing datasets
    X_train = X[0:4000]
    y_train = y[0:4000]

    X_test = X[4001:5571]
    y_test = y[4001:5571]

    model = get_neural_network(X.shape[1])
    plot_model(model, to_file='model.png')

    return
    model.fit(X_train, y_train, epochs=100, batch_size=512)

    predictions = model.predict(X_test)
    fitted_predictions = []

    # Only classify as spam if we are 90% sure
    for prediction in predictions:
        if prediction[0] > 0.9:
            fitted_predictions.append([1, 0])
        else:
            fitted_predictions.append([0, 1])

    fitted_predictions = np.array(fitted_predictions)

    accuracy = accuracy_score(y_test, fitted_predictions)
    print(accuracy)

    matrix = confusion_matrix(y_test.argmax(axis=1),
                              fitted_predictions.argmax(axis=1))
    binary = np.array(matrix)

    fig, ax = plot_confusion_matrix(conf_mat=binary)
    plt.show()
Exemplo n.º 21
0
up_time = 1440  # 1d
lw_time = 50  # 50m
up_dist = 100  # ??
lw_dist = 1

# Training Parameters
batch_size = 2
learning_rate = 0.001
momentum = 0.9
evaluate_every = 1

# Data Preparation
# ===========================================================
# Load data
print("Loading data...")
user_cnt, poi2id, train_user, train_time, train_lati, train_longi, train_loc, valid_user, valid_time, valid_lati, valid_longi, valid_loc, test_user, test_time, test_lati, test_longi, test_loc = data_loader.load_data(
    train_file)

#np.save("poi2id_30", poi2id)
print("User/Location: {:d}/{:d}".format(user_cnt, len(poi2id)))
print(
    "=================================================================================="
)


class STRNNModule(nn.Module):
    def __init__(self):
        super(STRNNModule, self).__init__()

        # embedding:
        self.user_weight = Variable(torch.randn(user_cnt, dim),
                                    requires_grad=False).type(ftype)
Exemplo n.º 22
0
def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000,
                           dataset='mnist.pkl.gz',
                           batch_size=600):

    datasets = data_loader.load_data(dataset)

    train_set_x, train_set_y = datasets["data"][0]
    valid_set_x, valid_set_y = datasets["data"][1]
    test_set_x, test_set_y = datasets["data"][2]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
    n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
    n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size

    print '... building the model'
    # allocate symbolic variables for the data
    index = T.lscalar()  # index to a [mini]batch
    # generate symbolic variables for input (x and y represent a
    # minibatch)
    x = T.matrix('x')  # data, presented as rasterized images
    y = T.ivector('y')  # labels, presented as 1D vector of [int] labels

    # construct the logistic regression class

    classifier = logistic_layer.LogisticRegression(
        input=x, n_in=datasets["info"]["xdim"],
        n_out=datasets["info"]["y_num_categories"])

    # the cost we minimize during training is the negative log likelihood of
    # the model in symbolic format
    cost = classifier.negative_log_likelihood(y)

    # compiling a Theano function that computes the mistakes that are made by
    # the model on a minibatch
    test_model = theano.function(
        inputs=[index],
        outputs=classifier.errors(y),
        givens={
            x: test_set_x[index * batch_size: (index + 1) * batch_size],
            y: test_set_y[index * batch_size: (index + 1) * batch_size]
        }
    )
    validate_model = theano.function(
        inputs=[index],
        outputs=classifier.errors(y),
        givens={
            x: valid_set_x[index * batch_size: (index + 1) * batch_size],
            y: valid_set_y[index * batch_size: (index + 1) * batch_size]
        }
    )

    # compute the gradient of cost with respect to theta = (W,b)
    g_W = T.grad(cost=cost, wrt=classifier.W)
    g_b = T.grad(cost=cost, wrt=classifier.b)

    # start-snippet-3
    # specify how to update the parameters of the model as a list of
    # (variable, update expression) pairs.
    updates = [(classifier.W, classifier.W - learning_rate * g_W),
               (classifier.b, classifier.b - learning_rate * g_b)]

    # compiling a Theano function `train_model` that returns the cost, but in
    # the same time updates the parameter of the model based on the rules
    # defined in `updates`
    train_model = theano.function(
        inputs=[index],
        outputs=cost,
        updates=updates,
        givens={
            x: train_set_x[index * batch_size: (index + 1) * batch_size],
            y: train_set_y[index * batch_size: (index + 1) * batch_size]
        }
    )
    # end-snippet-3

    ###############
    # TRAIN MODEL #
    ###############

    # Below, the iter counter counts the number of minibatch steps we've
    # taken.

    print '... training the model'
    # early-stopping parameters
    # look as this many examples regardless (I think patience
    # is actually in the units of minibatches)
    patience = 5000
    patience_increase = 2  # wait this much longer when a new best is
    # found
    improvement_threshold = 0.995  # a relative improvement of this much is
    # considered significant
    validation_frequency = min(n_train_batches, patience / 2)
    # go through this many
    # minibatches before checking the network
    # on the validation set; in this case we
    # check every epoch

    best_validation_loss = numpy.inf
    test_score = 0.
    start_time = timeit.default_timer()

    done_looping = False
    epoch = 0
    while (epoch < n_epochs) and (not done_looping):
        epoch = epoch + 1
        for minibatch_index in xrange(n_train_batches):

            minibatch_avg_cost = train_model(minibatch_index)
            # iteration number (minibatch number)
            iter = (epoch - 1) * n_train_batches + minibatch_index

            if (iter + 1) % validation_frequency == 0:
                # compute zero-one loss on validation set
                validation_losses = [validate_model(i)
                                     for i in xrange(n_valid_batches)]
                this_validation_loss = numpy.mean(validation_losses)

                print(
                    'epoch %i, minibatch %i/%i, validation error %f %%' %
                    (
                        epoch,
                        minibatch_index + 1,
                        n_train_batches,
                        this_validation_loss * 100.
                    )
                )

                # if we got the best validation score until now
                if this_validation_loss < best_validation_loss:
                    #improve patience if loss improvement is good enough
                    if this_validation_loss < best_validation_loss *  \
                       improvement_threshold:
                        patience = max(patience, iter * patience_increase)

                    best_validation_loss = this_validation_loss
                    # test it on the test set

                    test_losses = [test_model(i)
                                   for i in xrange(n_test_batches)]
                    test_score = numpy.mean(test_losses)

                    print(
                        (
                            '     epoch %i, minibatch %i/%i, test error of'
                            ' best model %f %%'
                        ) %
                        (
                            epoch,
                            minibatch_index + 1,
                            n_train_batches,
                            test_score * 100.
                        )
                    )

                    # save the best model
                    with open('best_model.pkl', 'w') as f:
                        cPickle.dump(classifier, f)

            if patience <= iter:
                done_looping = True
                break

    end_time = timeit.default_timer()
    print(
        (
            'Optimization complete with best validation score of %f %%,'
            'with test performance %f %%'
        )
        % (best_validation_loss * 100., test_score * 100.)
    )
    print 'The code run for %d epochs, with %f epochs/sec' % (
        epoch, 1. * epoch / (end_time - start_time))
    try:
        print >> sys.stderr, ('The code for file ' +
                              os.path.split(__file__)[1] +
                              ' ran for %.1fs' % ((end_time -
                                                   start_time)))
    except:
        print >> sys.stderr, ('The code (run interactively)' +
                              'ran for %.1fs' % ((end_time - start_time)))
Exemplo n.º 23
0
ideas not implemented:
dropout
Hessian technique fuer grdient calculation
other activation function : rectified linear neurons, tanh()...

learning-rate- schedule, learning rate halves if validation_accuracy satisfies the no-improvement_in_20 rule and terminater if the learning rate has 1/128 of it's original value.
momentum based gradient descent
"""


file_directory="."
file_name="TH2D_A00_TB10.root"
hist_name="LHCChi2_CMSSM_nObs1061_A00_TB10"

full_data,training_data,test_data=data_loader.load_data(10000,file_directory,file_name,hist_name)

#structure of full_data [(np.array([[m0],[m12]]),np.array([[chi^2]])),(np.array([[m0],[m12]]),np.array([[chi^2]])),...]
#structure of training_data : as full_data
#structure of test_data     [(np.array([[m0],[m12]]),chi^2),(np.array([[m0],[m12]]),chi^2),...]

num_inputs=2
num_outputs=1

max_inputs=[0]*num_inputs
max_outputs=[0]*num_outputs


#first normalize full_data set! This will automatically normalize the training data set(see data_loader). This also normalizes the input of test_data, but not the output 
#of test data, because this is saved not as numpy array, but as plain number!
from sklearn.model_selection import train_test_split

image_size = 32
cropped_size = 28
num_channels = 1
pixel_depth = 255
num_labels = 5
num_digits = 10
depth = 32

patch_size_1 = 1
patch_size_3 = 3
patch_size_5 = 5
patch_size_7 = 7

train_data, train_labels, valid_data, valid_labels = data_loader.load_data()

print("Train data", train_data.shape)
print("Train labels", train_labels.shape)
print("Valid data", valid_data.shape)
print("Valid labels", valid_labels.shape)


def accuracy(labels, predictions):
    return (100.0 * np.sum(np.argmax(predictions, 1) == labels) /
            predictions.shape[0])


def TrainConvNet():
    def LecunLCN(X, image_shape, threshold=1e-4, radius=7, use_divisor=True):
        """Local Contrast Normalization"""
Exemplo n.º 25
0
def main():
	# number of particles
	Nsu = 300
	global nameBit

	#names = ['sims_11_medium']
	names = ['sims_01_medium','sims_10_medium','sims_11_medium']# test case
	#names = ['sims_01_slow','sims_01_medium','sims_01_fast','sims_10_slow','sims_10_medium','sims_10_fast','sims_11_slow','sims_11_medium','sims_11_fast']
	for namecounter in range(len(names)):
		nameNow = names[namecounter]
		(tsim,XK,YK,mu0,P0,Ns,dt,tf) = data_loader.load_data(nameNow,'../sim_data/')

		Ns = 100

		nameBit = int(nameNow[5:7],2)
		# parse the name
		if nameBit == 1:
			# tuned noise levels for the SIR with white noise forcing
			Qk = np.array([[1.0]])
			if dt < 0.09: # fast sampling
				Qk = np.array([[100.0]])
			if dt > 0.11: # slow sampling
				Qk = np.array([[0.1]])
			Rk = np.array([[1.0]])
		if nameBit == 2:
			# tuned noise levels for the SIR with cosine forcing
			Qk = np.array([[3.16]])
			if dt < 0.09: # fast sampling
				Qk = np.array([[316.0]])#tuned-ish
			if dt > 0.11: # slow sampling
				Qk = np.array([[0.5]])
			Rk = np.array([[1.0]])
		if nameBit == 3:
			# tuned noise levels for the SIR with white noise forcing
			Qk = np.array([[3.16]])
			if dt < 0.09: # fast sampling
				Qk = np.array([[31.6]])
			if dt > 0.11: # slow sampling
				Qk = np.array([[1.0]])
			Rk = np.array([[1.0]])
		print(Qk[0,0])
		# number of steps in each simulation
		nSteps = len(tsim)
		nees_history = np.zeros((nSteps,Ns))
		e_sims = np.zeros((Ns*nSteps,2))
		for counter in range(Ns):
			xk = XK[:,(2*counter):(2*counter+2)]
			yk = YK[:,counter]

			(xf,Pf,px1,px2,weights) = sir_test(dt,tf,mu0,P0,yk,Qk,Rk,Nsu)
			# TODO compute the unit variance transformation of the error
			e1 = np.zeros((nSteps,2))
			chi2 = np.zeros(nSteps)
			for k in range(nSteps):
				P = Pf[k,:].reshape((2,2))
				Pinv = np.linalg.inv(P)
				chi2[k] = np.dot(xk[k,:]-xf[k,:],np.dot(Pinv,xk[k,:]-xf[k,:]))
			# chi2 is the NEES statistic. Take the mean
			nees_history[:,counter] = chi2.copy()
			mean_nees = np.sum(chi2)/float(nSteps)
			print(mean_nees)
			# mean NEES
			mse = np.sum(np.power(xk-xf,2.0),axis=0)/float(nSteps)
			e_sims[(counter*nSteps):(counter*nSteps+nSteps),:] = xk-xf

			print("MSE: %f,%f" % (mse[0],mse[1]))
		if Ns < 2:
			# plot of the discrete PDF and maximum likelihood estimate
			# len(tsim) x Ns matrix of times
			tMesh = np.kron(np.ones((Nsu,1)),tsim).transpose()
			fig = plt.figure()
			ax = []
			for k in range(6):
				if k < 2:
					nam = 'pdf' + str(k+1)
				elif k < 4:
					nam = 'xf' + str(k-1)
				else:
					nam = 'ef' + str(k-3)
				ax.append( fig.add_subplot(3,2,k+1,ylabel=nam) )
				if k < 2:
					if k == 0:
						# plot the discrete PDF as a function of time
						mex = tMesh.reshape((len(tsim)*Nsu,))
						mey = px1.reshape((len(tsim)*Nsu,))
						mez = weights.reshape((len(tsim)*Nsu,))
					elif k == 1:
						# plot the discrete PDF as a function of time
						mex = tMesh.reshape((len(tsim)*Nsu,))
						mey = px2.reshape((len(tsim)*Nsu,))
						mez = weights.reshape((len(tsim)*Nsu,))
					idx = mez.argsort()
					mexx,meyy,mezz = mex[idx],mey[idx],mez[idx]

					cc = ax[k].scatter(mexx,meyy,c=mezz,s=20,edgecolor='')
					fig.colorbar(cc,ax=ax[k])
					# plot the truth
					ax[k].plot(tsim,xk[:,k],'b-')
				elif k < 4:
					ax[k].plot(tsim,xf[:,k-2],'m--')
					ax[k].plot(tsim,xk[:,k-2],'b-')
				else:
					ax[k].plot(tsim,xf[:,k-4]-xk[:,k-4],'b-')
					ax[k].plot(tsim,3.0*np.sqrt(Pf[:,k-4 + 2*(k-4)]),'r--')
					ax[k].plot(tsim,-3.0*np.sqrt(Pf[:,k-4 + 2*(k-4)]),'r--')
				ax[k].grid()

			fig.show()

		mse_tot = np.mean(np.power(e_sims,2.0),axis=0)
		print("mse_tot: %f,%f" % (mse_tot[0],mse_tot[1]))
		
		# get the mean NEES value versus simulation time across all sims
		nees_mean = np.sum(nees_history,axis=1)/Ns
		# get 95% confidence bounds for chi-sqaured... the df is the number of sims times the dimension of the state
		chiUpper = stats.chi2.ppf(.975,2.0*Ns)/float(Ns)
		chiLower = stats.chi2.ppf(.025,2.0*Ns)/float(Ns)

		# plot the mean NEES with the 95% confidence bounds
		fig2 = plt.figure(figsize=(6.0,3.37)) #figsize tuple is width, height
		tilt = "SIR, Ts = %.2f, %d sims, %d particles, " % (dt, Ns, Nsu)
		if nameBit == 0:
			tilt = tilt + 'unforced'
		if nameBit == 1:
			#white-noise only
			tilt = tilt + 'white-noise forcing'
		if nameBit == 2:
			tilt = tilt + 'cosine forcing'
		if nameBit == 3:
			#white-noise and cosine forcing
			tilt = tilt + 'white-noise and cosine forcing'
		ax = fig2.add_subplot(111,ylabel='mean NEES')#,title=tilt)
		ax.set_title(tilt,fontsize = 12)
		ax.plot(tsim,chiUpper*np.ones(nSteps),'r--')
		ax.plot(tsim,chiLower*np.ones(nSteps),'r--')
		ax.plot(tsim,nees_mean,'b-')
		ax.grid()
		fig2.show()
		# save the figure
		fig2.savefig('nees_sir_' + str(Nsu) + "_" + nameNow + '.png')
		# find fraction of inliers
		l1 = (nees_mean < chiUpper).nonzero()[0]
		l2 = (nees_mean > chiLower).nonzero()[0]
		# get number of inliers
		len_in = len(set(l1).intersection(l2))
		# get number of super (above) liers (sic)
		len_super = len((nees_mean > chiUpper).nonzero()[0])
		# get number of sub-liers (below)
		len_sub = len((nees_mean < chiLower).nonzero()[0])

		print("Conservative (below 95%% bounds): %f" % (float(len_sub)/float(nSteps)))
		print("Optimistic (above 95%% bounds): %f" % (float(len_super)/float(nSteps)))

		# save metrics
		FID = open('metrics_sir_' + str(Nsu) + "_" + nameNow + '.txt','w')
		FID.write("mse1,mse2,nees_below95,nees_above95\n")
		FID.write("%f,%f,%f,%f\n" % (mse_tot[0],mse_tot[1],float(len_sub)/float(nSteps),float(len_super)/float(nSteps)))
		FID.close()

	raw_input("Return to quit")

	print("Leaving sir_trials")

	return
Exemplo n.º 26
0
def train(
        epochs=HYPERPARAMS.epochs,
        random_state=HYPERPARAMS.
    random_state,  ##Get all necessary hyperparameters, and train the model (if you load in an already trained model, train_model can be turned to False.)
        kernel=HYPERPARAMS.kernel,
        decision_function=HYPERPARAMS.decision_function,
        gamma=HYPERPARAMS.gamma,
        train_model=True):

    print("loading dataset " + DATASET.name + "...")
    if train_model:  #Train model
        data, validation = load_data(validation=True)
    else:  #Or simply run the test data
        data, validation, test = load_data(validation=True, test=True)

    if train_model:
        # Training phase
        print("building model...")
        model = SVC(
            random_state=random_state,
            max_iter=epochs,
            kernel=kernel,
            decision_function_shape=decision_function,
            gamma=gamma
        )  ##Uses the scikit learning library to create a SVM with the parameters as set
        ##in 'parameters.py'
        ##All these print statements just give the user a lot of info
        print("start training...")
        print("--")
        print("kernel: {}".format(kernel))
        print("decision function: {} ".format(decision_function))
        print("max epochs: {} ".format(epochs))
        print("gamma: {} ".format(gamma))
        print("--")
        print("Training samples: {}".format(len(data['Y'])))
        print("Validation samples: {}".format(len(validation['Y'])))
        print("--")
        ##Training time is already being measured!!
        start_time = time.time()
        model.fit(data['X'], data['Y'])
        training_time = time.time() - start_time
        print("training time = {0:.1f} sec".format(training_time))

        if TRAINING.save_model:  ##After training has been completed, one can decide to have the mdel saved.
            print("saving model...")
            with open(TRAINING.save_model_path, 'wb') as f:
                cPickle.dump(model, f)

        print("evaluating...")
        validation_accuracy = evaluate(
            model, validation['X'],
            validation['Y'])  ##Calculation the accuracy
        print("  - validation accuracy = {0:.1f}".format(validation_accuracy *
                                                         100))
        return validation_accuracy
    else:
        # Testing phase : load saved model and evaluate on test dataset
        print("start evaluation...")
        print("loading pretrained model...")
        if os.path.isfile(TRAINING.save_model_path):
            with open(TRAINING.save_model_path, 'rb') as f:
                model = cPickle.load(f)
        else:
            print("Error: file '{}' not found".format(
                TRAINING.save_model_path))
            exit()

        print("--")
        print("Validation samples: {}".format(len(validation['Y'])))
        print("Test samples: {}".format(len(test['Y'])))
        print("--")
        print("evaluating...")
        ##Testing period is also timed :)
        start_time = time.time()
        validation_accuracy = evaluate(model, validation['X'], validation['Y'])
        print("  - validation accuracy = {0:.1f}".format(validation_accuracy *
                                                         100))
        test_accuracy = evaluate(model, test['X'], test['Y'])
        print("  - test accuracy = {0:.1f}".format(test_accuracy * 100))
        print("  - evalution time = {0:.1f} sec".format(time.time() -
                                                        start_time))
        return test_accuracy
Exemplo n.º 27
0
def train(cf):

    ###############
    #  load data  #
    ###############

    print('-' * 75)
    print('Loading data')
    #TODO ; prepare a public version of the data loader
    train_iter, val_iter, test_iter = load_data(cf.dataset,
                                                train_crop_size=cf.train_crop_size,
                                                batch_size=cf.batch_size,
                                                horizontal_flip=True,
                                                )

    n_classes = train_iter.get_n_classes()
    void_labels = train_iter.get_void_labels()

    print('Number of images : train : {}, val : {}, test : {}'.format(
        train_iter.get_n_samples(), val_iter.get_n_samples(), test_iter.get_n_samples()))

    ###################
    #   Build model   #
    ###################

    # Build model and display summary    
    net = cf.net
    net.summary()

    # Restore
    if hasattr(cf, 'pretrained_model'):
        print('Using a pretrained model : {}'.format(cf.pretrained_model))
        net.restore(cf.pretrained_model)

    # Compile functions
    print('Compilation starts at ' + str(datetime.now()).split('.')[0])
    params = lasagne.layers.get_all_params(net.output_layer, trainable=True)
    lr_shared = theano.shared(np.array(cf.learning_rate, dtype='float32'))
    lr_decay = np.array(cf.lr_sched_decay, dtype='float32')

    # Create loss and metrics
    for key in ['train', 'valid']:

        # LOSS
        pred = get_output(net.output_layer, deterministic=key == 'valid',
                          batch_norm_update_averages=False, batch_norm_use_averages=False)
        loss = crossentropy(pred, net.target_var, void_labels)

        if cf.weight_decay:
            weightsl2 = regularize_network_params(net.output_layer, lasagne.regularization.l2)
            loss += cf.weight_decay * weightsl2

        # METRICS
        I, U, acc = theano_metrics(pred, net.target_var, n_classes, void_labels)

        # COMPILE
        start_time_compilation = time.time()
        if key == 'train':
            updates = cf.optimizer(loss, params, learning_rate=lr_shared)
            train_fn = theano.function([net.input_var, net.target_var], [loss, I, U, acc], updates=updates)
        else:
            val_fn = theano.function([net.input_var, net.target_var], [loss, I, U, acc])

        print('{} compilation took {:.3f} seconds'.format(key, time.time() - start_time_compilation))

    ###################
    #    Main loops   #
    ###################

    # metric's sauce
    init_history = lambda: {'loss': [], 'jaccard': [], 'accuracy': []}
    history = {'train': init_history(), 'val': init_history(), 'test': init_history()}
    patience = 0
    best_jacc_val = 0
    best_epoch = 0

    if hasattr(cf, 'pretrained_model'):
        print('Validation score before training')
        print batch_loop(val_iter, val_fn, 0, 'val', {'val': init_history()})

    # Training main loop
    print('-' * 30)
    print('Training starts at ' + str(datetime.now()).split('.')[0])
    print('-' * 30)

    for epoch in range(cf.num_epochs):

        # Train
        start_time_train = time.time()
        history = batch_loop(train_iter, train_fn, epoch, 'train', history)
        # Validation
        start_time_valid = time.time()
        history = batch_loop(val_iter, val_fn, epoch, 'val', history)

        # Print
        out_str = \
            '\r\x1b[2 Epoch {} took {}+{} sec. ' \
            'loss = {:.5f} | jacc = {:.5f} | acc = {:.5f} || ' \
            'loss = {:.5f} | jacc = {:.5f} | acc = {:.5f}'.format(
                epoch, int(start_time_valid - start_time_train), int(time.time() - start_time_valid),
                history['train']['loss'][-1], history['train']['jaccard'][-1], history['train']['accuracy'][-1],
                history['val']['loss'][-1], history['val']['jaccard'][-1], history['val']['accuracy'][-1])

        # Monitoring jaccard
        if history['val']['jaccard'][-1] > best_jacc_val:
            out_str += ' (BEST)'
            best_jacc_val = history['val']['jaccard'][-1]
            best_epoch = epoch
            patience = 0
            net.save(os.path.join(cf.savepath, 'model.npz'))
        else:
            patience += 1

        print out_str

        np.savez(os.path.join(cf.savepath, 'errors.npz'), metrics=history, best_epoch=best_epoch)

        # Learning rate scheduler
        lr_shared.set_value(lr_shared.get_value() * lr_decay)

        # Finish training if patience has expired or max nber of epochs reached
        if patience == cf.max_patience or epoch == cf.num_epochs - 1:
            # Load best model weights
            net.restore(os.path.join(cf.savepath, 'model.npz'))

            # Test
            print('Training ends\nTest')
            if test_iter.get_n_samples() == 0:
                print 'No test set'
            else:
                history = batch_loop(test_iter, val_fn, epoch, 'test', history)

                print ('Average cost test = {:.5f} | jacc test = {:.5f} | acc_test = {:.5f} '.format(
                    history['test']['loss'][-1],
                    history['test']['jaccard'][-1],
                    history['test']['accuracy'][-1]))

                np.savez(os.path.join(cf.savepath, 'errors.npz'), metrics=history, best_epoch=best_epoch)

            # Exit
            return
Exemplo n.º 28
0
                    default=32,
                    help='dimension of user and entity embeddings')
parser.add_argument(
    '--n_iter',
    type=int,
    default=2,
    help='number of iterations when computing entity representation')
parser.add_argument('--batch_size', type=int, default=65536, help='batch size')
parser.add_argument('--l2_weight',
                    type=float,
                    default=1e-7,
                    help='weight of l2 regularization')
parser.add_argument('--lr', type=float, default=2e-2, help='learning rate')
parser.add_argument('--ratio',
                    type=float,
                    default=1,
                    help='size of training dataset')

show_loss = False
show_time = True
show_topk = False

t = time()

args = parser.parse_args()
data = load_data(args)
train(args, data, show_loss, show_topk)

if show_time:
    print('time used: %d s' % (time() - t))
Exemplo n.º 29
0
from utils import plot_confusion_matrix
from utils import timeSince

import time

dim_word2vec = 200

batch_size = 16
epochs = 5
learning_rate = 1e-4
model_fn = '../results/rnn_tc/rnn_tc_LSTM.11180355.pt'

glov_wv = word2vec.load_wv_from_model('../data/word2vec/retrained_word2vec/reddit_word2vec_200d')

data_train, label_train = load_data('../data/train.csv')
data_test, label_test = load_data('../data/test.csv')

#check number of labels in each class
tmp = label_test.tolist()
for label in np.unique(label_test):
    print('{}: {}'.format(label, tmp.count(label)))

tmp2 = label_train.tolist()
for label in np.unique(label_train):
    print('{}: {}'.format(label, tmp2.count(label)))

def avg_length(sentences):
    N = len(sentences)
    all_length = 0
    for sentence in sentences:
    predict_titles = []
    for i in tqdm(range(1, min(1001, eval_len)),
                  desc='collecting test inputs'):
        random_num = random.randint(1, eval_len)
        actual_titles.append(data.iloc[random_num]['target_text'])
        actual_abstracts.append("summarize: " +
                                data.iloc[random_num]['input_text'])

    predict_titles = model.predict(actual_abstracts)

    predict_df = pd.DataFrame({
        'predict_title': predict_titles,
        'actual_title': actual_titles,
        'actual_abstract': actual_abstracts
    })

    predict_df.to_json('./data/predict_result.json',
                       orient="split",
                       index=False)
    transformers_logger.info(
        'predicted file saved to ./data/predict_result.json')
    return predict_df


if __name__ == '__main__':
    train_df, eval_df, test_df = load_data()
    transformers_logger.info(train_df.shape, eval_df.shape, test_df.shape)
    model = get_model()
    predict_df = predict(model, test_df)
    transformers_logger.info(predict_df)
        action="store_true",
        help="Plot results for PCA initialization",
    )
    parser.add_argument(
        "dirs",
        type=str,
        nargs="+",
        metavar="DIR",
        help="The directory containing the simulation output files.",
    )

    cli_args = parser.parse_args()
    plot_flag = cli_args.show
    pickle_flag = cli_args.pickle

    df, final_val_tbl, conv_tbl, rt60, parameters = load_data(
        cli_args.dirs, pickle=pickle_flag)

    # Draw the figure
    print("Plotting...")

    df_melt = df.melt(id_vars=df.columns[:-5], var_name="metric")
    # df_melt = df_melt.replace(substitutions)

    # Aggregate the convergence curves
    df_agg = (df_melt.groupby(by=[
        "Algorithm",
        "Sources",
        "Interferers",
        "SINR",
        "Mics",
        "Iteration",
Exemplo n.º 32
0
import data_loader
import network
import numpy as np
import pylab as pl


"""If we want to train our neural network"""
## training data is like a list where each element of the list is a data set and its answer in matrix form
training_data, validation_data, test_data = data_loader.load_data()
for i in range(0,len(validation_data)):
	validation_data[i][1] = np.argmax(validation_data[i][1])
for i in range(0,len(test_data)):
	test_data[i][1] = np.argmax(test_data[i][1])

O = 3
I = 3
# S = len(training_data)
# H = int((S/3-O)/(I+O+1))
H = 10

net = network.Network([I, H ,O] ,cost=network.CrossEntropyCost, plot = True)
net.SGD(training_data,30,10,0.1,lmbda=5.0,evaluation_data = validation_data ,monitor_evaluation_accuracy=True,monitor_training_accuracy=True)

"""If we want to test on any dataset"""

# f = open("array.txt")
# data = f.read()
# f.close()
# data = data.split("\n")
# a = []
# for d in data:
Exemplo n.º 33
0
def select_cate(l,c):
    img = []
    for i in xrange(len(l[0])):
        if l[1][i] == c:
            img.append(l[0][i])
    return img

from matplotlib import pyplot as plt
def show_img(num, data, c):
    l = select_cate(data,c)
    f = plt.figure()
    for i in range(num):
        f.add_subplot(num/5,5,i)
        plt.imshow(l[i], interpolation='nearest', cmap='Greys_r')

    plt.show()

import data_loader
train = data_loader.load_data()
show_img(5,train,4)
Exemplo n.º 34
0
    elif args.dataset == 'officehome' or 'office_home' or 'office-home':
        return 'data/OfficeHome/'
    elif args.dataset == 'visda':
        return 'data/VisDA17/'
    elif args.dataset == 'imageclef' or 'image-clef':
        return 'data/image_CLEF'


if __name__ == '__main__':
    torch.manual_seed(10)

    # Load data
    root_dir = dataset()
    domain = {'src': str(args.source), 'tar': str(args.target)}
    dataloaders = {}
    dataloaders['tar'] = data_loader.load_data(root_dir, domain['tar'],
                                               BATCH_SIZE['tar'], 'tar')
    dataloaders['src'], dataloaders['val'] = data_loader.load_train(
        root_dir, domain['src'], BATCH_SIZE['src'], 'src')
    print(len(dataloaders['src'].dataset), len(dataloaders['val'].dataset))

    ## Finetune
    model = load_model(model_name).to(DEVICE)
    print('Source:{}, target:{}, model: {}'.format(domain['src'],
                                                   domain['tar'], model_name))
    optimizer = get_optimizer(model_name)
    model_best, best_acc, acc_hist = finetune(model, dataloaders, optimizer)

    ## Extract features for the target domain
    model_path = 'save_model/best_{}_{}.pth'.format(args.model, args.source)
    extract_feature(args.model, model_path, dataloaders['tar'], args.source,
                    args.target)
Exemplo n.º 35
0
tf.flags.DEFINE_boolean("log_device_placement", False, "Log placement of ops on devices")

FLAGS = tf.flags.FLAGS
FLAGS.batch_size
print("\nParameters:")
for attr, value in sorted(FLAGS.__flags.iteritems()):
    print("{}={}".format(attr.upper(), value))
print("")


# Data Preparatopn
# ==================================================

# Load data
print("Loading data...")
raw_train, raw_test, x, y, x_test, vocabulary = data_loader.load_data()
print "x_test is:", type(x_test), len(x_test), len(raw_test)

#test if my predict function is right
testfile = pd.DataFrame()
testfile = pd.read_csv("./data/answer.csv")
y_test = testfile.loc[:,'labels'].values
y_test = data_loader.convert_y(y_test)
y_test = np.array(y_test)
sampled_x_test = x_test[0:5000]
sampled_y_test = y_test[0:5000]

max_acc = 0
max_checkpoint = ""
# Randomly shuffle data
np.random.seed(10)
Exemplo n.º 36
0
def train(epochs=HYPERPARAMS.epochs, random_state=HYPERPARAMS.random_state, 
          kernel=HYPERPARAMS.kernel, decision_function=HYPERPARAMS.decision_function, gamma=HYPERPARAMS.gamma, train_model=True):

        print "loading dataset " + DATASET.name + "..."
        if train_model:
                data, validation = load_data(validation=True)
        else:
                data, validation, test = load_data(validation=True, test=True)
        
        if train_model:
            # Training phase
            print "building model..."
            model = SVC(random_state=random_state, max_iter=epochs, kernel=kernel, decision_function_shape=decision_function, gamma=gamma)

            print "start training..."
            print "--"
            print "kernel: {}".format(kernel)
            print "decision function: {} ".format(decision_function)
            print "max epochs: {} ".format(epochs)
            print "gamma: {} ".format(gamma)
            print "--"
            print "Training samples: {}".format(len(data['Y']))
            print "Validation samples: {}".format(len(validation['Y']))
            print "--"
            start_time = time.time()
            model.fit(data['X'], data['Y'])
            training_time = time.time() - start_time
            print "training time = {0:.1f} sec".format(training_time)

            if TRAINING.save_model:
                print "saving model..."
                with open(TRAINING.save_model_path, 'wb') as f:
                        cPickle.dump(model, f)

            print "evaluating..."
            validation_accuracy = evaluate(model, validation['X'], validation['Y'])
            print "  - validation accuracy = {0:.1f}".format(validation_accuracy*100)
            return validation_accuracy
        else:
            # Testing phase : load saved model and evaluate on test dataset
            print "start evaluation..."
            print "loading pretrained model..."
            if os.path.isfile(TRAINING.save_model_path):
                with open(TRAINING.save_model_path, 'rb') as f:
                        model = cPickle.load(f)
            else:
                print "Error: file '{}' not found".format(TRAINING.save_model_path)
                exit()

            print "--"
            print "Validation samples: {}".format(len(validation['Y']))
            print "Test samples: {}".format(len(test['Y']))
            print "--"
            print "evaluating..."
            start_time = time.time()
            validation_accuracy = evaluate(model, validation['X'],  validation['Y'])
            print "  - validation accuracy = {0:.1f}".format(validation_accuracy*100)
            test_accuracy = evaluate(model, test['X'], test['Y'])
            print "  - test accuracy = {0:.1f}".format(test_accuracy*100)
            print "  - evalution time = {0:.1f} sec".format(time.time() - start_time)
            return test_accuracy
Exemplo n.º 37
0
import open3d
import models
import data_loader
import torch.optim as optim
import torch
import numpy as np
from torch.utils.tensorboard import SummaryWriter

model = torch.load("./models/model_60000.ckpt").cuda()
dataset = data_loader.load_data("shapenet")
my_dataset_loader = torch.utils.data.DataLoader(dataset=dataset,
                                                batch_size=1,
                                                shuffle=False)
for input_tensor, gt_tensor in my_dataset_loader:
    input_tensor = input_tensor.cuda()
    gt_tensor = gt_tensor.cuda()
    coarse, fine = model(input_tensor)
    print(coarse.shape, fine.shape, gt_tensor.shape)
    pcd = open3d.PointCloud()
    pcd.points = open3d.Vector3dVector(fine.data.cpu().numpy()[0] +
                                       np.array([1.0, 0.0, 0.0]))
    pcd.colors = open3d.Vector3dVector(
        np.ones((fine.shape[1], 3)) * [0.76, 0.23, 0.14])

    pcd2 = open3d.PointCloud()
    pcd2.points = open3d.Vector3dVector(gt_tensor.data.cpu().numpy()[0] +
                                        np.array([-1.0, 0.0, 0.0]))
    pcd2.colors = open3d.Vector3dVector(
        np.ones((gt_tensor.shape[1], 3)) * [0.16, 0.23, 0.14])

    pcd3 = open3d.PointCloud()
Exemplo n.º 38
0
from matplotlib import cm
import thesis_redux_tools as rt

name_pho = np.loadtxt("../../inputs/SFHs_set3/set3_list.log",
                      usecols=(0, ),
                      dtype="|S")
name_sed = np.loadtxt("../../outputs/spectroscopic_fit/names.log", dtype="|S")
order_sed = [
    j for i in xrange(120) for j in xrange(12000)
    if name_pho[i] == name_sed[j][:12] + ".fits.gz"
]

osed = np.loadtxt(
    "../../outputs/spectroscopic_fit/table_din.v3.log")[order_sed]

sdss = dl.load_data("SDSS")
jpas = dl.load_data("JPAS")

nams = [
    "M_lib", "M_mod", "log_t_M_lib", "log_t_M_mod", "log_t_L_lib",
    "log_t_L_mod", "Z_M_lib", "Z_M_mod", "Av_lib", "Av_mod"
]

sdss.physical[nams[6]] = np.log10(sdss.physical[nams[6]])
res_sed = [
    rt.err(sdss.physical[n], osed[:, i]) if i == 0 else rt.err(
        sdss.physical[n], osed[:, i], False) for i, n in enumerate(nams[::2])
]
sdss.physical[nams[6]] = 10**sdss.physical[nams[6]]

fig, axs = plt.subplots(2, 3, sharex=True, figsize=(10, 6))
Exemplo n.º 39
0
def evaluate_lenet5(learning_rate=0.1,
                    n_epochs=200,
                    dataset='mnist.pkl.gz',
                    nkerns=[20, 50],
                    batch_size=500):
    """ Demonstrates lenet on MNIST dataset

    :type learning_rate: float
    :param learning_rate: learning rate used (factor for the stochastic
                          gradient)

    :type n_epochs: int
    :param n_epochs: maximal number of epochs to run the optimizer

    :type dataset: string
    :param dataset: path to the dataset used for training /testing (MNIST here)

    :type nkerns: list of ints
    :param nkerns: number of kernels (feature maps) on each layer
    """

    rng = numpy.random.RandomState(23455)

    datasets = load_data(dataset)

    train_set_x, train_set_y = datasets[0]
    valid_set_x, valid_set_y = datasets[1]
    test_set_x, test_set_y = datasets[2]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0]
    n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
    n_test_batches = test_set_x.get_value(borrow=True).shape[0]
    n_train_batches //= batch_size
    n_valid_batches //= batch_size
    n_test_batches //= batch_size

    # allocate symbolic variables for the data
    index = T.lscalar()  # index to a [mini]batch

    x = T.matrix('x')  #images
    y = T.ivector('y')  #labels

    ######################
    # BUILD ACTUAL MODEL #
    ######################
    print('... building the model')

    # Reshape matrix of rasterized images of shape (batch_size, 28 * 28)
    # to a 4D tensor, compatible with our LeNetConvPoolLayer
    # (28, 28) is the size of MNIST images.
    # Only b/w image, so input_feature_map size = 1

    layer0_input = x.reshape((batch_size, 1, 28, 28))

    layer0 = convlayer.LeNetConvPoolLayer(rng,
                                          input=layer0_input,
                                          image_shape=(batch_size, 1, 28, 28),
                                          filter_shape=(nkerns[0], 1, 5, 5),
                                          poolsize=(2, 2))

    # Construct the second convolutional pooling layer
    # filtering reduces the image size to (12-5+1, 12-5+1) = (8, 8)
    # maxpooling reduces this further to (8/2, 8/2) = (4, 4)
    # 4D output tensor is thus of shape (batch_size, nkerns[1], 4, 4)
    layer1 = convlayer.LeNetConvPoolLayer(rng,
                                          input=layer0.output,
                                          image_shape=(batch_size, nkerns[0],
                                                       12, 12),
                                          filter_shape=(nkerns[1], nkerns[0],
                                                        5, 5),
                                          poolsize=(2, 2))

    # the HiddenLayer being fully-connected, it operates on 2D matrices of
    # shape (batch_size, num_pixels) (i.e matrix of rasterized images).
    # This will generate a matrix of shape (batch_size, nkerns[1] * 4 * 4),
    # or (500, 50 * 4 * 4) = (500, 800) with the default values.
    layer2_input = layer1.output.flatten(2)

    layer2 = HiddenLayer(rng,
                         input=layer2_input,
                         n_in=nkerns[1] * 4 * 4,
                         n_out=500,
                         activation=T.tanh)

    layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=10)

    cost = layer3.negative_log_likelihood(y)

    test_model = theano.function(
        [index],
        layer3.errors(y),
        givens={
            x: test_set_x[index * batch_size:(index + 1) * batch_size],
            y: test_set_y[index * batch_size:(index + 1) * batch_size]
        })

    validate_model = theano.function(
        [index],
        layer3.errors(y),
        givens={
            x: valid_set_x[index * batch_size:(index + 1) * batch_size],
            y: valid_set_y[index * batch_size:(index + 1) * batch_size]
        })

    params = layer3.params + layer2.params + layer1.params + layer0.params

    # create a list of gradients for all model parameters
    grads = T.grad(cost, params)

    # train_model is a function that updates the model parameters by
    # SGD Since this model has many parameters, it would be tedious to
    # manually create an update rule for each model parameter. We thus
    # create the updates list by automatically looping over all
    # (params[i], grads[i]) pairs.

    updates = [(param_i, param_i - learning_rate * grad_i)
               for param_i, grad_i in zip(params, grads)]

    train_model = theano.function(
        [index],
        cost,
        updates=updates,
        givens={
            x: train_set_x[index * batch_size:(index + 1) * batch_size],
            y: train_set_y[index * batch_size:(index + 1) * batch_size]
        })

    ###############
    # TRAIN MODEL #
    ###############
    print('... training')
    # early-stopping parameters
    patience = 10000  # look as this many examples regardless
    patience_increase = 2  # wait this much longer when a new best is
    # found
    improvement_threshold = 0.995  # a relative improvement of this much is
    # considered significant
    validation_frequency = min(n_train_batches, patience // 2)
    # go through this many
    # minibatche before checking the network
    # on the validation set; in this case we
    # check every epoch

    best_validation_loss = numpy.inf
    best_iter = 0
    test_score = 0.
    start_time = timeit.default_timer()

    epoch = 0
    done_looping = False

    while (epoch < n_epochs) and (not done_looping):
        epoch += 1
        for minibatch_index in range(n_train_batches):

            iter = (epoch - 1) * n_train_batches + minibatch_index

            if iter % 100 == 0:
                print('training @ iter = ', iter)
            cost_ij = train_model(minibatch_index)

            if (iter + 1) % validation_frequency == 0:

                # compute zero-one loss on validation set
                validation_losses = [
                    validate_model(i) for i in range(n_valid_batches)
                ]
                this_validation_loss = numpy.mean(validation_losses)
                print('epoch %i, minibatch %i/%i, validation error %f %%' %
                      (epoch, minibatch_index + 1, n_train_batches,
                       this_validation_loss * 100.))

                # if we got the best validation score until now
                if this_validation_loss < best_validation_loss:

                    #improve patience if loss improvement is good enough
                    if this_validation_loss < best_validation_loss *  \
                       improvement_threshold:
                        patience = max(patience, iter * patience_increase)

                    # save best validation score and iteration number
                    best_validation_loss = this_validation_loss
                    best_iter = iter

                    # test it on the test set
                    test_losses = [
                        test_model(i) for i in range(n_test_batches)
                    ]
                    test_score = numpy.mean(test_losses)
                    print(('     epoch %i, minibatch %i/%i, test error of '
                           'best model %f %%') %
                          (epoch, minibatch_index + 1, n_train_batches,
                           test_score * 100.))

            if patience <= iter:
                done_looping = True
                break

    end_time = timeit.default_timer()
    print('Optimization complete.')
    print('Best validation score of %f %% obtained at iteration %i, '
          'with test performance %f %%' %
          (best_validation_loss * 100., best_iter + 1, test_score * 100.))
    print(
        ('The code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' %
         ((end_time - start_time) / 60.)),
        file=sys.stderr)
Exemplo n.º 40
0
from cs231n.classifiers.rbm import *

# model = RBM()
# solver = None

# #data = get_CIFAR10_data()
# data = load_data()
# for k, v in list(data.items()):
#   print(('%s: ' % k, v.shape))

# solver = Solver(model, data, update_rule='sgd', optim_config={'learning_rate': 1e-3}, print_every=10, lr_decay=0.9, num_epochs=10, batch_size=100)
# solver.train_unsupervise()

import pickle as pickle

data = load_data()
checkpoint = pickle.load(open('RBM_epoch_10.pkl', 'rb'))
rbm_params = checkpoint['model']

model = TwoLayerNet(input_dim=28 * 28)
solver = Solver(model,
                data,
                update_rule='sgd',
                optim_config={'learning_rate': 1e-3},
                print_every=100,
                lr_decay=0.9,
                num_epochs=10,
                batch_size=200)
solver.train()
solver.check_accuracy(data['X_test'], data['y_test'])
plot_solver(solver)
Exemplo n.º 41
0
def evaluate_lenet5(learning_rate=0.005, n_epochs=5,data = None,nkerns= 64, batch_size=30):


    #for i in range(len(x_val)):
        #if len(x_val[i]) == 490 and len(x_val[i][0]) == 640:
            #x1.append(x_val[i])
            #y1.append(y_val[i]-1)
            #if len(x1) == 80:
                #break

    from data_loader import load_data
    train, validate, test = load_data()
    x_train = np.array(train[0],'float32')
    y_train = train[1]
    x_valid = np.array(validate[0],'float32')
    y_valid = validate[1]
    x_test = np.array(test[0],'float32')
    y_test = test[1]
    x_train2 = theano.shared(numpy.asarray(x_train,dtype=theano.config.floatX))
    y_train_2 = theano.shared(numpy.asarray(y_train,dtype=theano.config.floatX))
    x_valid2 = theano.shared(numpy.asarray(x_valid,dtype=theano.config.floatX))
    y_valid_2 = theano.shared(numpy.asarray(y_valid,dtype=theano.config.floatX))
    x_test2 = theano.shared(numpy.asarray(x_test,dtype=theano.config.floatX))
    y_test_2 = theano.shared(numpy.asarray(y_test,dtype=theano.config.floatX))

    y_train2 = T.cast(y_train_2, 'int32')
    y_test2 = T.cast(y_test_2, 'int32')
    y_valid2 = T.cast(y_valid_2, 'int32')

    print len(x_train)
    print len(y_train)

    rng = numpy.random.RandomState(23455)

    n_train_batches = len(y_train)/batch_size
    n_valid_batches = len(y_valid)/batch_size
    n_test_batches = len(y_test)/batch_size
    index = T.lscalar()  # index to a [mini]batch

    x = T.matrix('x')   # the data is presented as rasterized images
    y = T.ivector('y')  # the labels are p

    layer0_input = x.reshape((batch_size, 1, 64, 64))

    '''构建第一层网络:
    image_shape:输入大小为490*640的特征图,batch_size个训练数据,每个训练数据有1个特征图
    filter_shape:卷积核个数为nkernes=64,因此本层每个训练样本即将生成64个特征图
    经过卷积操作,图片大小变为(490-7+1 , 640-7+1) = (484, 634)
    经过池化操作,图片大小变为 (484/2, 634/2) = (242, 317)
    最后生成的本层image_shape为(batch_size, nklearn, 242, 317)'''

    layer0 = LeNetConvPoolLayer(
        rng,
        input=layer0_input,
        image_shape=(batch_size, 1, 64, 64),
        filter_shape=(nkerns, 1, 7, 7),
        poolsize=(2, 2)
    )

    # the HiddenLayer being fully-connected, it operates on 2D matrices of
    # shape (batch_size, num_pixels) (i.e matrix of rasterized images).
    # This will generate a matrix of shape (batch_size, nkerns * 7 * 7),
    # (100, 64*7*7) with the default values.
    layer2_input = layer0.output.flatten(2)

    '''全链接:输入layer2_input是一个二维的矩阵,第一维表示样本,第二维表示上面经过卷积下采样后
    每个样本所得到的神经元,也就是每个样本的特征,HiddenLayer类是一个单层网络结构
    下面的layer2把神经元个数由800个压缩映射为500个'''
    layer2 = HiddenLayer(
        rng,
        input=layer2_input,
        n_in=nkerns * 29 * 29,
        n_out=500,
        activation=T.tanh
    )

    layer2.output = dropout_layer(layer2.output,0.5)

    # 最后一层:逻辑回归层分类判别,把500个神经元,压缩映射成10个神经元,分别对应于手写字体的0~9
    layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=8)

    # the cost we minimize during training is the NLL of the model
    cost = layer3.negative_log_likelihood(y)

    # create a function to compute the mistakes that are made by the model
    test_model = theano.function(
        [index],
        layer3.errors(y),
        givens={
            y: y_test2[index * batch_size: (index + 1) * batch_size],
            x: x_test2[index * batch_size: (index + 1) * batch_size]
        }
    )

    validate_model = theano.function(
        [index],
        layer3.errors(y),
        givens={
            x: x_valid2[index * batch_size: (index + 1) * batch_size],
            y: y_valid2[index * batch_size: (index + 1) * batch_size]
        }
    )

    #把所有的参数放在同一个列表里,可直接使用列表相加
    params = layer3.params + layer2.params  + layer0.params

    #梯度求导
    grads = T.grad(cost, params)

    updates = [
        (param_i, param_i - learning_rate * grad_i)
        for param_i, grad_i in zip(params, grads)
    ]

    train_model = theano.function(
        [index],
        cost,
        updates=updates,
        givens={
            x: x_train2[index * batch_size: (index + 1) * batch_size],
            y: y_train2[index * batch_size: (index + 1) * batch_size]
        }
    )

    print '... training'
    # early-stopping parameters
    patience = 10000  # look as this many examples regardless
    patience_increase = 2  # wait this much longer when a new best is
                           # found
    improvement_threshold = 0.2  # a relative improvement of this much is
                                   # considered significant
    validation_frequency = min(n_train_batches, patience / 2)
                                  # go through this many
                                  # minibatche before checking the network
                                  # on the validation set; in this case we
                                  # check every epoch

    best_validation_loss = numpy.inf
    best_iter = 0
    test_score = 0.
    start_time = timeit.default_timer()

    epoch = 0
    done_looping = False

    while (epoch < n_epochs) and (not done_looping):
    #while epoch < n_epochs:
        epoch = epoch + 1
        for minibatch_index in xrange(n_train_batches):#每一批训练数据

            cost_ij = train_model(minibatch_index)
            iter = (epoch - 1) * n_train_batches + minibatch_index
            if (iter + 1) % validation_frequency == 0:

                # compute zero-one loss on validation set
                validation_losses = [validate_model(i) for i
                                     in xrange(n_valid_batches)]
                this_validation_loss = numpy.mean(validation_losses)
                print('epoch %i, minibatch %i/%i, validation error %f %%' %
                      (epoch, minibatch_index + 1, n_train_batches,
                       this_validation_loss * 100.))

                # if we got the best validation score until now
                if this_validation_loss < best_validation_loss:

                    #improve patience if loss improvement is good enough
                    if this_validation_loss < best_validation_loss *  \
                       improvement_threshold:
                        patience = max(patience, iter * patience_increase)

                    # save best validation score and iteration number
                    best_validation_loss = this_validation_loss
                    best_iter = iter

                    # test it on the test set
                    test_losses = [
                        test_model(i)
                        for i in xrange(n_test_batches)
                    ]
                    test_score = numpy.mean(test_losses)
                    print(('     epoch %i, minibatch %i/%i, test error of '
                           'best model %f %%') %
                          (epoch, minibatch_index + 1, n_train_batches,
                           test_score * 100.))

            if patience <= iter:
                done_looping = True
                break

    with open('param0.pkl', 'wb') as f0:
                        pickle.dump(layer0.params, f0)
    f0.close()
    with open('param2.pkl', 'wb') as f2:
                        pickle.dump(layer2.params, f2)
    f2.close()
    with open('param3.pkl', 'wb') as f3:
                        pickle.dump(layer3.params, f3)
    f3.close()

    end_time = timeit.default_timer()
    print('Optimization complete.')
    print('Best validation score of %f %% obtained at iteration %i, '
          'with test performance %f %%' %
          (best_validation_loss * 100., best_iter + 1, test_score * 100.))
    print >> sys.stderr, ('The code for file ' +
                          os.path.split(__file__)[1] +
                          ' ran for %.2fm' % ((end_time - start_time) / 60.))
Exemplo n.º 42
0
    def train(self):
        """Training Function."""
        # Load Dataset from the dataset folder
        # 注意:这里的inputs可是有四张图片啊images_i,images_j,image_i,image_j
        #而且image_i,j是三维的,而images_i,j是四维的
        self.inputs = data_loader.load_data(self._dataset_name,
                                            self._size_before_crop, True,
                                            self._do_flipping)

        # Build the network
        self.model_setup()

        # Loss function calculations
        self.compute_losses()

        # Initializing the global variables
        init = (tf.global_variables_initializer(),
                tf.local_variables_initializer())
        #Saver类提供了向checkpoints文件保存和从checkpoints文件中恢复变量的相关方法。
        # Checkpoints文件是一个二进制文件,它把变量名映射到对应的tensor值
        saver = tf.train.Saver()

        max_images = cyclegan_datasets.DATASET_TO_SIZES[self._dataset_name]

        with tf.Session() as sess:
            sess.run(init)

            # Restore the model to run the model from last checkpoint
            if self._to_restore:
                #tf.train.latest_checkpoint来自动获取最后一次保存的模型
                chkpt_fname = tf.train.latest_checkpoint(self._checkpoint_dir)
                if chkpt_fname is not None:
                    #模型的恢复用的是restore()函数,它需要两个参数restore(sess, save_path),
                    # save_path指的是保存的模型路径。我们可以使用tf.train.latest_checkpoint()
                    # 来自动获取最后一次保存的模型
                    saver.restore(sess, chkpt_fname)
            #这个是tensorflowborder中的组件,主要用于记录变量的变化过程,
            writer = tf.summary.FileWriter(self._output_dir,
                                           tf.get_default_graph())

            if not os.path.exists(self._output_dir):
                os.makedirs(self._output_dir)

            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(coord=coord)

            # Training Loop
            for epoch in range(sess.run(self.global_step), self._max_step):
                print("In the epoch ", epoch)
                saver.save(sess,
                           os.path.join(self._output_dir, "cyclegan"),
                           global_step=epoch)

                # Dealing with the learning rate as per the epoch number
                if epoch < 100:
                    curr_lr = self._base_lr
                else:
                    curr_lr = self._base_lr - \
                        self._base_lr * (epoch - 100) / 100

                #将输入input_a,input_b,fake_a,fake_b,cycle_a,cycle_b根据第几轮,第几张图进行保存
                self.save_images(sess, epoch)

                #进行损失函数的优化过程,一次优化的次数和图像的数量有关,因为minibatch_size的大小是1
                for i in range(0, max_images):
                    print("Processing batch {}/{}".format(i, max_images))
                    #这里self.inputs开始起效,就是开始加载图片进来(其实在self.save_images(sess,epoch)中inputs就开始生效了)
                    inputs = sess.run(self.inputs)

                    # Optimizing the G_A network----这里有3个返回值,分别是什么????????
                    #原来返回的就是这三个值。
                    _, fake_B_temp, summary_str = sess.run(
                        [
                            self.g_A_trainer,
                            self.fake_images_b,  #????????????????
                            self.g_A_loss_summ
                        ],
                        feed_dict={
                            self.input_a: inputs['images_i'],
                            self.input_b: inputs['images_j'],
                            self.learning_rate: curr_lr
                        })
                    #加入到tensorflowborder中
                    writer.add_summary(summary_str, epoch * max_images + i)

                    #这里维持着一个容量为50的图像池,存放于self.fake_images_B,在容量没达到50之前,返回的是fake_B_temp,容量达到50后,随机返回池中的图像
                    # (实质就是容量满了以后,fake_B_temp返回的概率是0.5,其它50返回的概率是0.5)
                    fake_B_temp1 = self.fake_image_pool(
                        self.num_fake_inputs, fake_B_temp, self.fake_images_B)

                    # Optimizing the D_B network
                    _, summary_str = sess.run(
                        [self.d_B_trainer, self.d_B_loss_summ],
                        feed_dict={
                            self.input_a: inputs['images_i'],
                            self.input_b: inputs['images_j'],
                            self.learning_rate: curr_lr,
                            self.fake_pool_B: fake_B_temp1
                        })
                    writer.add_summary(summary_str, epoch * max_images + i)

                    # Optimizing the G_B network
                    _, fake_A_temp, summary_str = sess.run(
                        [
                            self.g_B_trainer, self.fake_images_a,
                            self.g_B_loss_summ
                        ],
                        feed_dict={
                            self.input_a: inputs['images_i'],
                            self.input_b: inputs['images_j'],
                            self.learning_rate: curr_lr
                        })
                    writer.add_summary(summary_str, epoch * max_images + i)

                    fake_A_temp1 = self.fake_image_pool(
                        self.num_fake_inputs, fake_A_temp, self.fake_images_A)

                    # Optimizing the D_A network
                    _, summary_str = sess.run(
                        [self.d_A_trainer, self.d_A_loss_summ],
                        feed_dict={
                            self.input_a: inputs['images_i'],
                            self.input_b: inputs['images_j'],
                            self.learning_rate: curr_lr,
                            self.fake_pool_A: fake_A_temp1
                        })
                    writer.add_summary(summary_str, epoch * max_images + i)

                    writer.flush()
                    self.num_fake_inputs += 1

                sess.run(tf.assign(self.global_step, epoch + 1))

            coord.request_stop()
            coord.join(threads)
            writer.add_graph(sess.graph)
Exemplo n.º 43
0
import numpy as np
import matplotlib.pyplot as plt
import scipy.stats.mstats as st
import data_loader as dl
from itertools import product
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
from matplotlib import cm
import thesis_redux_tools as rt

name_pho  = np.loadtxt("../../inputs/SFHs_set3/set3_list.log", usecols = (0,), dtype = "|S")
name_sed  = np.loadtxt("../../outputs/spectroscopic_fit/names.log", dtype = "|S")
order_sed = [j for i in xrange(120) for j in xrange(12000) if name_pho[i] == name_sed[j][:12] + ".fits.gz"]

osed = np.loadtxt("../../outputs/spectroscopic_fit/table_din.v3.log")[order_sed]

sdss = dl.load_data("SDSS")
jpas = dl.load_data("JPAS")

nams = ["M_lib", "M_mod", "log_t_M_lib", "log_t_M_mod", "log_t_L_lib", "log_t_L_mod", "Z_M_lib", "Z_M_mod", "Av_lib", "Av_mod"]

sdss.physical[nams[6]] = np.log10(sdss.physical[nams[6]])
res_sed = [rt.err(sdss.physical[n], osed[:, i]) if i == 0 else rt.err(sdss.physical[n], osed[:, i], False) for i, n in enumerate(nams[::2])]
sdss.physical[nams[6]] = 10 ** sdss.physical[nams[6]]

fig, axs = plt.subplots(2, 3, sharex = True, figsize = (10, 6))
axins    = inset_axes(axs[0, 0], width = "50%", height = "4%", loc = 2)

for i, j in product(xrange(axs.shape[0]), xrange(axs.shape[1])) :

  if j == 0 :
    if i == 0 :
Exemplo n.º 44
0
def main(args):
    # fix random seeds
    print('start training')
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed_all(args.seed)
    np.random.seed(args.seed)
    now = datetime.now()
    # load the data
    dataloader, dataset_train, dataloader_val, dataset_val, tsamples = load_data(
        args.path, args.bs, train_ratio=0.8, test_ratio=0.2)
    #load vgg
    model = Models.__dict__["vgg16"](args.sobel)  # pretrained weights?
    fd = int(model.top_layer.weight.size()[1])
    model.top_layer = None  # why? do we need it here?
    model.features = torch.nn.DataParallel(model.features)
    model.cuda()
    cudnn.benchmark = True

    # create optimizer
    optimizer = torch.optim.SGD(
        filter(lambda x: x.requires_grad, model.parameters()),
        lr=args.lr,
        momentum=args.momentum,
        weight_decay=10**args.wd,
    )

    # define loss function
    criterion = nn.CrossEntropyLoss().cuda()

    # optionally resume from a checkpoint
    if args.resume:
        if os.path.isfile(args.resume):
            print("=> loading checkpoint '{}'".format(args.resume))
            checkpoint = torch.load(args.resume)
            args.start_epoch = checkpoint['epoch']
            # remove top_layer parameters from checkpoint
            for key in checkpoint['state_dict']:
                if 'top_layer' in key:
                    del checkpoint['state_dict'][key]
            model.load_state_dict(checkpoint['state_dict'])
            optimizer.load_state_dict(checkpoint['optimizer'])
            print("=> loaded checkpoint '{}' (epoch {})".format(
                args.resume, checkpoint['epoch']))
        else:
            print("=> no checkpoint found at '{}'".format(args.resume))

    # creating checkpoint repo
    exp_check = os.path.join(args.exp, 'checkpoints')
    if not os.path.isdir(exp_check):
        os.makedirs(exp_check)

    # creating cluster assignments log
    cluster_log = Logger(os.path.join(args.exp, 'clusters'))

    losses = np.zeros(args.ep)  # loss per epoch, array of size ep x 1
    accuracies = np.zeros(args.ep)
    losses_val = np.zeros(args.ep)
    accuracies_val = np.zeros(args.ep)
    labels = [
        573, 671
    ]  # move to another location, maybe outside for-loop, outside training method

    # for all epochs
    for epoch in range(args.ep):
        # remove head
        model.top_layer = None
        model.classifier = nn.Sequential(
            *list(model.classifier.children())[:-1]
        )  # The actual classifier seems missing here, why are just the children added to a list?
        # get the features for the whole dataset

        features = compute_features(dataloader, model, len(dataset_train),
                                    args.bs, labels)
        features_val = compute_features(dataloader_val, model,
                                        len(dataset_val), args.bs, labels)
        print('PCA')
        pre_data = preprocessing(model, features)
        pre_data_val = preprocessing(model, features_val)
        # clustering algorithm to use
        deepcluster = clustering.__dict__[args.clustering](args.k)
        print('clustering')

        deepcluster_val = clustering.__dict__[args.clustering](args.k)

        clustering_loss = deepcluster.cluster(pre_data, verbose=args.verbose)
        clustering_loss_val = deepcluster_val.cluster(pre_data_val,
                                                      verbose=args.verbose)
        images_list = deepcluster.images_lists
        images_list_val = deepcluster_val.images_lists

        # pseudo labels
        print('train pseudolabels')
        train_dataset = clustering.cluster_assign(images_list, dataset_train)
        val_dataset = clustering.cluster_assign(images_list_val, dataset_val)
        len_d = len(train_dataset)
        len_val = len(val_dataset)
        # uniformly sample per target
        sampler = UnifLabelSampler(int(args.reassign * len_d), images_list)
        sampler2 = UnifLabelSampler(int(args.reassign * len_val),
                                    images_list_val)
        train_dataloader = torch.utils.data.DataLoader(
            train_dataset,
            batch_size=args.bs,
            sampler=sampler,
            pin_memory=True,
        )
        val_dataloader = torch.utils.data.DataLoader(
            val_dataset,
            batch_size=args.bs,
            sampler=sampler2,
            pin_memory=True,
        )
        # set last fully connected layer
        mlp = list(model.classifier.children())
        mlp.append(nn.ReLU(inplace=True).cuda())
        model.classifier = nn.Sequential(*mlp)
        model.top_layer = nn.Linear(fd, len(images_list))
        model.top_layer.weight.data.normal_(0, 0.01)
        model.top_layer.bias.data.zero_()
        model.top_layer.cuda()
        # train network with clusters as pseudo-labels
        # train network with clusters as pseudo-labels
        end = time.time()
        losses[epoch], accuracies[epoch] = train(train_dataloader, model,
                                                 criterion, optimizer, epoch,
                                                 args.lr, args.wd)
        print(f'epoch {epoch} ended with loss {losses[epoch]}')
        losses_val[epoch], accuracies_val[epoch] = validate(
            val_dataloader, model, criterion)
        plot_loss_acc(losses[0:epoch], losses[0:epoch], accuracies[0:epoch],
                      accuracies[0:epoch], now, epoch, args.k, tsamples,
                      args.ep)

        # print log
        if args.verbose:
            print('###### Epoch [{0}] ###### \n'
                  'Time: {1:.3f} s\n'
                  'Clustering loss: {2:.3f} \n'
                  'ConvNet loss: {3:.3f}'.format(epoch,
                                                 time.time() - end,
                                                 losses[epoch]))

        # save running checkpoint
        torch.save(
            {
                'epoch': epoch + 1,
                'arch': args.arch,
                'state_dict': model.state_dict(),
                'optimizer': optimizer.state_dict()
            },
            os.path.join(args.exp,
                         f'checkpoint_{now}_k{args.k}_ep{epoch}.pth.tar'))

        # save cluster assignments
        cluster_log.log(images_list)
Exemplo n.º 45
0
def main(train=False):
    if train:
        data = data_loader.load_data()
    else:
        data = []
    net = network.Network([16, 10, 4])
    #return
    alive = True
    b = board.Board(SIZE)
    """
    for text based game
    
    for row in b.get_array():
        print(row)
    """

    win = GraphWin("2048", 500, 500, autoflush=False)
    p = Point(50, 50)
    for row in b.get_array():
        for cell in row:
            r = Rectangle(p, Point(p.getX() + 100, p.getY() + 100))
            r.draw(win)
            if cell != 0:
                t = Text(Point(p.getX() + 50, p.getY() + 50), cell)
                t.draw(win)
            p.move(100, 0)
        p.move(-400, 100)

    # draw the score
    score = Text(Point(250, 25), "Score: {0}".format(0))
    score.draw(win)

    while alive:
        # pauses to get key press
        key = win.getKey()

        # keep a variable that holds the array before it is changed
        tmp = np.copy(b.get_array())

        # move depending on key press
        if key == "Up":
            b.move("up")
        elif key == "Left":
            b.move("left")
        elif key == "Down":
            b.move("down")
        elif key == "Right":
            b.move("right")
        elif key == "Escape":
            alive = False
            win.close()
        else:
            continue

        data.append([tmp, key])

        # checks whether board changed at all
        different = False
        for row in range(SIZE):
            for col in range(SIZE):
                if tmp[row][col] != b.get_array()[row][col]:
                    different = True
                    break
            if different:
                break

        # only update if the board changed
        if different:
            # new number generator
            rand_x = random.randint(0, SIZE - 1)
            rand_y = random.randint(0, SIZE - 1)
            while b.get_array()[rand_x][rand_y] != 0:
                rand_x = random.randint(0, SIZE - 1)
                rand_y = random.randint(0, SIZE - 1)
            rand_two = random.random()
            if rand_two < 0.95:
                b.get_array()[rand_x][rand_y] = 2
            else:
                b.get_array()[rand_x][rand_y] = 4
            """
            for text based game
            
            print("\n")
            # print array
            for row in b.get_array():
                print(row)
            """

            # erase all items on window
            for item in win.items[:]:
                item.undraw()

            score.setText("Score : {0}".format(b.score))
            score.draw(win)

            # restart drawing from upper left corner
            p = Point(50, 50)
            # draw a rectangle for each element in the array
            # draw a number for each non zero element in the array
            for row in b.get_array():
                for cell in row:
                    r = Rectangle(p, Point(p.getX() + 100, p.getY() + 100))
                    r.draw(win)
                    if cell != 0:
                        t = Text(Point(p.getX() + 50, p.getY() + 50), cell)
                        t.draw(win)
                    # move over by a rectangle width
                    p.move(100, 0)
                # reset back four rectangles and down 1 to draw next row
                p.move(-400, 100)
            # update at 30 frames per sec if animation is used
            update(30)

        # pause so it doesn't overrun, unnecessary for gui since it pauses for
        # each keystroke
        #time.sleep(.2)
        else:
            # check if you can make anymore moves
            directions = ["up", "left", "down", "right"]
            for dir in directions:
                tmp_b = copy.deepcopy(b)
                tmp_b.move(dir)
                different = False
                for row in range(SIZE):
                    for col in range(SIZE):
                        if tmp[row][col] != tmp_b.get_array()[row][col]:
                            different = True
                            break
                    if different:
                        break
                if different:
                    break
            if not different:
                alive = False
    if not train:
        file_num = 0
        while os.path.exists("datafile{0}".format(file_num)):
            file_num += 1
        data_loader.save_data(data, 'datafile{0}'.format(file_num))
Exemplo n.º 46
0
    def train(self):
        """Training Function."""
        # Load Dataset from the dataset folder
        self.inputs = data_loader.load_data(self._dataset_name,
                                            self._size_before_crop, False,
                                            self._do_flipping)

        # Build the network
        print('model setup started...')
        self.model_setup()
        print('model setup completed...')

        # Loss function calculations
        print('computing losses started...')
        self.compute_losses()
        print('computing losses completed...')

        # Initializing the global variables
        init = (tf.global_variables_initializer(),
                tf.local_variables_initializer())

        print('train saving started...')
        saver = tf.train.Saver(max_to_keep=None)
        print('train saving completed...')

        print('get max_images started...')
        max_images = cyclegan_datasets.DATASET_TO_SIZES[self._dataset_name]
        print('get max_images completed...')

        half_training = int(self._max_step / 2)

        print('Session.......')
        with tf.Session() as sess:
            sess.run(init)
            # Restore the model to run the model from last checkpoint
            if self._to_restore:
                chkpt_fname = tf.train.latest_checkpoint(self._checkpoint_dir)
                saver.restore(sess, chkpt_fname)

            writer = tf.summary.FileWriter(self._output_dir)

            if not os.path.exists(self._output_dir):
                os.makedirs(self._output_dir)

            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(coord=coord)

            # Training Loop
            for epoch in range(sess.run(self.global_step), self._max_step):
                print("In the epoch ", epoch)
                saver.save(sess,
                           os.path.join(self._output_dir, "AGGAN"),
                           global_step=epoch)

                # Dealing with the learning rate as per the epoch number
                if epoch < half_training:
                    curr_lr = self._base_lr
                else:
                    curr_lr = self._base_lr - \
                        self._base_lr * (epoch - half_training) / half_training

                if epoch < self._switch:
                    curr_tr = 0.
                    donorm = True
                    to_train_A = self.g_A_trainer
                    to_train_B = self.g_B_trainer
                else:
                    curr_tr = self._threshold_fg
                    donorm = False
                    to_train_A = self.g_A_trainer_bis
                    to_train_B = self.g_B_trainer_bis

                self.save_images(sess, epoch, curr_tr)

                for i in range(0, max_images):
                    print("Processing batch {}/{}".format(i, max_images))

                    inputs = sess.run(self.inputs)
                    # Optimizing the G_A network
                    _, fake_B_temp, smask_a, summary_str = sess.run(
                        [
                            to_train_A, self.fake_images_b, self.masks[0],
                            self.g_A_loss_summ
                        ],
                        feed_dict={
                            self.input_a: inputs['images_i'],
                            self.input_b: inputs['images_j'],
                            self.learning_rate: curr_lr,
                            self.transition_rate: curr_tr,
                            self.donorm: donorm,
                        })
                    writer.add_summary(summary_str, epoch * max_images + i)

                    fake_B_temp1 = self.fake_image_pool(
                        self.num_fake_inputs, fake_B_temp, smask_a,
                        self.fake_images_B)

                    # Optimizing the D_B network
                    _, summary_str = sess.run(
                        [self.d_B_trainer, self.d_B_loss_summ],
                        feed_dict={
                            self.input_a: inputs['images_i'],
                            self.input_b: inputs['images_j'],
                            self.learning_rate: curr_lr,
                            self.fake_pool_B: fake_B_temp1['im'],
                            self.fake_pool_B_mask: fake_B_temp1['mask'],
                            self.transition_rate: curr_tr,
                            self.donorm: donorm,
                        })
                    writer.add_summary(summary_str, epoch * max_images + i)

                    # Optimizing the G_B network
                    _, fake_A_temp, smask_b, summary_str = sess.run(
                        [
                            to_train_B, self.fake_images_a, self.masks[1],
                            self.g_B_loss_summ
                        ],
                        feed_dict={
                            self.input_a: inputs['images_i'],
                            self.input_b: inputs['images_j'],
                            self.learning_rate: curr_lr,
                            self.transition_rate: curr_tr,
                            self.donorm: donorm,
                        })
                    writer.add_summary(summary_str, epoch * max_images + i)

                    fake_A_temp1 = self.fake_image_pool(
                        self.num_fake_inputs, fake_A_temp, smask_b,
                        self.fake_images_A)

                    # Optimizing the D_A network
                    _, mask_tmp__, summary_str = sess.run(
                        [self.d_A_trainer, self.masks_, self.d_A_loss_summ],
                        feed_dict={
                            self.input_a: inputs['images_i'],
                            self.input_b: inputs['images_j'],
                            self.learning_rate: curr_lr,
                            self.fake_pool_A: fake_A_temp1['im'],
                            self.fake_pool_A_mask: fake_A_temp1['mask'],
                            self.transition_rate: curr_tr,
                            self.donorm: donorm,
                        })
                    writer.add_summary(summary_str, epoch * max_images + i)

                    writer.flush()
                    self.num_fake_inputs += 1

                sess.run(tf.assign(self.global_step, epoch + 1))

            coord.request_stop()
            coord.join(threads)
            writer.add_graph(sess.graph)
Exemplo n.º 47
0
def main():
    global nameBit
    ## number of particles to use
    Nsu = 100
    names = ["sims_01_bifurcation_noninformative"]
    flag_informative = False
    for namecounter in range(len(names)):
        nameNow = names[namecounter]
        (tsim, XK, YK, mu0, P0, Ns, dt, tf) = data_loader.load_data(nameNow, "../sim_data/")

        """
		tsim = tsim[0:5]
		XK = XK[0:5,:]
		YK = YK[0:5,:]
		tf = tsim[4]
		"""

        Ns = 1

        nameBit = int(nameNow[5:7], 2)
        # parse the name
        if nameBit == 1:
            # noise levels for the ENKF with white noise forcing
            Qk = np.array([[1.0]])
            Rk = np.array([[0.1]])
        if nameBit == 2:
            # noise levels for the UKF with cosine forcing
            Qk = np.array([[3.16 / dt]])
            Rk = np.array([[0.1]])
            # number of steps in each simulation
        nSteps = len(tsim)
        nees_history = np.zeros((nSteps, Ns))
        Nf_history = np.zeros((nSteps, Ns))
        e_sims = np.zeros((Ns * nSteps, 2))
        for counter in range(Ns):
            xk = XK[:, (2 * counter) : (2 * counter + 2)]
            yk = YK[:, counter]

            # (Xf,Pf,Idx,Xp) = gmm_test(dt,tf,mu0,P0,yk,Qk,Rk,flag_informative)
            (Xf, pdf, pdfPts, alphai, Pki) = gmm_test(dt, tf, mu0, P0, yk, Qk, Rk, Nsu, flag_informative)
            print("gmm_clustering case %d/%d" % (counter + 1, Ns))

            if Ns == 1:
                fig = []
                for k in range(nSteps):
                    fig.append(plt.figure())
                    ax = fig[k].add_subplot(1, 1, 1, title="t = %f" % (tsim[k]))

                    # the number of active means
                    activeMeans = pdf.shape[1]
                    for jk in range(activeMeans):
                        mux = pdfPts[k, :, jk]
                        ax.plot(pdfPts[k, 0, jk], pdfPts[k, 1, jk], "o")
                        # plot the single-mean covariance ellipsoid
                        # draw points on a unit circle
                        thetap = np.linspace(0, 2 * math.pi, 20)
                        circlP = np.zeros((20, 2))
                        circlP[:, 0] = 3.0 * np.cos(thetap)
                        circlP[:, 1] = 3.0 * np.sin(thetap)
                        # transform the points circlP through P^(1/2)*circlP + mu
                        Phalf = np.real(scipy.linalg.sqrtm(Pki[k, :, :, jk]))
                        ellipsP = np.zeros(circlP.shape)
                        for kj in range(circlP.shape[0]):
                            ellipsP[kj, :] = np.dot(Phalf, circlP[kj, :]) + mux
                        ax.plot(ellipsP[:, 0], ellipsP[:, 1], "--")
                        # plot the truth state
                        ax.plot(xk[k, 0], xk[k, 1], "ks")
                    ax.grid()
                    fig[k].show()
                raw_input("Return to quit")
                for k in range(nSteps):
                    plt.close(fig[k])
            """
			(e1,chi2,mx,Pk) = cluster_processing.singleSimErrors(Xf,Idx,xk,yk)
			nees_history[:,counter] = chi2.copy()
			mean_nees = np.sum(chi2)/float(nSteps)
			print(mean_nees)
			# mean NEES
			mse = np.sum(np.power(e1,2.0),axis=0)/float(nSteps)
			e_sims[(counter*nSteps):(counter*nSteps+nSteps),:] = e1.copy()

			print("MSE: %f,%f" % (mse[0],mse[1]))

		if Ns < 2:
			# plot the mean trajectories and error
			fig1 = plt.figure()
			ax = []
			for k in range(4):
				if k < 2:
					nam = 'x' + str(k+1)
				else:
					nam = 'e' + str(k-1)
				ax.append(fig1.add_subplot(2,2,k+1,ylabel=nam))
				if k < 2:
					ax[k].plot(tsim,xk[:,k],'b-')
					ax[k].plot(tsim,mx[:,k],'m--')
				else:
					ax[k].plot(tsim,e1[:,k-2])
					ax[k].plot(tsim,3.0*np.sqrt(Pk[:,k-2,k-2]),'r--')
					ax[k].plot(tsim,-3.0*np.sqrt(Pk[:,k-2,k-2]),'r--')
				ax[k].grid()
			fig1.show()
		
		mse_tot = np.mean(np.power(e_sims,2.0),axis=0)
		print("mse_tot: %f,%f" % (mse_tot[0],mse_tot[1]))
		
		# get the mean NEES value versus simulation time across all sims
		nees_mean = np.sum(nees_history,axis=1)/Ns
		# get the mean number of particles in time
		Nf_mean = np.sum(Nf_history,axis=1)/Ns
		# get 95% confidence bounds for chi-sqaured... the df is the number of sims times the dimension of the state
		chiUpper = stats.chi2.ppf(.975,2.0*Ns)/float(Ns)
		chiLower = stats.chi2.ppf(.025,2.0*Ns)/float(Ns)

		# plot the mean NEES with the 95% confidence bounds
		fig2 = plt.figure(figsize=(6.0,3.37)) #figsize tuple is width, height
		tilt = "ENKF, Ts = %.2f, %d sims, " % (dt, Ns)
		if nameBit == 0:
			tilt = tilt + 'unforced'
		if nameBit == 1:
			#white-noise only
			tilt = tilt + 'white-noise forcing'
		if nameBit == 2:
			tilt = tilt + 'cosine forcing'
		if nameBit == 3:
			#white-noise and cosine forcing
			tilt = tilt + 'white-noise and cosine forcing'
		ax = fig2.add_subplot(111,ylabel='mean NEES',title=tilt)
		ax.plot(tsim,chiUpper*np.ones(nSteps),'r--')
		ax.plot(tsim,chiLower*np.ones(nSteps),'r--')
		ax.plot(tsim,nees_mean,'b-')
		ax.grid()
		fig2.show()
		# save the figure
		fig2.savefig('nees_enkf2_' + str(Ns) + '_' + nameNow + '.png')
		# find fraction of inliers
		l1 = (nees_mean < chiUpper).nonzero()[0]
		l2 = (nees_mean > chiLower).nonzero()[0]
		# get number of inliers
		len_in = len(set(l1).intersection(l2))
		# get number of super (above) liers (sic)
		len_super = len((nees_mean > chiUpper).nonzero()[0])
		# get number of sub-liers (below)
		len_sub = len((nees_mean < chiLower).nonzero()[0])

		print("Conservative (below 95%% bounds): %f" % (float(len_sub)/float(nSteps)))
		print("Optimistic (above 95%% bounds): %f" % (float(len_super)/float(nSteps)))

		# save metrics
		FID = open('metrics_enkf2_' + str(Ns) + '_' + nameNow + '.txt','w')
		FID.write("mse1,mse2,nees_below95,nees_above95\n")
		FID.write("%f,%f,%f,%f\n" % (mse_tot[0],mse_tot[1],float(len_sub)/float(nSteps),float(len_super)/float(nSteps)))
		FID.close()
	
	raw_input("Return to exit")
	"""
    return
Exemplo n.º 48
0
parser.add_argument('--subset_pct', type=float, default=100,
                    help='subset of training dataset to use (percentage)')
args = parser.parse_args(gen_be=False)

# hyperparameters from the reference
args.batch_size = 64
embed_dim = 620

valid_split = None
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))

# load the documents by giving the path and what extension the files are
data_file, vocab_file = load_data(args.data_dir, valid_split=valid_split,
                                  max_vocab_size=args.max_vocab_size,
                                  max_len_w=args.max_len_w,
                                  output_path=args.output_dir,
                                  file_ext=['txt'],
                                  subset_pct=args.subset_pct)
vocab, rev_vocab, word_count = load_obj(vocab_file)

vocab_size = len(vocab)
neon_logger.display("\nData loading complete.")
neon_logger.display("\nVocab size from the dataset is: {}".format(vocab_size))

index_from = 2  # 0: padding 1: oov
vocab_size_layer = vocab_size + index_from

init_embed_dev = Uniform(low=-0.1, high=0.1)

# sent2vec network
nhidden = 2400
Exemplo n.º 49
0
def main():
	# number of particles, 300 seems necessary for pretty good consistent performance
	Nsu = 100
	global nameBit
	names = ['sims_01_bifurcation_noninformative']
	flag_informative=False
	best_error = True
	for namecounter in range(len(names)):
		nameNow = names[namecounter]
		(tsim,XK,YK,mu0,P0,Ns,dt,tf) = data_loader.load_data(nameNow,'../sim_data/')

		'''
		tsim = tsim[0:2]
		XK = XK[0:2,:]
		YK = YK[0:2,:]
		tf = tsim[1]
		'''
		Ns = 100

		nameBit = int(nameNow[5:7],2)
		# parse the name
		if nameBit == 1:
			# noise levels for the SIR with white noise forcing
			Qk = np.array([[3.16]])
			Rk = np.array([[0.1]])
		'''
		if nameBit == 2:
			# noise levels for the SIR with cosine forcing
			Qk = np.array([[31.6]])
			Rk = np.array([[0.01]])
		'''
		# number of steps in each simulation
		nSteps = len(tsim)
		nees_history = np.zeros((nSteps,Ns))
		e_sims = np.zeros((Ns*nSteps,2))
		for counter in range(Ns):
			xk = XK[:,(2*counter):(2*counter+2)]
			yk = YK[:,counter]

			(xf,Pf,Xp,weights,Xs,xs) = sir_test(dt,tf,mu0,P0,yk,Qk,Rk,Nsu,flag_informative)
			# call PF cluster processing function
			weightsEqual = np.ones((nSteps,Xs.shape[2]))*1.0/float(Xs.shape[2])
			(e1,chi2,mxnu,Pxnu) = cluster_processing.singleSimErrorsPf(Xp,weights,xk,best_error)

			# chi2 is the NEES statistic. Take the mean
			nees_history[:,counter] = chi2.copy()
			#mean_nees = np.sum(chi2)/float(nSteps)
			mean_nees = np.mean(chi2)
			print(mean_nees)
			# mean NEES
			mse = np.sum(np.power(e1,2.0),axis=0)/float(nSteps)
			e_sims[(counter*nSteps):(counter*nSteps+nSteps),:] = e1.copy()
			print("sir_clustering case %d" % counter)
			print("MSE: %f,%f" % (mse[0],mse[1]))
		if Ns < 2:
			# loop over the particles after sampling and cluster using kmeans
			# errors for the bifurcated case
			'''
			e2case = np.zeros((2,nSteps,2))
			for k in range(nSteps):
				# cluster into two means
				(idxk,mui) = kmeans.kmeans(Xs[k,:,:].transpose(),2)
				# compute the errors for the two means
				for jk in range(2):
					e2case[jk,k,:] = mui[jk,:]-xk[k,:]
			'''
			# plot of the discrete PDF and maximum likelihood estimate
			# len(tsim) x Ns matrix of times
			tMesh = np.kron(np.ones((Nsu,1)),tsim).transpose()
			fig1 = plt.figure()
			ax = []
			for k in range(6):
				if k < 2:
					nam = 'pdf' + str(k+1)
				elif k < 4:
					nam = 'xf' + str(k-1)
				else:
					nam = 'ef' + str(k-3)
				ax.append( fig1.add_subplot(3,2,k+1,ylabel=nam) )
				if k < 2:
					if k == 0:
						# plot the discrete PDF as a function of time
						mex = tMesh.reshape((len(tsim)*Nsu,))
						mey = Xp[:,0,:].reshape((len(tsim)*Nsu,))
						mez = weights.reshape((len(tsim)*Nsu,))
					elif k == 1:
						# plot the discrete PDF as a function of time
						mex = tMesh.reshape((len(tsim)*Nsu,))
						mey = Xp[:,1,:].reshape((len(tsim)*Nsu,))
						mez = weights.reshape((len(tsim)*Nsu,))
					idx = mez.argsort()
					mexx,meyy,mezz = mex[idx],mey[idx],mez[idx]

					cc = ax[k].scatter(mexx,meyy,c=mezz,s=20,edgecolor='')
					fig1.colorbar(cc,ax=ax[k])
					# plot the truth
					ax[k].plot(tsim,xk[:,k],'b-')
				elif k < 4:
					ax[k].plot(tsim,xf[:,k-2],'m--')
					ax[k].plot(tsim,xk[:,k-2],'b-')
				else:
					#ax[k].plot(tsim,xf[:,k-4]-xk[:,k-4],'b-')
					# plot the error based on maximum likelihood
					ax[k].plot(tsim,e1[:,k-4],'y-')
					ax[k].plot(tsim,3.0*np.sqrt(Pxnu[:,k-4,k-4]),'r--')
					ax[k].plot(tsim,-3.0*np.sqrt(Pxnu[:,k-4,k-4]),'r--')
					#ax[k].plot(tsim,e2case[0,:,k-4],'y-')
					#ax[k].plot(tsim,e2case[1,:,k-4],'y-')
					#ax[k].plot(tsim,3.0*np.sqrt(Pf[:,k-4 + 2*(k-4)]),'r--')
					#ax[k].plot(tsim,-3.0*np.sqrt(Pf[:,k-4 + 2*(k-4)]),'r--')
				ax[k].grid()

			fig1.show()

		mse_tot = np.mean(np.power(e_sims,2.0),axis=0)
		print("mse_tot: %f,%f" % (mse_tot[0],mse_tot[1]))

		if Ns == 1:
				fig = []
				for k in range(nSteps):
					fig.append(plt.figure())
					ax = fig[k].add_subplot(1,1,1,title="t = %f" % (tsim[k]),xlim=(-25,25),ylim=(-20,20),ylabel='x2',xlabel='x1')
					ax.plot(Xp[k,0,:],Xp[k,1,:],'bd')#propagated values
					ax.plot(Xs[k,0,:],Xs[k,1,:],'ys')#re-sampled values
					#compute the number of active means
					# plot the truth state
					ax.plot(xk[k,0],xk[k,1],'cs')
					# plot the maximum likelihood cluster mean and covariance ellipse
					# plot the single-mean covariance ellipsoid
					# draw points on a unit circle
					ax.plot(mxnu[k,0],mxnu[k,1],'rd')
					fig[k].show()
				raw_input("Return to quit")
				for k in range(nSteps):
					fig[k].savefig('stepByStep/sir_' + str(Nsu) + "_" + str(k) + '.png')
					plt.close(fig[k])
		else:
			if best_error:
				
				# get the mean NEES value versus simulation time across all sims
				nees_mean = np.sum(nees_history,axis=1)/Ns
				# get 95% confidence bounds for chi-sqaured... the df is the number of sims times the dimension of the state
				chiUpper = stats.chi2.ppf(.975,2.0*Ns)/float(Ns)
				chiLower = stats.chi2.ppf(.025,2.0*Ns)/float(Ns)
				# find fraction of inliers
				l1 = (nees_mean < chiUpper).nonzero()[0]
				l2 = (nees_mean > chiLower).nonzero()[0]
				# get number of inliers
				len_in = len(set(l1).intersection(l2))
				# get number of super (above) liers (sic)
				len_super = len((nees_mean > chiUpper).nonzero()[0])
				# get number of sub-liers (below)
				len_sub = len((nees_mean < chiLower).nonzero()[0])

				print("Conservative (below 95%% bounds): %f" % (float(len_sub)/float(nSteps)))
				print("Optimistic (above 95%% bounds): %f" % (float(len_super)/float(nSteps)))

				# save metrics
				FID = open('bestErrors_sir_' + str(Nsu) + '_' + nameNow + '.txt','w')
				FID.write("mse1,mse2,nees_below95,nees_above95\n")
				FID.write("%f,%f,%f,%f\n" % (mse_tot[0],mse_tot[1],float(len_sub)/float(nSteps),float(len_super)/float(nSteps)))
				FID.close()
				return
			print("Passing to exit")
			pass
			# get the mean NEES value versus simulation time across all sims
			nees_mean = np.sum(nees_history,axis=1)/Ns
			# get 95% confidence bounds for chi-sqaured... the df is the number of sims times the dimension of the state
			chiUpper = stats.chi2.ppf(.975,2.0*Ns)/float(Ns)
			chiLower = stats.chi2.ppf(.025,2.0*Ns)/float(Ns)

			# plot the mean NEES with the 95% confidence bounds
			fig2 = plt.figure(figsize=(6.0,3.37)) #figsize tuple is width, height
			tilt = "SIR, Ts = %.2f, %d sims, %d particles, " % (dt, Ns, Nsu)
			if nameBit == 0:
				tilt = tilt + 'unforced'
			if nameBit == 1:
				#white-noise only
				tilt = tilt + 'white-noise forcing'
			if nameBit == 2:
				tilt = tilt + 'cosine forcing'
			if nameBit == 3:
				#white-noise and cosine forcing
				tilt = tilt + 'white-noise and cosine forcing'
			ax = fig2.add_subplot(111,ylabel='mean NEES')#,title=tilt)
			ax.set_title(tilt,fontsize = 12)
			ax.plot(tsim,chiUpper*np.ones(nSteps),'r--')
			ax.plot(tsim,chiLower*np.ones(nSteps),'r--')
			ax.plot(tsim,nees_mean,'b-')
			ax.grid()
			fig2.show()
			# save the figure
			fig2.savefig('nees_sir_' + str(Nsu) + "_" + nameNow + '.png')
			# find fraction of inliers
			l1 = (nees_mean < chiUpper).nonzero()[0]
			l2 = (nees_mean > chiLower).nonzero()[0]
			# get number of inliers
			len_in = len(set(l1).intersection(l2))
			# get number of super (above) liers (sic)
			len_super = len((nees_mean > chiUpper).nonzero()[0])
			# get number of sub-liers (below)
			len_sub = len((nees_mean < chiLower).nonzero()[0])

			print("Conservative (below 95%% bounds): %f" % (float(len_sub)/float(nSteps)))
			print("Optimistic (above 95%% bounds): %f" % (float(len_super)/float(nSteps)))

			# save metrics
			FID = open('metrics_sir_' + str(Nsu) + "_" + nameNow + '.txt','w')
			FID.write("mse1,mse2,nees_below95,nees_above95\n")
			FID.write("%f,%f,%f,%f\n" % (mse_tot[0],mse_tot[1],float(len_sub)/float(nSteps),float(len_super)/float(nSteps)))
			FID.close()

	print("Leaving sir_clustering")

	return
Exemplo n.º 50
0
    type=str,
    default='TransE',
    help=
    'knowledge graph embedding method, please ensure that the specified input file exists'
)
parser.add_argument(
    '--entity_dim',
    type=int,
    default=50,
    help=
    'dimension of entity embeddings, please ensure that the specified input file exists'
)
parser.add_argument(
    '--word_dim',
    type=int,
    default=50,
    help=
    'dimension of word embeddings, please ensure that the specified input file exists'
)
parser.add_argument(
    '--max_title_length',
    type=int,
    default=10,
    help=
    'maximum length of news titles, should be in accordance with the input datasets'
)
args = parser.parse_args()

train_data, test_data = load_data(args)
train(args, train_data, test_data)
Exemplo n.º 51
0
from models.property import Property
from data_loader import load_data

if __name__ == '__main__':
    load_data()
Exemplo n.º 52
0
def generate_features(img, lab):
    return ave_brightness(img)


# Converts features of music over time to features of images over time
def create_mf_to_if_model(strength):
    def mf_to_if_model(mf):
        im_feats = {
            5: strength * np.maximum(0, mf),
            6: strength * np.minimum(0, mf),
            -1: strength * mf
        }
        return im_feats

    return mf_to_if_model


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("-strength", type=float, default=5)
    args = parser.parse_args()
    mf = np.sin(np.linspace(-np.pi, np.pi, 30 * 5))
    mf_to_if_model = create_mf_to_if_model(args.strength)
    dataset = data_loader.load_data()
    start_image, lab = dataset.take(1).make_one_shot_iterator().get_next()
    print(lab)
    print(lab.shape)
    create_music_video(start_image, mf, mf_to_if_model, dataset)
    frames_to_video.create_video_from_frames()
Exemplo n.º 53
0
def main():
	global nameBit
	#names = ['sims_11_slow']# test case
	names = ['sims_10_medium','sims_01_medium','sims_11_medium']
	#names = ['sims_01_slow','sims_01_medium','sims_01_fast','sims_10_slow','sims_10_medium','sims_10_fast','sims_11_slow','sims_11_medium','sims_11_fast']
	for namecounter in range(len(names)):
		nameNow = names[namecounter]
		(tsim,XK,YK,mu0,P0,Ns,dt,tf) = data_loader.load_data(nameNow,'../sim_data/')

		nameBit = int(nameNow[5:7],2)
		# parse the name
		if nameBit == 1:
			Rk = np.array([[1.0]])
			# tuned UKF with white noise forcing
			if dt > .9:# slow sampling
				Qk = np.array([[0.00316]])
			elif dt > 0.09:# medium sampling
				Qk = np.array([[0.01]])
			else:# fast sampling
				Qk = np.array([[0.00316]])
		if nameBit == 2:
			# tuned noise levels for the UKF with cosine forcing
			if dt > .9:# slow sampling
				Qk = np.array([[0.25]])
			elif dt > 0.09:# medium sampling
				Qk = np.array([[1.5]])
			else:# fast sampling
				Qk = np.array([[20.0]])
			Rk = np.array([[1.0]])
		if nameBit == 3:
			# noise levels for the UKF with cosine forcing and white noise
			if dt > .9:# slow sampling
				Qk = np.array([[0.25]])
			elif dt > 0.09:# medium sampling
				Qk = np.array([[1.5]])
			else:# fast sampling
				Qk = np.array([[16.0]])
			Rk = np.array([[1.0]])
		# number of steps in each simulation
		nSteps = len(tsim)
		nees_history = np.zeros((nSteps,Ns))
		e_sims = np.zeros((Ns*nSteps,2))
		for counter in range(Ns):
			xk = XK[:,(2*counter):(2*counter+2)]
			yk = YK[:,counter]

			(xf,Pf) = ukf_test(dt,tf,mu0,P0,yk,Qk,Rk)

			# compute the unit variance transformation of the error
			e1 = np.zeros((nSteps,2))
			chi2 = np.zeros(nSteps)
			for k in range(nSteps):
				P = Pf[k,:].reshape((2,2))
				Pinv = np.linalg.inv(P)
				chi2[k] = np.dot(xk[k,:]-xf[k,:],np.dot(Pinv,xk[k,:]-xf[k,:]))
			# chi2 is the NEES statistic. Take the mean
			nees_history[:,counter] = chi2.copy()
			mean_nees = np.sum(chi2)/float(nSteps)
			print(mean_nees)
			# mean NEES
			mse = np.sum(np.power(xk-xf,2.0),axis=0)/float(nSteps)
			e_sims[(counter*nSteps):(counter*nSteps+nSteps),:] = xk-xf

			print("MSE: %f,%f" % (mse[0],mse[1]))

			# chi-square test statistics
			# (alpha) probability of being less than the returned value: stats.chi2.ppf(alpha,df=Nsims)
		if Ns < 2:
			fig1 = plt.figure()
			ax = []
			for k in range(4):
				if k < 2:
					nam = 'x' + str(k+1)
				else:
					nam = 'e' + str(k-1)
				ax.append(fig1.add_subplot(2,2,k+1,ylabel=nam))
				if k < 2:
					ax[k].plot(tsim,xk[:,k],'b-')
					ax[k].plot(tsim,xf[:,k],'m--')
					if k == 0:
						ax[k].plot(tsim,yk,'r--')
				else:
					ax[k].plot(tsim,xk[:,k-2]-xf[:,k-2])
					ax[k].plot(tsim,3.0*np.sqrt(Pf[:,3*(k-2)]),'r--')
					ax[k].plot(tsim,-3.0*np.sqrt(Pf[:,3*(k-2)]),'r--')
				ax[k].grid()
			fig1.show()

			fig2 = plt.figure()
			ax = []
			ax.append(fig2.add_subplot(111,ylabel = 'nees metric'))
			ax[0].plot(tsim,chi2)
			ax[0].grid()
			fig2.show()
		else:
			trials_processing.errorParsing(e_sims,nees_history,'ukf',nameNow)

			mse_tot = np.mean(np.power(e_sims,2.0),axis=0)
			print("mse_tot: %f,%f" % (mse_tot[0],mse_tot[1]))
			
			# get the mean NEES value versus simulation time across all sims
			nees_mean = np.sum(nees_history,axis=1)/Ns
			# get 95% confidence bounds for chi-sqaured... the df is the number of sims times the dimension of the state
			chiUpper = stats.chi2.ppf(.975,2.0*Ns)/float(Ns)
			chiLower = stats.chi2.ppf(.025,2.0*Ns)/float(Ns)

			# plot the mean NEES with the 95% confidence bounds
			fig2 = plt.figure(figsize=(6.0,3.37)) #figsize tuple is width, height
			tilt = "UKF, Ts = %.2f, %d sims, " % (dt, Ns)
			if nameBit == 0:
				tilt = tilt + 'unforced'
			if nameBit == 1:
				#white-noise only
				tilt = tilt + 'white-noise forcing'
			if nameBit == 2:
				tilt = tilt + 'cosine forcing'
			if nameBit == 3:
				#white-noise and cosine forcing
				tilt = tilt + 'white-noise and cosine forcing'
			ax = fig2.add_subplot(111,ylabel='mean NEES',title=tilt)
			ax.plot(tsim,chiUpper*np.ones(nSteps),'r--')
			ax.plot(tsim,chiLower*np.ones(nSteps),'r--')
			ax.plot(tsim,nees_mean,'b-')
			ax.grid()
			fig2.show()
			# save the figure
			fig2.savefig('nees_ukf_' + nameNow + '.png')
			# find fraction of inliers
			l1 = (nees_mean < chiUpper).nonzero()[0]
			l2 = (nees_mean > chiLower).nonzero()[0]
			# get number of inliers
			len_in = len(set(l1).intersection(l2))
			# get number of super (above) liers (sic)
			len_super = len((nees_mean > chiUpper).nonzero()[0])
			# get number of sub-liers (below)
			len_sub = len((nees_mean < chiLower).nonzero()[0])

			print("Conservative (below 95%% bounds): %f" % (float(len_sub)/float(nSteps)))
			print("Optimistic (above 95%% bounds): %f" % (float(len_super)/float(nSteps)))

			# save metrics
			FID = open('metrics_ukf_' + nameNow + '.txt','w')
			FID.write("mse1,mse2,nees_below95,nees_above95\n")
			FID.write("%f,%f,%f,%f\n" % (mse_tot[0],mse_tot[1],float(len_sub)/float(nSteps),float(len_super)/float(nSteps)))
			FID.close()

			# plot all NEES
			fig = plt.figure(figsize=(6.0,3.37))
			ax = fig.add_subplot(111,ylabel='NEES')
			ax.plot(tsim,nees_history,'b-')
			ax.grid()
			fig.show()

	raw_input("Return to quit")

	print("Leaving ukf_trials")
	return
Exemplo n.º 54
0
import data_loader as dl
import learner
from sklearn import metrics
from copy import deepcopy
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

# params
pool_size = 100
bsize = 50
budget = 2500

# load data
xtrain, ytrain = dl.load_data('Data/DIFFICULT_TRAIN.csv')
xtest, ytest = dl.load_data('Data/DIFFICULT_TEST.csv')
xtest = np.array(xtest)

xtrain_label, xtrain_nolabel, ytrain_label, ytrain_nolabel = dl.pool_data(
    pool_size, xtrain, ytrain)
xtrain_label_rand, xtrain_nolabel_rand, ytrain_label_rand, ytrain_nolabel_rand = dl.pool_data(
    pool_size, xtrain, ytrain)

active_errors = []
random_errors = []
batches = []

min_error = 1
min_cost = 0
best_model = None
from keras.layers.core import Dense, Activation
from keras.layers.recurrent import LSTM
from keras.models import Sequential
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt
import time
import data_loader
import numpy as np

WINDOW = 10

X, Y, options = data_loader.load_data(
    "etherium_data_pretty.json",
    window_size=WINDOW,
    remove_features=["date"],
    preprocess_args={'normaliser': StandardScaler()})

x_train, x_test, y_train, y_test = train_test_split(X,
                                                    Y,
                                                    test_size=0.3,
                                                    random_state=0)

model = Sequential()

model.add(
    LSTM(input_shape=(None, 7), units=100, dropout=0.2, return_sequences=True))

model.add(LSTM(200, dropout=0.2, return_sequences=False))

model.add(Dense(units=1))
Exemplo n.º 56
0
from create_data import create_labeled_data, create_unlabeled_data, create_testing_data
from networks import FNet, GNet
from metrics import LossDiscriminative, LossGenerative
from classifier import test_classification

n_epochs = 30
batch_size = 32
alpha = 0.05
beta = 0.0001

random_seed = 0
torch.manual_seed(random_seed)
torch.backends.cudnn.enabled = True

# Load the data
train_loader, test_loader = load_data()

# Training data
train_set = enumerate(train_loader)
train_batch_idx, (train_data, train_targets) = next(train_set)

# Testing data
test_set = enumerate(test_loader)
test_batch_idx, (test_data, test_targets) = next(test_set)

# Vizualize MNIST dataset
plot_MNIST_data(train_data, train_targets)

# Create labeled, unlabeled and test data
n = 100
labeled_pos, labeled_neg = create_labeled_data(n, train_targets)
Exemplo n.º 57
0
except :
  print __doc__
  sys.exit(1)

f = open("../../inputs/photoz3/set3.counts")
z_id, counts = [], []
for line in f.readlines()[1:] :
  zi, ci = line[:-1].split()

  z_id.append(zi.replace(".", "p"))
  counts.append(eval(ci))

z_id   = np.array(z_id)
counts = np.array(counts)

jrun0 = dl.load_data(ID = "z0p00", n_SFH = counts[z_id == "z0p00"][0], n_trials = 50, n_bands = 56)
jrunz = dl.load_data(ID = z, n_SFH = counts[z_id == z][0], n_trials = 50, n_bands = 56)

itrialz = 103#np.random.randint(counts[z_id == z][0] * 50)
itrial0 = np.where(jrun0.library["SFH_name"] == jrunz.library["SFH_name"][itrialz].replace(z, "z0p00"))[0][0]

print
print "selected galaxy", itrialz, jrun0.library["SFH_name"][itrial0], jrunz.library["SFH_name"][itrialz]
print

def tra(val):
  if 220 <= val < 220*2:
    new = val - 220
  elif 220*2 <= val < 660:
    new = val - 220*2
  else:
Exemplo n.º 58
0
def train(optimizer=HYPERPARAMS.optimizer,
          optimizer_param=HYPERPARAMS.optimizer_param,
          learning_rate=HYPERPARAMS.learning_rate,
          keep_prob=HYPERPARAMS.keep_prob,
          learning_rate_decay=HYPERPARAMS.learning_rate_decay,
          decay_step=HYPERPARAMS.decay_step,
          train_model=True):

    print("loading dataset " + DATASET.name + "...")
    if train_model:
        data, validation = load_data(validation=True)
    else:
        data, validation, test = load_data(validation=True, test=True)

    with tf.Graph().as_default():
        print("building model...")
        network = build_model(optimizer, optimizer_param, learning_rate,
                              keep_prob, learning_rate_decay, decay_step)
        model = DNN(network,
                    tensorboard_dir=TRAINING.logs_dir,
                    tensorboard_verbose=0,
                    checkpoint_path=TRAINING.checkpoint_dir,
                    max_checkpoints=TRAINING.max_checkpoints)

        #tflearn.config.init_graph(seed=None, log_device=False, num_cores=6)

        if train_model:
            # Training phase
            print("start training...")
            print("  - emotions = {}".format(NETWORK.output_size))
            print("  - optimizer = '{}'".format(optimizer))
            print("  - learning_rate = {}".format(learning_rate))
            print("  - learning_rate_decay = {}".format(learning_rate_decay))
            print("  - otimizer_param ({}) = {}".format(
                'beta1' if optimizer == 'adam' else 'momentum',
                optimizer_param))
            print("  - keep_prob = {}".format(keep_prob))
            print("  - epochs = {}".format(TRAINING.epochs))
            print("  - use landmarks = {}".format(NETWORK.use_landmarks))
            print("  - use hog + landmarks = {}".format(
                NETWORK.use_hog_and_landmarks))
            print("  - use hog sliding window + landmarks = {}".format(
                NETWORK.use_hog_sliding_window_and_landmarks))
            print("  - use batchnorm after conv = {}".format(
                NETWORK.use_batchnorm_after_conv_layers))
            print("  - use batchnorm after fc = {}".format(
                NETWORK.use_batchnorm_after_fully_connected_layers))

            start_time = time.time()
            if NETWORK.use_landmarks:
                model.fit(
                    [data['X'], data['X2']],
                    data['Y'],
                    validation_set=([validation['X'],
                                     validation['X2']], validation['Y']),
                    # for batch-norm
                    shuffle=True,
                    snapshot_step=TRAINING.snapshot_step,
                    show_metric=TRAINING.vizualize,
                    batch_size=TRAINING.batch_size,
                    n_epoch=TRAINING.epochs)
            else:
                model.fit(data['X'],
                          data['Y'],
                          validation_set=(validation['X'], validation['Y']),
                          shuffle=True,
                          snapshot_step=TRAINING.snapshot_step,
                          show_metric=TRAINING.vizualize,
                          batch_size=TRAINING.batch_size,
                          n_epoch=TRAINING.epochs)
                validation['X2'] = None
            training_time = time.time() - start_time
            print("training time = {0:.1f} sec".format(training_time))

            if TRAINING.save_model:
                print("saving model...")
                model.save(TRAINING.save_model_path)
                if not(os.path.isfile(TRAINING.save_model_path)) and \
                        os.path.isfile(TRAINING.save_model_path + ".meta"):
                    os.rename(TRAINING.save_model_path + ".meta",
                              TRAINING.save_model_path)

            print("evaluating...")
            validation_accuracy = evaluate(model, validation['X'],
                                           validation['X2'], validation['Y'])
            print("  - validation accuracy = {0:.1f}".format(
                validation_accuracy * 100))
            return validation_accuracy
        else:
            # Testing phase : load saved model and evaluate on test dataset
            print("start evaluation...")
            print("loading pretrained model...")
            if os.path.isfile(TRAINING.save_model_path):
                model.load(TRAINING.save_model_path)
            else:
                print("Error: file '{}' not found".format(
                    TRAINING.save_model_path))
                exit()

            if not NETWORK.use_landmarks:
                validation['X2'] = None
                test['X2'] = None

            print("--")
            print("Validation samples: {}".format(len(validation['Y'])))
            print("Test samples: {}".format(len(test['Y'])))
            print("--")
            print("evaluating...")
            start_time = time.time()
            validation_accuracy = evaluate(model, validation['X'],
                                           validation['X2'], validation['Y'])
            print("  - validation accuracy = {0:.1f}".format(
                validation_accuracy * 100))
            test_accuracy = evaluate(model, test['X'], test['X2'], test['Y'])
            print("  - test accuracy = {0:.1f}".format(test_accuracy * 100))
            print("  - evalution time = {0:.1f} sec".format(time.time() -
                                                            start_time))
            return test_accuracy
Exemplo n.º 59
0
    def train(self, epochs, batch_size=128, save_interval=50):

        # Load the dataset
        # (X_train, _), (_, _) = mnist.load_data()
        generator = load_data(batch_size, self.img_rows, self.img_cols)

        # Rescale -1 to 1
        # X_train = X_train / 127.5 - 1.
        # X_train = np.expand_dims(X_train, axis=3)

        # Adversarial ground truths
        valid = np.ones((batch_size, 1))
        fake = np.zeros((batch_size, 1))

        for epoch in range(epochs):

            # ---------------------
            #  Train Discriminator
            # ---------------------

            # Select a random half of images
            # idx = np.random.randint(0, X_train.shape[0], batch_size)
            # imgs = X_train[idx]
            imgs, _ = next(generator)
            imgs = imgs / 127.5 - 1.
            if imgs.shape[0] != batch_size:
                continue

            # Sample noise and generate a batch of new images
            noise = np.random.uniform(-1, 1, (batch_size, self.latent_dim))
            # noise = np.random.normal(0, 0.02, (batch_size, self.latent_dim))
            gen_imgs = self.generator.predict(noise)

            # Train the discriminator (real classified as ones and generated as zeros)
            # X = np.concatenate((imgs, gen_imgs))
            # y = np.concatenate((valid, fake))
            # d_loss = self.discriminator.train_on_batch(X, y)
            d_loss_real = self.discriminator.train_on_batch(imgs, valid)
            d_loss_fake = self.discriminator.train_on_batch(gen_imgs, fake)
            d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)

            # ---------------------
            #  Train Generator
            # ---------------------

            # if epoch < 200:
            #     print('Train %d, d_loss: %f' % (epoch, d_loss[0]))
            # else:
            # Train the generator (wants discriminator to mistake images as real)
            self.discriminator.trainable = False
            g_loss = self.combined.train_on_batch(noise, valid)
            self.discriminator.trainable = True

            # Plot the progress
            print(
                "%07d [D loss: %7.4f, acc: %6.2f%%] [G loss: %7.4f, acc: %6.2f%%]"
                % (epoch, d_loss[0], 100 * d_loss[1], g_loss[0],
                   100 * g_loss[1]))

            # If at save interval => save generated image samples
            if epoch % save_interval == 0:
                self.save_imgs(epoch)
                self._save_weight(epoch)
Exemplo n.º 60
0
def main(ds_name, model_filename, mask_prot_attrs):
    class_attr, prot_attrs, input_vars, train_x, train_y, test_x, test_y = load_data(
        ds_name, mask_prot_attrs)

    print('Train Data: X shape = ', train_x.shape, ' Y shape = ',
          train_y.shape)
    print('Test Data: X shape = ', test_x.shape, ' Y shape = ', test_y.shape)

    kernel = 'poly'
    max_iter = 10000

    # Training params
    d = 2  # kernel degree
    C, gamma, r = 1., 0.001, 0.

    if os.path.exists(model_filename) and LOAD_MODEL_IF_EXISTS:
        print('Loading model from file: ', model_filename, flush=True)
        clf = joblib.load(model_filename)
    else:
        print('Training model...', flush=True)
        clf = SVC(kernel=kernel,
                  verbose=True,
                  max_iter=max_iter,
                  C=C,
                  gamma=gamma,
                  coef0=r,
                  degree=d)
        print('Time check:',
              time.strftime("%H:%M:%S", time.localtime()),
              flush=True)
        clf.fit(train_x, train_y)
        print('Training complete...', flush=True)
        print('Time check:',
              time.strftime("%H:%M:%S", time.localtime()),
              flush=True)

        # Save model to file
        joblib.dump(clf, model_filename)
        print('Saved model to file: ', model_filename, flush=True)
    # End if

    d = clf.degree
    C = clf.C
    gamma = clf.gamma
    r = clf.coef0

    y_pred_train = clf.predict(train_x)
    y_pred_test = clf.predict(test_x)

    acc = accuracy_score(train_y, y_pred_train)
    cm = confusion_matrix(train_y, y_pred_train)

    print('Confusion Matrix (Train):', cm)
    print('Accuracy score (Train):', acc)

    acc = accuracy_score(test_y, y_pred_test)
    cm = confusion_matrix(test_y, y_pred_test)

    print('Confusion Matrix (Test):', cm)
    print('Accuracy score (Test):', acc)

    print('\nBuilding polynomial model...', flush=True)
    svm_P = svm2poly(sv_weights=np.ravel(clf.dual_coef_),
                     sv_x=clf.support_vectors_,
                     bias=clf.intercept_,
                     gamma=gamma,
                     r=r,
                     d=d)

    # svm_P_preds = np.zeros_like(y_pred_train)
    # for i in range(train_x.shape[0]):
    # 	y_i = svm_P.evaluate(train_x[i])
    # 	svm_P_preds[i] = (y_i > 0).astype(np.int32)
    # print('Checks preds: ', np.all(y_pred_train == svm_P_preds), flush = True)

    print('Num SV = %d, C = %f, gamma = %f, r = %f, d = %d' %
          (clf.support_vectors_.shape[0], C, gamma, r, d),
          flush=True)

    print('\nVERIFYING INDIVIDUAL BIAS\n---------------------------',
          flush=True)
    print('Time check: ',
          time.strftime("%H:%M:%S", time.localtime()),
          flush=True)
    t0 = time.perf_counter()

    # Do bias verification here
    res = verify(svm_P, input_vars)

    t1 = time.perf_counter()
    print('Total Time taken: %.3f secs (%.3f mins)' % (t1 - t0,
                                                       (t1 - t0) / 60.0),
          flush=True)

    no_bias = True
    for r in res:
        if r != 0:
            print('FINAL RESULT: Possible bias.', flush=True)
            no_bias = False
    # End for
    if no_bias:
        print('FINAL RESULT: No bias.', flush=True)

    print('END', flush=True)