def __init__(self, mini_batch_size):
        self.image_shape = (16, 128, 128, 3)
        learning_rate = 0.03
        opt = keras.optimizers.Adam(lr=learning_rate)
        opt1 = keras.optimizers.Adam(lr=learning_rate)
        opt_slow = keras.optimizers.Adam(lr=1)
        #Build and compile the discriminator
        self.discriminator = create_discriminator_model()
        self.discriminator.compile(loss='binary_crossentropy',
                                   optimizer=opt,
                                   metrics=[
                                       'accuracy',
                                       tf.keras.metrics.TruePositives(),
                                       tf.keras.metrics.FalsePositives(),
                                       tf.keras.metrics.TrueNegatives(),
                                       tf.keras.metrics.FalseNegatives()
                                   ])
        #Build and compile the generator
        self.generator = AutoEncoderModel()
        self.generator.compile(loss='mse', optimizer=opt_slow)

        #the generator takes a video as input and generates a modified video
        z = Input(shape=(self.image_shape))
        img = self.generator(z)
        self.discriminator.trainable = False
        validity = self.discriminator(img)
        self.combined = Model(z, validity)
        self.combined.compile(loss='binary_crossentropy',
                              optimizer=opt1,
                              metrics=['accuracy'])
        self.dir_path = '/kaggle/input/ucf-crime-training-subset/tfrecords2/'
        self.ds = build_dataset(self.dir_path,
                                batch_size=mini_batch_size,
                                file_buffer=512 * 1024)
Ejemplo n.º 2
0
    def setup(self, X, num_centers, alpha, znum, save_to='dec_model'):
        # Read previously trained _SAE
        ae_model = AutoEncoderModel(self.xpu,
                                    [X.shape[1], 500, 500, 2000, znum],
                                    pt_dropout=0.2)
        ae_model.load(
            os.path.join(
                save_to,
                'SAE_zsize{}_wimgfeatures_descStats_zeromean.arg'.format(
                    str(znum))))  #_Nbatch_wimgfeatures
        logging.log(
            logging.INFO,
            "Reading Autoencoder from file..: %s" % (os.path.join(
                save_to, 'SAE_zsize{}_wimgfeatures_descStats_zeromean.arg'.
                format(znum))))
        self.ae_model = ae_model
        logging.log(logging.INFO, "finished reading Autoencoder from file..: ")

        self.dec_op = DECModel.DECLoss(num_centers, alpha)
        label = mx.sym.Variable('label')
        self.feature = self.ae_model.encoder
        self.loss = self.dec_op(data=self.ae_model.encoder,
                                label=label,
                                name='dec')
        self.args.update({
            k: v
            for k, v in self.ae_model.args.items()
            if k in self.ae_model.encoder.list_arguments()
        })
        self.args['dec_mu'] = mx.nd.empty(
            (num_centers, self.ae_model.dims[-1]), ctx=self.xpu)
        self.args_grad.update({
            k: mx.nd.empty(v.shape, ctx=self.xpu)
            for k, v in self.args.items()
        })
        self.args_mult.update(
            {k: k.endswith('bias') and 2.0 or 1.0
             for k in self.args})
        self.num_centers = num_centers
        self.best_args = {}
        self.best_args['num_centers'] = num_centers
        self.best_args['znum'] = znum
Ejemplo n.º 3
0
######################
from decModel_wimgF_dualopt_descStats import *
from utilities import *
import model

save_to = r'Z:\Cristina\Section3\paper_notes_section3_MODIFIED\save_to\SAEmodels'
input_size = combX_allNME.shape[1]
latent_size = [input_size / rxf for rxf in [15, 10, 5, 2]]

############ BEST PERFORMIGN of Step 1 nzum = 2x
znum = 261

X = combX_allNME
y = roi_labels
xpu = mx.cpu()
ae_model = AutoEncoderModel(xpu, [X.shape[1], 500, 500, 2000, znum],
                            pt_dropout=0.2)
print('Loading autoencoder of znum = {}, post training'.format(znum))
ae_model.load(
    os.path.join(
        save_to,
        'SAE_zsize{}_wimgfeatures_descStats_zeromean.arg'.format(str(znum))))

data_iter = mx.io.NDArrayIter({'data': X},
                              batch_size=X.shape[0],
                              shuffle=False,
                              last_batch_handle='pad')
# extract only the encoder part of the SAE
feature = ae_model.encoder
zspace = model.extract_feature(feature, ae_model.args, None, data_iter,
                               X.shape[0], xpu).values()[0]
Ejemplo n.º 4
0
    # train/test splits (test is 10% of labeled data)
    sep = int(combX_allNME.shape[0] * 0.10)
    X_val = combX_allNME[:sep]
    y_val = YnxG_allNME[1][:sep]
    X_train = combX_allNME[sep:]
    y_train = YnxG_allNME[1][sep:]
    batch_size = 125  # 160 32*5 = update_interval*5

    allAutoencoders = []
    for output_size in latent_size:
        # Train or Read autoencoder: interested in encoding/decoding the input nxg features into LD latent space
        # optimized for clustering with DEC
        xpu = mx.cpu()
        ae_model = AutoEncoderModel(
            xpu, [X_train.shape[1], 500, 500, 2000, output_size],
            pt_dropout=0.2)
        ##  Pre-train
        ae_model.layerwise_pretrain(X_train,
                                    batch_size,
                                    50000,
                                    'sgd',
                                    l_rate=0.1,
                                    decay=0.0,
                                    lr_scheduler=mx.misc.FactorScheduler(
                                        20000, 0.1))
        ##  finetune
        ae_model.finetune(X_train,
                          batch_size,
                          100000,
                          'sgd',
Ejemplo n.º 5
0
# pylint: skip-file
import mxnet as mx
import numpy as np
import logging
import data
from autoencoder import AutoEncoderModel

if __name__ == '__main__':
    # set to INFO to see less information during training
    logging.basicConfig(level=logging.DEBUG)
    ae_model = AutoEncoderModel(mx.gpu(0), [784,500,500,2000,10], pt_dropout=0.2,
        internal_act='relu', output_act='relu')

    X, _ = data.get_mnist()
    train_X = X[:60000]
    val_X = X[60000:]

    ae_model.layerwise_pretrain(train_X, 256, 50000, 'sgd', l_rate=0.1, decay=0.0,
                             lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
    ae_model.finetune(train_X, 256, 100000, 'sgd', l_rate=0.1, decay=0.0,
                   lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
    ae_model.save('mnist_pt.arg')
    ae_model.load('mnist_pt.arg')
    print "Training error:", ae_model.eval(train_X)
    print "Validation error:", ae_model.eval(val_X)
Ejemplo n.º 6
0
                                                           lambda_u, lv, K)
    fp.write('p%d: lambda_v/lambda_u/ratio/K: %f/%f/%f/%d\n' % \
            (p,lambda_v,lambda_u,lv,K))
    fp.close()
    if is_dummy:
        X = data.get_dummy_mult()
        R = data.read_dummy_user()
    else:
        X = data.get_mult()
        R = data.read_user()
    # set to INFO to see less information during training
    logging.basicConfig(level=logging.DEBUG)
    #ae_model = AutoEncoderModel(mx.gpu(0), [784,500,500,2000,10], pt_dropout=0.2,
    #    internal_act='relu', output_act='relu')
    ae_model = AutoEncoderModel(mx.cpu(2), [X.shape[1], 100, K],
                                pt_dropout=0.2,
                                internal_act='relu',
                                output_act='relu')

    train_X = X

    #ae_model.layerwise_pretrain(train_X, 256, 50000, 'sgd', l_rate=0.1, decay=0.0,
    #                         lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
    #V = np.zeros((train_X.shape[0],10))
    V = np.random.rand(train_X.shape[0], K) / 10
    lambda_v_rt = np.ones((train_X.shape[0], K)) * sqrt(lv)
    U, V, theta, BCD_loss = ae_model.finetune(
        train_X,
        R,
        V,
        lambda_v_rt,
        lambda_u,
Ejemplo n.º 7
0
                    help='the number of hidden units for the layers of the encoder.' \
                         'The decoder layers are created in the reverse order.')

# set to INFO to see less information during training
logging.basicConfig(level=logging.DEBUG)
opt = parser.parse_args()
logging.info(opt)
print_every = opt.print_every
batch_size = opt.batch_size
pretrain_num_iter = opt.pretrain_num_iter
finetune_num_iter = opt.finetune_num_iter
visualize = opt.visualize
layers = [int(i) for i in opt.num_units.split(',')]

if __name__ == '__main__':
    ae_model = AutoEncoderModel(mx.cpu(0), layers, pt_dropout=0.2,
        internal_act='relu', output_act='relu')

    X, _ = data.get_mnist()
    train_X = X[:60000]
    val_X = X[60000:]

    ae_model.layerwise_pretrain(train_X, batch_size, pretrain_num_iter, 'sgd', l_rate=0.1,
                                decay=0.0, lr_scheduler=mx.misc.FactorScheduler(20000,0.1),
                                print_every=print_every)
    ae_model.finetune(train_X, batch_size, finetune_num_iter, 'sgd', l_rate=0.1, decay=0.0,
                      lr_scheduler=mx.misc.FactorScheduler(20000,0.1), print_every=print_every)
    ae_model.save('mnist_pt.arg')
    ae_model.load('mnist_pt.arg')
    print("Training error:", ae_model.eval(train_X))
    print("Validation error:", ae_model.eval(val_X))
    if visualize:
Ejemplo n.º 8
0
    def setup(self, X, num_centers, alpha, znum, save_to='dec_model'):
        self.sep = int(X.shape[0] * 0.75)
        X_train = X[:self.sep]
        X_val = X[self.sep:]
        batch_size = 32  # 160 32*5 = update_interval*5
        # Train or Read autoencoder: note is not dependent on number of clusters just on z latent size
        ae_model = AutoEncoderModel(self.xpu,
                                    [X.shape[1], 500, 500, 2000, znum],
                                    pt_dropout=0.2)
        if not os.path.exists(save_to + '_pt.arg'):
            ae_model.layerwise_pretrain(X_train,
                                        batch_size,
                                        50000,
                                        'sgd',
                                        l_rate=0.1,
                                        decay=0.0,
                                        lr_scheduler=mx.misc.FactorScheduler(
                                            20000, 0.1))
            ae_model.finetune(X_train,
                              batch_size,
                              100000,
                              'sgd',
                              l_rate=0.1,
                              decay=0.0,
                              lr_scheduler=mx.misc.FactorScheduler(20000, 0.1))
            ae_model.save(save_to + '_pt.arg')
            logging.log(
                logging.INFO,
                "Autoencoder Training error: %f" % ae_model.eval(X_train))
            logging.log(
                logging.INFO,
                "Autoencoder Validation error: %f" % ae_model.eval(X_val))
        else:
            ae_model.load(save_to + '_pt.arg')
            logging.log(
                logging.INFO,
                "Reading Autoencoder from file..: %s" % (save_to + '_pt.arg'))
            logging.log(
                logging.INFO,
                "Autoencoder Training error: %f" % ae_model.eval(X_train))
            logging.log(
                logging.INFO,
                "Autoencoder Validation error: %f" % ae_model.eval(X_val))

        self.ae_model = ae_model
        logging.log(logging.INFO, "finished reading Autoencoder from file..: ")
        # prep model for clustering
        self.dec_op = DECModel.DECLoss(num_centers, alpha)
        label = mx.sym.Variable('label')
        self.feature = self.ae_model.encoder
        self.loss = self.dec_op(data=self.ae_model.encoder,
                                label=label,
                                name='dec')
        self.args.update({
            k: v
            for k, v in self.ae_model.args.items()
            if k in self.ae_model.encoder.list_arguments()
        })
        self.args['dec_mu'] = mx.nd.empty(
            (num_centers, self.ae_model.dims[-1]), ctx=self.xpu)
        self.args_grad.update({
            k: mx.nd.empty(v.shape, ctx=self.xpu)
            for k, v in self.args.items()
        })
        self.args_mult.update(
            {k: k.endswith('bias') and 2.0 or 1.0
             for k in self.args})
        self.num_centers = num_centers
        self.znum = znum
        self.batch_size = batch_size
        self.G = self.ae_model.eval(X_train) / self.ae_model.eval(X_val)
    def setup(self, X, num_centers, alpha, save_to='dec_model'):
        sep = X.shape[0] * 9 / 10
        X_train = X[:sep]
        X_val = X[sep:]
        ae_model = AutoEncoderModel(self.xpu, [X.shape[1], 500, 500, 2000, 10],
                                    pt_dropout=0.2)
        if not os.path.exists(save_to + '_pt.arg'):
            ae_model.layerwise_pretrain(X_train,
                                        256,
                                        50000,
                                        'sgd',
                                        l_rate=0.1,
                                        decay=0.0,
                                        lr_scheduler=mx.misc.FactorScheduler(
                                            20000, 0.1))
            ae_model.finetune(X_train,
                              256,
                              100000,
                              'sgd',
                              l_rate=0.1,
                              decay=0.0,
                              lr_scheduler=mx.misc.FactorScheduler(20000, 0.1))
            ae_model.save(save_to + '_pt.arg')
            logging.log(
                logging.INFO,
                "Autoencoder Training error: %f" % ae_model.eval(X_train))
            logging.log(
                logging.INFO,
                "Autoencoder Validation error: %f" % ae_model.eval(X_val))
        else:
            ae_model.load(save_to + '_pt.arg')
        self.ae_model = ae_model

        self.dec_op = DECModel.DECLoss(num_centers, alpha)
        label = mx.sym.Variable('label')
        self.feature = self.ae_model.encoder
        self.loss = self.dec_op(data=self.ae_model.encoder,
                                label=label,
                                name='dec')
        self.args.update({
            k: v
            for k, v in self.ae_model.args.items()
            if k in self.ae_model.encoder.list_arguments()
        })
        self.args['dec_mu'] = mx.nd.empty(
            (num_centers, self.ae_model.dims[-1]), ctx=self.xpu)
        self.args_grad.update({
            k: mx.nd.empty(v.shape, ctx=self.xpu)
            for k, v in self.args.items()
        })
        self.args_mult.update(
            {k: k.endswith('bias') and 2.0 or 1.0
             for k in self.args})
        self.num_centers = num_centers
Ejemplo n.º 10
0
Archivo: cdl.py Proyecto: pjpan/R
    fp.write('p%d: lambda_v/lambda_u/ratio/K: %f/%f/%f/%d\n' % \
            (p,lambda_v,lambda_u,lv,K))
    fp.close()
    if is_dummy:
        X = data.get_dummy_mult()
        R = data.read_dummy_user()
    else:
        X = data.get_mult()
        R = data.read_user()
    # set to INFO to see less information during training
    logging.basicConfig(level=logging.DEBUG)
    #ae_model = AutoEncoderModel(mx.gpu(0), [784,500,500,2000,10], pt_dropout=0.2,
    #    internal_act='relu', output_act='relu')

    #mx.cpu() no param needed for cpu.
    ae_model = AutoEncoderModel(mx.cpu(), [X.shape[1],100,K],
        pt_dropout=0.2, internal_act='relu', output_act='relu')

    train_X = X

    #ae_model.layerwise_pretrain(train_X, 256, 50000, 'sgd', l_rate=0.1, decay=0.0,
    #                         lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
    #V = np.zeros((train_X.shape[0],10))
    V = np.random.rand(train_X.shape[0],K)/10
    lambda_v_rt = np.ones((train_X.shape[0],K))*sqrt(lv)
    U, V, theta, BCD_loss = ae_model.finetune(train_X, R, V, lambda_v_rt, lambda_u,
            lambda_v, dir_save, batch_size,
            num_iter, 'sgd', l_rate=0.1, decay=0.0,
            lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
    #ae_model.save('cdl_pt.arg')
    np.savetxt(dir_save+'/final-U.dat',U,fmt='%.5f',comments='')
    np.savetxt(dir_save+'/final-V.dat',V,fmt='%.5f',comments='')
Ejemplo n.º 11
0
import mxnet as mx
import numpy as np
import logging
from autoencoder import AutoEncoderModel


if __name__ == '__main__':
    # set to INFO to see less information during training
    logging.basicConfig(level=logging.DEBUG)
    ae_model = AutoEncoderModel(mx.gpu(0), [784,500,500,2000,10], pt_dropout=0.2,
        internal_act='relu', output_act='relu')

    X=np.random.rand(100000,784)
    train_X = X[:60000]
    val_X = X[60000:]

    ae_model.layerwise_pretrain(train_X, 256, 50000, 'sgd', l_rate=0.1, decay=0.0,
                             lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
    ae_model.finetune(train_X, 256, 100000, 'sgd', l_rate=0.1, decay=0.0,
                   lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
    ae_model.save('mnist_pt.arg')
    ae_model.load('mnist_pt.arg')
    print "Training error:", ae_model.eval(train_X)
    print "Validation error:", ae_model.eval(val_X)
Ejemplo n.º 12
0
                    help='the number of hidden units for the layers of the encoder.' \
                         'The decoder layers are created in the reverse order.')

# set to INFO to see less information during training
logging.basicConfig(level=logging.DEBUG)
opt = parser.parse_args()
logging.info(opt)
print_every = opt.print_every
batch_size = opt.batch_size
pretrain_num_iter = opt.pretrain_num_iter
finetune_num_iter = opt.finetune_num_iter
visualize = opt.visualize
layers = [int(i) for i in opt.num_units.split(',')]

if __name__ == '__main__':
    ae_model = AutoEncoderModel(mx.cpu(0), layers, pt_dropout=0.2,
        internal_act='relu', output_act='relu')

    X, _ = data.get_mnist()
    train_X = X[:60000]
    val_X = X[60000:]

    ae_model.layerwise_pretrain(train_X, batch_size, pretrain_num_iter, 'sgd', l_rate=0.1,
                                decay=0.0, lr_scheduler=mx.misc.FactorScheduler(20000,0.1),
                                print_every=print_every)
    ae_model.finetune(train_X, batch_size, finetune_num_iter, 'sgd', l_rate=0.1, decay=0.0,
                      lr_scheduler=mx.misc.FactorScheduler(20000,0.1), print_every=print_every)
    ae_model.save('mnist_pt.arg')
    ae_model.load('mnist_pt.arg')
    print("Training error:", ae_model.eval(train_X))
    print("Validation error:", ae_model.eval(val_X))
    if visualize:
class GAN():
    def __init__(self, mini_batch_size):
        self.image_shape = (16, 128, 128, 3)
        learning_rate = 0.03
        opt = keras.optimizers.Adam(lr=learning_rate)
        opt1 = keras.optimizers.Adam(lr=learning_rate)
        opt_slow = keras.optimizers.Adam(lr=1)
        #Build and compile the discriminator
        self.discriminator = create_discriminator_model()
        self.discriminator.compile(loss='binary_crossentropy',
                                   optimizer=opt,
                                   metrics=[
                                       'accuracy',
                                       tf.keras.metrics.TruePositives(),
                                       tf.keras.metrics.FalsePositives(),
                                       tf.keras.metrics.TrueNegatives(),
                                       tf.keras.metrics.FalseNegatives()
                                   ])
        #Build and compile the generator
        self.generator = AutoEncoderModel()
        self.generator.compile(loss='mse', optimizer=opt_slow)

        #the generator takes a video as input and generates a modified video
        z = Input(shape=(self.image_shape))
        img = self.generator(z)
        self.discriminator.trainable = False
        validity = self.discriminator(img)
        self.combined = Model(z, validity)
        self.combined.compile(loss='binary_crossentropy',
                              optimizer=opt1,
                              metrics=['accuracy'])
        self.dir_path = '/kaggle/input/ucf-crime-training-subset/tfrecords2/'
        self.ds = build_dataset(self.dir_path,
                                batch_size=mini_batch_size,
                                file_buffer=512 * 1024)

    def train(self, epochs, mini_batch_size):
        #this function will need to be added later
        tf.summary.trace_off()
        for epoch in range(epochs):
            d_loss_sum = tf.zeros(6)
            reconstruct_error_sum = 0
            g_loss_sum = tf.zeros(2)
            no_of_minibatches = 0
            for minibatch, labels in self.ds:
                # ---------------------
                #  Train Discriminator
                # ---------------------
                #normalize inputs
                no_of_minibatches += 1
                minibatch = tf.cast(tf.math.divide(minibatch, 255), tf.float32)
                gen_vids = self.generator.predict(minibatch)
                #might have to combine these to improve batch norm
                d_loss_real = self.discriminator.train_on_batch(
                    minibatch, tf.ones((mini_batch_size, 1)))
                d_loss_fake = self.discriminator.train_on_batch(
                    gen_vids, tf.zeros((mini_batch_size, 1)))
                d_loss = 0.5 * tf.math.add(d_loss_real, d_loss_fake)
                # ---------------------
                #  Train Generator
                # ---------------------
                # The generator wants the discriminator to label the generated samples as valid (ones)
                valid_y = tf.ones((mini_batch_size, 1))
                # Train the generator
                g_loss = self.combined.train_on_batch(minibatch, valid_y)
                reconstruct_error = self.generator.train_on_batch(
                    minibatch, minibatch)
                d_loss_sum += d_loss
                g_loss_sum += g_loss
                reconstruct_error_sum += reconstruct_error
            print(no_of_minibatches)
            self.combined.save_weights('/kaggle/working/weights_epoch%d' %
                                       (epoch))
            g_loss = g_loss_sum / no_of_minibatches
            d_loss = d_loss_sum / no_of_minibatches
            reconstruct_error = reconstruct_error_sum / no_of_minibatches
            # Plot the progress
            print(
                "%d [D loss: %f, acc.: %.2f%%] [G loss: %f, accuracy %.2f%% from which %f is combined loss and %f is reconstruction loss]"
                % (epoch, d_loss[0], 100 * d_loss[1],
                   g_loss[0] + reconstruct_error, g_loss[1] * 100, g_loss[0],
                   reconstruct_error))
        tf.summary.trace_on()

    def test(self, dev_set_path, mini_batch_size):
        dev_set = build_test_dataset(dev_set_path,
                                     batch_size=mini_batch_size,
                                     file_buffer=500 * 1024)
        no_of_minibatches = 0
        ans_final = tf.zeros(6)
        for minibatch, labels in dev_set:
            no_of_minibatches += 1
            ans = self.discriminator.test_on_batch(minibatch, (labels == 0),
                                                   reset_metrics=False)
            ans_final = ans
        print(no_of_minibatches, ans_final[0], ans_final[1], ans_final[2],
              ans_final[3], ans_final[4], ans_final[5])

    def test_real_vs_fake(self, dev_set_path, mini_batch_size):
        dev_set = build_dataset(dev_set_path,
                                batch_size=mini_batch_size,
                                file_buffer=500 * 1024)
        ans_final = tf.zeros(6)
        no_of_minibatches = 0
        for minibatch, labels in dev_set:
            no_of_minibatches += 1
            ans = self.discriminator.test_on_batch(minibatch,
                                                   labels,
                                                   reset_metrics=False)
            fake_vals = np.random.random((mini_batch_size, 16, 128, 128, 3))
            ans = self.discriminator.test_on_batch(fake_vals,
                                                   tf.zeros(
                                                       (mini_batch_size, 1)),
                                                   reset_metrics=False)
            ans_final = ans
        print(no_of_minibatches, ans_final[0], ans_final[1], ans_final[2],
              ans_final[3], ans_final[4], ans_final[5])

    def visualise_autoencoder_outputs(self, no_of_minibatches):
        fourcc = VideoWriter_fourcc(
            *'MP42')  #some code required for VideoWriter
        video = VideoWriter('/kaggle/working/reconstructed_video.avi', fourcc,
                            float(24),
                            (128, 128))  #creates video to store 1st segment
        for i in range(no_of_minibatches):
            inp = np.load(
                "../input/tom-and-jerry-clips/minibatches/minibatch%d.npz" %
                (i))
            inp = inp['arr_0']
            gen_vids = self.generator.predict(inp)
            gen_vids *= 255
            for j in range(16):
                for k in range(16):
                    frame = np.uint8(gen_vids[j][k])
                    video.write(frame)
Ejemplo n.º 14
0
import mxnet as mx
import sys
sys.path.append("../")
sys.path.append("../../autoencoder")
import logging
import numpy as np
from autoencoder import AutoEncoderModel
#from visualize import visualize
from data_all import news_iterator
#ae_model = AutoEncoderModel(mx.gpu(0), [5000,100], pt_dropout=0.5) 
ae_model = AutoEncoderModel(mx.gpu(3),[5000,100],internal_act='sigmoid', output_act='sigmoid', sparseness_penalty=1e-4, pt_dropout=0)
logging.basicConfig(level=logging.DEBUG) 
#ae_model.load('../../autoencoder/news_20classes_small.arg')#classes_small.arg') #news_20_ltest.arg
ae_model.load('news_20classes_small_1e-4_non-neg.arg')
batch_size = 100

fea_sym = ae_model.loss.get_internals()#[3]
logging.info(fea_sym.list_outputs())
output=fea_sym['sparse_encoder_0_output']
fc3 = mx.symbol.FullyConnected(data=output, num_hidden=20)
softmax = mx.symbol.SoftmaxOutput(data=fc3, name='softmax')
#logging.info(softmax.list_arguments())

args=ae_model.args
datashape=(100,5000)
train, val, _ = news_iterator(input_size = 5000,batchsize=100)

#fc = softmax.get_internals()
#logging.info(fc.list_arguments())
args_shape,ow,aw = softmax.get_internals().infer_shape(data=datashape)
#logging.info(args_shape)
Ejemplo n.º 15
0
layers = str(size) + ",500,500,2000,10"
layers = [int(i) for i in layers.split(',')]

xpu = mx.gpu() if gpu else mx.cpu()
print("Training on {}".format("GPU" if gpu else "CPU"))

if (gpu):
    hyperparameters['device'] = 'GPU'
else:
    hyperparameters['device'] = 'CPU'

ae_model = AutoEncoderModel(xpu,
                            layers,
                            eval_batch_size,
                            hyperparameters=hyperparameters,
                            pt_dropout=0.2,
                            internal_act='relu',
                            output_act='relu')

split_point = int(len(X) * 0.75)
train_X = X[:split_point]
val_X = X[split_point:]

ae_model.layerwise_pretrain(train_X,
                            batch_size,
                            pretrain_num_iter,
                            'sgd',
                            l_rate=0.001,
                            decay=0.0,
                            lr_scheduler=mx.lr_scheduler.FactorScheduler(
Ejemplo n.º 16
0
# pylint: skip-file
import mxnet as mx
import numpy as np
import logging
import mnist_data as data
from math import sqrt
from autoencoder import AutoEncoderModel

if __name__ == '__main__':
    lv = 1e-2# lv/ln in CDL
    # set to INFO to see less information during training
    logging.basicConfig(level=logging.DEBUG)
    #ae_model = AutoEncoderModel(mx.gpu(0), [784,500,500,2000,10], pt_dropout=0.2,
    #    internal_act='relu', output_act='relu')
    ae_model = AutoEncoderModel(mx.cpu(2), [784,500,500,2000,10], pt_dropout=0.2,
        internal_act='relu', output_act='relu')

    X, _ = data.get_mnist()
    train_X = X[:60000]
    val_X = X[60000:]

    #ae_model.layerwise_pretrain(train_X, 256, 50000, 'sgd', l_rate=0.1, decay=0.0,
    #                         lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
    #V = np.zeros((train_X.shape[0],10))
    V = np.random.rand(train_X.shape[0],10)/10
    lambda_v_rt = np.ones((train_X.shape[0],10))*sqrt(lv)
    ae_model.finetune(train_X, V, lambda_v_rt, 256,
            20, 'sgd', l_rate=0.1, decay=0.0,
            lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
    ae_model.save('mnist_pt.arg')
    ae_model.load('mnist_pt.arg')
Ejemplo n.º 17
0
    def setup(self, X, num_centers, alpha, save_to='dec_model'):
        sep = X.shape[0]*9/10
        X_train = X[:sep]
        X_val = X[sep:]
        ae_model = AutoEncoderModel(self.xpu, [X.shape[1],500,500,2000,10], pt_dropout=0.2)
        if not os.path.exists(save_to+'_pt.arg'):
            ae_model.layerwise_pretrain(X_train, 256, 50000, 'sgd', l_rate=0.1, decay=0.0,
                                        lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
            ae_model.finetune(X_train, 256, 100000, 'sgd', l_rate=0.1, decay=0.0,
                              lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
            ae_model.save(save_to+'_pt.arg')
            logging.log(logging.INFO, "Autoencoder Training error: %f"%ae_model.eval(X_train))
            logging.log(logging.INFO, "Autoencoder Validation error: %f"%ae_model.eval(X_val))
        else:
            ae_model.load(save_to+'_pt.arg')
        self.ae_model = ae_model

        self.dec_op = DECModel.DECLoss(num_centers, alpha)
        label = mx.sym.Variable('label')
        self.feature = self.ae_model.encoder
        self.loss = self.dec_op(data=self.ae_model.encoder, label=label, name='dec')
        self.args.update({k:v for k,v in self.ae_model.args.items() if k in self.ae_model.encoder.list_arguments()})
        self.args['dec_mu'] = mx.nd.empty((num_centers, self.ae_model.dims[-1]), ctx=self.xpu)
        self.args_grad.update({k: mx.nd.empty(v.shape, ctx=self.xpu) for k,v in self.args.items()})
        self.args_mult.update({k: k.endswith('bias') and 2.0 or 1.0 for k in self.args})
        self.num_centers = num_centers
Ejemplo n.º 18
0
    gpus = [mx.gpu(i) for i in range(num_gpu)]
    dev=mx.gpu()

#    train, val, voc = news_iterator(voc_count, 100)
    vocfile = "/home/tingyubi/20w/data/tfidf_extraction-1_26.voc"
    with open(vocfile,'r') as f:
        voc = f.read().decode('utf-8').split(" ")
    """
    data = mx.symbol.Variable('data')
    fc1 = mx.symbol.FullyConnected(name='encoder_%d'%istack, data=data, num_hidden=20)
    sig1 = mx.symbol.Activation(data=fc1, act_type='sigmoid')
    sparse1 = mx.symbol.SparseReg(data=sig1, penalty=1e-3, sparseness_target=0.1)
    fc2 = mx.symbol.FullyConnected(name='decoder_%d'%istack, data=sparse1, num_hidden=5000)
    loss = mx.symbol.LinearRegressionOutput(data=fc2, name='softmax')
"""
    aem = AutoEncoderModel(mx.gpu(3),[voc_count,1000,200],internal_act='sigmoid', output_act='sigmoid', sparseness_penalty=1e-4, pt_dropout=0)
    aem.load('/home/tingyubi/20w/autoencoder/20w_1000_200_1e-4_non-neg.arg')
    #print aem.loss.get_internals().list_outputs()
    fc2 = aem.encoder
    print aem.loss.get_internals().list_outputs()
    #print aem.loss.get_internals().list_arguments()
    model = aem
    #print model.arg_params['encoder_0_weight'].shape

    """
    for k in range(2):
        words_index=np.argsort(model.args['encoder_0_weight'].asnumpy()[k,:])
        logging.info('Topic %d' % k)
        logging.info([voc[i] for i in words_index[-20:]])
    """