def __init__(self, model, alpha=0.5, epsilon=0.5):
        """
        :param NeuralNetwork model: an instantiated instance of a NeuralNetwork
        :param float alpha: the constant by which to weight the adversarial objective
        :param float epsilon: the size of the perturbation used to make adversarial examples
        """

        #===============================================================================
        # make sure that the NeuralNetwork passed in has all the methods and fields that
        # it will need
        required_class_methods = set([
            "train_model", "cost", "embedded_input", "build_architecture",
            "compute_gradients", "classify_batch"
        ])
        assert len(required_class_methods - set(dir(model))) == 0

        util.colorprint("building adversarial cost...", 'red')

        #===============================================================================
        # modifies the model's cost function to include an adversarial term
        self.make_cost_adversarial(model, alpha, epsilon)

        #===============================================================================
        # Now that we have modified the cost, recompute the gradients
        model.compute_gradients()

        self.model = model
    def __init__(self, model, alpha=0.5, epsilon=0.5):
        """
        :param NeuralNetwork model: an instantiated instance of a NeuralNetwork
        :param float alpha: the constant by which to weight the adversarial objective
        :param float epsilon: the size of the perturbation used to make adversarial examples
        """


        #===============================================================================
        # make sure that the NeuralNetwork passed in has all the methods and fields that
        # it will need
        required_class_methods = set(["train_model", "cost", "embedded_input", "build_architecture", "compute_gradients", "classify_batch"])            
        assert len(required_class_methods - set(dir(model))) == 0   


        util.colorprint("building adversarial cost...", 'red')
        
        #===============================================================================
        # modifies the model's cost function to include an adversarial term                 
        self.make_cost_adversarial(model, alpha, epsilon)

        #===============================================================================
        # Now that we have modified the cost, recompute the gradients
        model.compute_gradients()

        self.model = model
Exemple #3
0
def pullAll():
    global bakingMan
    colorprint("pullAll", 33)
    result = bakingMan.getAllJobs()
    # print(result)
    jsonResult = json.dumps(result,
                            sort_keys=True,
                            indent=4,
                            separators=(',', ': '))
    response.content_type = "application/json"
    return jsonResult
Exemple #4
0
    def train_model(self, data_fname, 
                        epochs=1,
                        batch_size=16, 
                        max_epochs=500,
                        saveFreq=1110,
                        dispFreq=500, 
                        ):
        """
        :param function get_data_reader: a function that returns a generator which yields a tuple of vector, label.
            I'm sorry.
        :param int epochs: the number of full sweeps through the data to perform
        :param dict **kwargs: any other key-word arguments to be passed in to the 
        """
        update_index = 0

        for epoch in range(epochs):
            print "epoch %s..."%epoch
            # batch_reader = copy.copy(data_reader)

            #note that get_data_reader() can choose to shuffle the data
            batch_reader = self.model.get_data_reader(data_fname)

            #===========================================================================
            # let's do a batch update! 
            while True: 
                update_index += 1
                #===============================================================================
                # normal update
                batch_x, batch_y, success = self._get_minibatch(batch_reader, batch_size)
                # batch_x = self._prepend_intercept(batch_x)
                if not success: break;


                cur_cost_val = self.model.update_params(batch_x, batch_y, scale=self.alpha)

                if np.isnan(cur_cost_val) or np.isinf(cur_cost_val):
                    util.colorprint('something screwy happened on update %s, iteration %s: cur_cost_val = %s'%(update_index, epoch, cur_cost_val), "red")
                    return None

                #===============================================================================
                # adversarial update
                sub_batch_x, sub_batch_y = self.subsample_batch(batch_x, batch_y, self.alpha)

                adv_batch_x = self.get_adversarial_examples(batch_x, batch_y)   
                cur_adv_cost_val = self.model.update_params(adv_batch_x, batch_y, scale=1.0-self.alpha) 


                #===============================================================================
                #                 
                if np.mod(update_index, dispFreq) == 0:
                    print 'Epoch ', epoch, 'Update ', update_index, 'Cost ', cur_cost_val, 'adv. Cost ', cur_adv_cost_val
Exemple #5
0
def pullState(jobId: str):
    global bakingMan
    colorprint("pullState id {}".format(jobId), 33)

    result = {"state": "undefined"}
    if bakingMan.hasJob(jobId):
        result = bakingMan.getJob(jobId)

    # print(result)
    jsonResult = json.dumps(result,
                            sort_keys=True,
                            indent=4,
                            separators=(',', ': '))
    response.content_type = "application/json"
    return jsonResult
Exemple #6
0
def bakeDirect():
    global bakingMan
    # print(request)
    # print(request.POST)
    # print(request.POST.__dict__)
    # print(request.headers.__dict__)
    # print(request.method)

    jobSource = extractPostParams(request)

    igxcString = jobSource["igxcContent"]
    # print(igxcString)
    if not igxcString or igxcString == "null":
        colorprint("No igxcContent found in POST request in bakeDirect/", 31)
        return {"error": "No igxcContent found in POST request in bakeDirect/"}

    try:
        if isinstance(igxcString, str):
            igxcContent = json.loads(igxcString)
        else:
            igxcContent = igxcString
    except Exception as e:
        colorprint("Exception in bakeDirect/", 31)
        print(e)
        return {"error": "igxcContent couldn't be parsed"}
    # print(igxcContent)

    basePath = jobSource["basePath"]
    # print(basepath)

    resolutionValue = aoConfig["resolution"]
    resolutionParam = jobSource["resolution"]
    if resolutionParam is not None:
        resolutionValue = int(resolutionParam)

    args = {
        "basePath": basePath,
        "igxcContent": igxcContent,
        "resolution": resolutionValue
    }
    # print(args)

    jobId = bakingMan.addJob(args)
    response.content_type = "application/json"
    return {"jobId": jobId}
    def classify_batch(self, data_fname):
        data_reader = self.get_data_reader(data_fname)
        y_pred = []
        y_true = []
        for example_vec, label in data_reader:
            # example_vec = self._prepend_intercept(example_vec)
            y_true.append(label)
            pred = self.f_pred(example_vec)[0]
            y_pred.append(pred)

        #===============================================================================
        # below is some kick-arse debugging tools
        if len(set(y_pred)) == 1:
            util.colorprint("Warning: all predictions are of value %s"%y_pred[0], "flashing")
            util.colorprint("Here are the hidden layer activations for the lasst training example:", "flashing")
            self.print_sample_activations(example_vec)

        return y_true, y_pred
Exemple #8
0
    def run(self):
        self.running = True
        while self.running:
            if not self.isProcessRunning:
                if len(self.queue) > 0:
                    BakingMan.isProcessRunning = True
                    self.currentJob = self.queue.pop(0)
                    try:
                        self.runJob()
                    except FileNotFoundError as e:
                        BakingMan.isProcessRunning = False
                        colorprint(
                            "File not found for jobId {}".format(
                                self.currentJob.jobId), 31)
                        self.currentJob = None
                        print(e)
                    except json.decoder.JSONDecodeError as e:
                        BakingMan.isProcessRunning = False
                        colorprint(
                            "JSON not valid for jobId {}".format(
                                self.currentJob.jobId), 31)
                        self.currentJob = None
                        print(e)
                    except Exception as e:
                        BakingMan.isProcessRunning = False
                        colorprint(
                            "Exception for jobId {}".format(
                                self.currentJob.jobId), 31)
                        self.currentJob = None
                        print(e)

                else:
                    sleep(0.5)
    def compute_gradients(self):
        # maybe doesn't need to be a class variable
        self.grads = T.grad(self.cost, wrt=self.tparams.values())

        #lrate: learning rate
        self.f_populate_gradients, self.f_update_params = self.optimizer()


        # =====================================================================
        # print out the computational graph and make an image of it too
        if self.debug and False:
            # util.colorprint("Following is the graph of the final hidden layer:", "blue")
            # final_activation_fn = theano.function([self.input], final_activation)
            # theano.printing.debugprint(final_activation_fn.maker.fgraph.outputs[0])   
            # util.colorprint("Also, saving png of computational graph:", "blue")
            # theano.printing.pydotprint(final_activation_fn, 
            #     outfile="output/lmlp_final_act_viz.png", 
            #     compact=True,
            #     scan_graphs=True,
            #     var_with_name_simple=True)
            util.colorprint("Following is the graph of the first of the derivatives:", "blue")
            final_grad_fn = theano.function([self.input, self.y], self.grads[0])
            theano.printing.debugprint(final_grad_fn.maker.fgraph.outputs[0]) 
            util.colorprint("Yay colorprinted:", "blue")
            print theano.pp(self.final_activation)
            util.colorprint("Also, saving png of computational graph:", "blue")
            theano.printing.pydotprint(final_grad_fn, 
                outfile="output/lmlp_final_grad_viz.png", 
                compact=True,
                scan_graphs=True,
                var_with_name_simple=True)            
Exemple #10
0
    def runJob(self):
        colorprint(
            "Starting runJob with jobId {}".format(self.currentJob.jobId), 32)

        self.currentJob = self.currentJob._replace(state="running")
        output = startWithDirectArgs(self.currentJob.jobArgs)

        result = {}
        if "error" in output and output["error"] is not None:
            result = {
                "jobId": self.currentJob.jobId,
                "jobArgs": self.currentJob.jobArgs,
                "state": "error",
                "error": output["error"]
            }
            colorprint("Error in startWithDirectArgs", 31)

        else:
            result = {
                "jobId": self.currentJob.jobId,
                "jobArgs": self.currentJob.jobArgs,
                "urlAoMapImage": output["urlAoMapImage"],
                "urlAoMappingJson": output["urlAoMappingJson"],
                "urlIgxcModified": output["urlIgxcModified"],
                "urlIgxcOriginal": output["urlIgxcOriginal"],
                "transforms": output["transforms"],
                "state": "finished",
                "igxcModified": output["igxcModified"]
            }
            colorprint(
                "Finished runJob with jobId {}".format(self.currentJob.jobId),
                32)

        self.results.append(result)
        self.isProcessRunning = False
        self.currentJob = None
        return result
Exemple #11
0
def getFile(filename):
    colorprint("getFile " + filename, 33)
    return staticFileWithCors(filename, './out/', download=filename)
Exemple #12
0
    def train_lstm(self,
        saveto, # The best model will be saved there
        dataset,

        #----------------------------------------------------------------------
        #algorithmic hyperparameters
        encoder='lstm',  # TODO: can be removed must be lstm.
        l2_reg_U=0.,  # Weight decay for the classifier applied to the U weights.
        lrate=0.0001,  # Learning rate for sgd (not used for adadelta and rmsprop)
        optimizer="adadelta",  # sgd, adadelta and rmsprop available, sgd very hard to use, not recommanded (probably need momentum and decaying learning rate).
        batch_size=16,  # The batch size during training.
        wemb_init='word2vec',

        #----------------------------------------------------------------------
        #parameters related to convergence, saving, and similar
        max_epochs=5000,  # The maximum number of epoch to run
        patience=10,  # Number of epoch to wait before early stop if no progress
        dispFreq=10,  # Display to stdout the training progress every N updates
        n_words=10000,  # Vocabulary size
        validFreq=370,  # Compute the validation error after this number of update.
        saveFreq=1110,  # Save the parameters after every saveFreq updates
        valid_batch_size=64,  # The batch size used for validation/test set.

        #----------------------------------------------------------------------
        # Parameter for extra option (whatever that means)
        noise_std=0.,
        use_dropout=True,  # if False slightly faster, but worst test error
                           # This frequently need a bigger model.
        reload_model=None,  # Path to a saved model we want to start from.
        return_after_reloading=False,  # Path to a saved model we want to start from.
        test_size=-1,  # If >0, we keep only this number of test example.
    ):

        optimizer = OPTIMIZERS[optimizer]
        # Model options
        self.model_options = locals().copy()


        if reload_model:
            self.faulty_load_params(reload_model)
            # self.init_tparams()
            _, self.wdim = self.params['Wemb'].shape
            self.hdim, ydim = self.params['U'].shape

            self.model_options['ydim'] = ydim
            print _, self.wdim, self.hdim, ydim


        self.model_options['hdim'] = self.hdim
        self.model_options['wdim'] = self.wdim

        self.model_options['grad_clip_thresh'] = self.grad_clip_thresh
        print "model options", self.model_options

        # load_data, prepare_data = get_dataset(dataset)

        print 'Loading data'
        #each of the below is a tuple of
        # (list of sentences, where each is a list fo word indices,
        #  list of integer labels)
        if not reload_model:
            train, valid, test =  load_data(n_words=n_words, valid_portion=0.05,
                                           maxlen=self.maxlen, path=dataset)

            if test_size > 0:
                # The test set is sorted by size, but we want to keep random
                # size example.  So we must select a random selection of the
                # examples.
                idx = numpy.arange(len(test[0]))
                numpy.random.shuffle(idx)
                idx = idx[:test_size]
                test = ([test[0][n] for n in idx], [test[1][n] for n in idx])

            ydim = numpy.max(train[1]) + 1

            self.model_options['ydim'] = ydim

        print 'Building model'

        if not reload_model:
            # initialize the word embedding matrix and the parameters of the model (U and b) randomly
            # self.params is a dict mapping name (string) -> numpy ndarray
            self.init_params(self.model_options)

        # This creates Theano Shared Variable from the parameters.
        # Dict name (string) -> Theano Tensor Shared Variable
        # self.params and self.tparams have different copy of the weights.
        self.init_tparams()

        # use_noise is for dropout
        (use_noise, x, mask, y) =\
             self.build_model(self.model_options,)
         # f_pred_prob, self.f_pred, cost)


        if l2_reg_U > 0.:
            l2_reg_U = theano.shared(numpy_floatX(l2_reg_U), name='l2_reg_U')
            weight_decay = 0.
            weight_decay += (self.tparams['U'] ** 2).sum()
            weight_decay *= l2_reg_U
            self.cost += weight_decay

        f_cost = theano.function([x, mask, y], self.cost, name='f_cost')

        grads = tensor.grad(self.cost, wrt=self.tparams.values())
        f_grad = theano.function([x, mask, y], grads, name='f_grad')

        lr = tensor.scalar(name='lr')
        f_grad_shared, f_update = optimizer(lr, self.tparams, grads,
                                            x, mask, y, self.cost)

        if self.debug:
            util.colorprint("Following is the graph of the shared gradient function (f_grad_shared):", "blue")
            theano.printing.debugprint(f_grad_shared.maker.fgraph.outputs[0])

        if return_after_reloading:
            self.model_has_been_trained = True
            return

        print 'Optimization'

        kf_valid = self.get_minibatches_idx(len(valid[0]), valid_batch_size)
        kf_test = self.get_minibatches_idx(len(test[0]), valid_batch_size)

        print "%d train examples" % len(train[0])
        print "%d valid examples" % len(valid[0])
        print "%d test examples" % len(test[0])

        history_errs = []
        best_p = None
        bad_count = 0

        if validFreq == -1:
            validFreq = len(train[0]) / batch_size
        if saveFreq == -1:
            saveFreq = len(train[0]) / batch_size

        uidx = 0  # the number of update done
        estop = False  # early stop
        start_time = time.time()



        try:
            for epoch in xrange(max_epochs):
                sys.stdout.flush()
                n_samples = 0

                # Get new shuffled index for the training set.
                minibatches = self.get_minibatches_idx(len(train[0]), batch_size, shuffle=True)

                for _, train_index_list in minibatches:
                    uidx += 1
                    use_noise.set_value(1.)

                    # Select the random examples for this minibatch
                    y = [train[1][t] for t in train_index_list]
                    x = [train[0][t]for t in train_index_list]

                    # Get the data in numpy.ndarray format
                    # This swap the axis!
                    # Return something of shape (minibatch maxlen, n samples)
                    x, mask, y = prepare_data(x, y)
                    n_samples += x.shape[1]

                    cur_cost_val = f_grad_shared(x, mask, y)
                    f_update(lrate)

                    if numpy.isnan(cur_cost_val) or numpy.isinf(cur_cost_val):
                        print 'NaN detected'
                        return 1., 1., 1.

                    if numpy.mod(uidx, dispFreq) == 0:
                        print 'Epoch ', epoch, 'Update ', uidx, 'Cost ', cur_cost_val

                    if saveto and numpy.mod(uidx, saveFreq) == 0:
                        print 'Saving...',

                        if best_p is not None:
                            self.params = best_p
                        else:
                            self.params = self.unzip(self.tparams)
                        numpy.savez(saveto, history_errs=history_errs, **self.params)
                        pkl.dump(self.model_options, open('%s.pkl' % saveto, 'wb'), -1)
                        print 'Done'

                    if numpy.mod(uidx, validFreq) == 0:
                        use_noise.set_value(0.)
                        train_err = self.pred_error(self.f_pred, prepare_data, train, minibatches)
                        valid_err = self.pred_error(self.f_pred, prepare_data, valid,
                                               kf_valid)
                        test_err = self.pred_error(self.f_pred, prepare_data, test, kf_test)

                        history_errs.append([valid_err, test_err])

                        if (uidx == 0 or
                            valid_err <= numpy.array(history_errs)[:,
                                                                   0].min()):

                            best_p = self.unzip(self.tparams)
                            bad_counter = 0

                        print ('Train ', train_err, 'Valid ', valid_err,
                               'Test ', test_err)

                        if (len(history_errs) > patience and
                            valid_err >= numpy.array(history_errs)[:-patience,
                                                                   0].min()):
                            bad_counter += 1
                            if bad_counter > patience:
                                print 'Early Stop!'
                                estop = True
                                break

                print 'Seen %d samples' % n_samples

                if estop:
                    break

        except KeyboardInterrupt:
            print "Training interrupted"

        end_time = time.time()
        if best_p is not None:
            self.zipp(best_p, self.tparams)
        else:
            best_p = self.unzip(self.tparams)

        use_noise.set_value(0.)
        kf_train_sorted = self.get_minibatches_idx(len(train[0]), batch_size)
        train_err = self.pred_error(self.f_pred, prepare_data, train, kf_train_sorted)
        valid_err = self.pred_error(self.f_pred, prepare_data, valid, kf_valid)
        test_err = self.pred_error(self.f_pred, prepare_data, test, kf_test)

        print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err
        if saveto:
            numpy.savez(saveto, train_err=train_err,
                        valid_err=valid_err, test_err=test_err,
                        history_errs=history_errs, **best_p)
        print 'The code run for %d epochs, with %f sec/epochs' % (
            (epoch + 1), (end_time - start_time) / (1. * (epoch + 1)))
        print >> sys.stderr, ('Training took %.1fs' %
                              (end_time - start_time))
        self.model_has_been_trained = True
        return train_err, valid_err, test_err
Exemple #13
0
        run_descriptor_numeric = str(hash(run_descriptor)) + "_" + util.random_string_signature(4)

        run_descriptor_numeric += "_%s"%util.time_string()
        if ARGS.ID:
            run_descriptor_numeric = ARGS.ID + "_" + run_descriptor_numeric



    MODEL_SAVETO = 'saved_models/%s.npz'%run_descriptor
    RUN_OUTPUT_FNAME = 'output/adv=%s_%s.out'%(ARGS.ADVERSARIAL, run_descriptor_numeric)


    logfile = None
    if ARGS.REDIRECT_OUTPUT_TO_FILE:
        logfile = open(RUN_OUTPUT_FNAME, 'w')
        print util.colorprint("printing all standard output and error to %s...."%RUN_OUTPUT_FNAME, 'rand')
        sys.stdout = logfile
        sys.stderr = logfile

    print "full name of file: ", run_descriptor

    print MODEL_SAVETO

    # print "ARGS.ADVERSARIAL = %s"%ARGS.ADVERSARIAL
    # print "DATASET = %s"%DATASET
    # print "ARGS.MAX_EPOCHS = %s"%ARGS.MAX_EPOCHS
    # print "in summary: \n%s\n"%run_descriptor

    print "ARGS: "
    print ARGS
    print '-'*80
Exemple #14
0
    def train_lstm(
            self,
            saveto,  # The best model will be saved there
            dataset,

            #----------------------------------------------------------------------
            #algorithmic hyperparameters
            encoder='lstm',  # TODO: can be removed must be lstm.
            l2_reg_U=0.,  # Weight decay for the classifier applied to the U weights.
            lrate=0.0001,  # Learning rate for sgd (not used for adadelta and rmsprop)
            optimizer="adadelta",  # sgd, adadelta and rmsprop available, sgd very hard to use, not recommanded (probably need momentum and decaying learning rate).
            batch_size=16,  # The batch size during training.
            wemb_init='word2vec',

            #----------------------------------------------------------------------
            #parameters related to convergence, saving, and similar
            max_epochs=5000,  # The maximum number of epoch to run
            patience=10,  # Number of epoch to wait before early stop if no progress
            dispFreq=10,  # Display to stdout the training progress every N updates
            n_words=10000,  # Vocabulary size
            validFreq=370,  # Compute the validation error after this number of update.
            saveFreq=1110,  # Save the parameters after every saveFreq updates
            valid_batch_size=64,  # The batch size used for validation/test set.

            #----------------------------------------------------------------------
            # Parameter for extra option (whatever that means)
        noise_std=0.,
            use_dropout=True,  # if False slightly faster, but worst test error
            # This frequently need a bigger model.
        reload_model=None,  # Path to a saved model we want to start from.
            return_after_reloading=False,  # Path to a saved model we want to start from.
            test_size=-1,  # If >0, we keep only this number of test example.
    ):

        optimizer = OPTIMIZERS[optimizer]
        # Model options
        self.model_options = locals().copy()

        if reload_model:
            self.faulty_load_params(reload_model)
            # self.init_tparams()
            _, self.wdim = self.params['Wemb'].shape
            self.hdim, ydim = self.params['U'].shape

            self.model_options['ydim'] = ydim
            print _, self.wdim, self.hdim, ydim

        self.model_options['hdim'] = self.hdim
        self.model_options['wdim'] = self.wdim

        self.model_options['grad_clip_thresh'] = self.grad_clip_thresh
        print "model options", self.model_options

        # load_data, prepare_data = get_dataset(dataset)

        print 'Loading data'
        #each of the below is a tuple of
        # (list of sentences, where each is a list fo word indices,
        #  list of integer labels)
        if not reload_model:
            train, valid, test = load_data(n_words=n_words,
                                           valid_portion=0.05,
                                           maxlen=self.maxlen,
                                           path=dataset)

            if test_size > 0:
                # The test set is sorted by size, but we want to keep random
                # size example.  So we must select a random selection of the
                # examples.
                idx = numpy.arange(len(test[0]))
                numpy.random.shuffle(idx)
                idx = idx[:test_size]
                test = ([test[0][n] for n in idx], [test[1][n] for n in idx])

            ydim = numpy.max(train[1]) + 1

            self.model_options['ydim'] = ydim

        print 'Building model'

        if not reload_model:
            # initialize the word embedding matrix and the parameters of the model (U and b) randomly
            # self.params is a dict mapping name (string) -> numpy ndarray
            self.init_params(self.model_options)

        # This creates Theano Shared Variable from the parameters.
        # Dict name (string) -> Theano Tensor Shared Variable
        # self.params and self.tparams have different copy of the weights.
        self.init_tparams()

        # use_noise is for dropout
        (use_noise, x, mask, y) =\
             self.build_model(self.model_options,)
        # f_pred_prob, self.f_pred, cost)

        if l2_reg_U > 0.:
            l2_reg_U = theano.shared(numpy_floatX(l2_reg_U), name='l2_reg_U')
            weight_decay = 0.
            weight_decay += (self.tparams['U']**2).sum()
            weight_decay *= l2_reg_U
            self.cost += weight_decay

        f_cost = theano.function([x, mask, y], self.cost, name='f_cost')

        grads = tensor.grad(self.cost, wrt=self.tparams.values())
        f_grad = theano.function([x, mask, y], grads, name='f_grad')

        lr = tensor.scalar(name='lr')
        f_grad_shared, f_update = optimizer(lr, self.tparams, grads, x, mask,
                                            y, self.cost)

        if self.debug:
            util.colorprint(
                "Following is the graph of the shared gradient function (f_grad_shared):",
                "blue")
            theano.printing.debugprint(f_grad_shared.maker.fgraph.outputs[0])

        if return_after_reloading:
            self.model_has_been_trained = True
            return

        print 'Optimization'

        kf_valid = self.get_minibatches_idx(len(valid[0]), valid_batch_size)
        kf_test = self.get_minibatches_idx(len(test[0]), valid_batch_size)

        print "%d train examples" % len(train[0])
        print "%d valid examples" % len(valid[0])
        print "%d test examples" % len(test[0])

        history_errs = []
        best_p = None
        bad_count = 0

        if validFreq == -1:
            validFreq = len(train[0]) / batch_size
        if saveFreq == -1:
            saveFreq = len(train[0]) / batch_size

        uidx = 0  # the number of update done
        estop = False  # early stop
        start_time = time.time()

        try:
            for epoch in xrange(max_epochs):
                sys.stdout.flush()
                n_samples = 0

                # Get new shuffled index for the training set.
                minibatches = self.get_minibatches_idx(len(train[0]),
                                                       batch_size,
                                                       shuffle=True)

                for _, train_index_list in minibatches:
                    uidx += 1
                    use_noise.set_value(1.)

                    # Select the random examples for this minibatch
                    y = [train[1][t] for t in train_index_list]
                    x = [train[0][t] for t in train_index_list]

                    # Get the data in numpy.ndarray format
                    # This swap the axis!
                    # Return something of shape (minibatch maxlen, n samples)
                    x, mask, y = prepare_data(x, y)
                    n_samples += x.shape[1]

                    cur_cost_val = f_grad_shared(x, mask, y)
                    f_update(lrate)

                    if numpy.isnan(cur_cost_val) or numpy.isinf(cur_cost_val):
                        print 'NaN detected'
                        return 1., 1., 1.

                    if numpy.mod(uidx, dispFreq) == 0:
                        print 'Epoch ', epoch, 'Update ', uidx, 'Cost ', cur_cost_val

                    if saveto and numpy.mod(uidx, saveFreq) == 0:
                        print 'Saving...',

                        if best_p is not None:
                            self.params = best_p
                        else:
                            self.params = self.unzip(self.tparams)
                        numpy.savez(saveto,
                                    history_errs=history_errs,
                                    **self.params)
                        pkl.dump(self.model_options,
                                 open('%s.pkl' % saveto, 'wb'), -1)
                        print 'Done'

                    if numpy.mod(uidx, validFreq) == 0:
                        use_noise.set_value(0.)
                        train_err = self.pred_error(self.f_pred, prepare_data,
                                                    train, minibatches)
                        valid_err = self.pred_error(self.f_pred, prepare_data,
                                                    valid, kf_valid)
                        test_err = self.pred_error(self.f_pred, prepare_data,
                                                   test, kf_test)

                        history_errs.append([valid_err, test_err])

                        if (uidx == 0 or valid_err <=
                                numpy.array(history_errs)[:, 0].min()):

                            best_p = self.unzip(self.tparams)
                            bad_counter = 0

                        print('Train ', train_err, 'Valid ', valid_err,
                              'Test ', test_err)

                        if (len(history_errs) > patience and valid_err >=
                                numpy.array(history_errs)[:-patience,
                                                          0].min()):
                            bad_counter += 1
                            if bad_counter > patience:
                                print 'Early Stop!'
                                estop = True
                                break

                print 'Seen %d samples' % n_samples

                if estop:
                    break

        except KeyboardInterrupt:
            print "Training interrupted"

        end_time = time.time()
        if best_p is not None:
            self.zipp(best_p, self.tparams)
        else:
            best_p = self.unzip(self.tparams)

        use_noise.set_value(0.)
        kf_train_sorted = self.get_minibatches_idx(len(train[0]), batch_size)
        train_err = self.pred_error(self.f_pred, prepare_data, train,
                                    kf_train_sorted)
        valid_err = self.pred_error(self.f_pred, prepare_data, valid, kf_valid)
        test_err = self.pred_error(self.f_pred, prepare_data, test, kf_test)

        print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err
        if saveto:
            numpy.savez(saveto,
                        train_err=train_err,
                        valid_err=valid_err,
                        test_err=test_err,
                        history_errs=history_errs,
                        **best_p)
        print 'The code run for %d epochs, with %f sec/epochs' % (
            (epoch + 1), (end_time - start_time) / (1. * (epoch + 1)))
        print >> sys.stderr, ('Training took %.1fs' % (end_time - start_time))
        self.model_has_been_trained = True
        return train_err, valid_err, test_err
Exemple #15
0
        run_descriptor_numeric = str(
            hash(run_descriptor)) + "_" + util.random_string_signature(4)

        run_descriptor_numeric += "_%s" % util.time_string()
        if ARGS.ID:
            run_descriptor_numeric = ARGS.ID + "_" + run_descriptor_numeric

    MODEL_SAVETO = 'saved_models/%s.npz' % run_descriptor
    RUN_OUTPUT_FNAME = 'output/adv=%s_%s.out' % (ARGS.ADVERSARIAL,
                                                 run_descriptor_numeric)

    logfile = None
    if ARGS.REDIRECT_OUTPUT_TO_FILE:
        logfile = open(RUN_OUTPUT_FNAME, 'w')
        print util.colorprint(
            "printing all standard output and error to %s...." %
            RUN_OUTPUT_FNAME, 'rand')
        sys.stdout = logfile
        sys.stderr = logfile

    print "full name of file: ", run_descriptor

    print MODEL_SAVETO

    # print "ARGS.ADVERSARIAL = %s"%ARGS.ADVERSARIAL
    # print "DATASET = %s"%DATASET
    # print "ARGS.MAX_EPOCHS = %s"%ARGS.MAX_EPOCHS
    # print "in summary: \n%s\n"%run_descriptor

    print "ARGS: "
    print ARGS
    def build_architecture(self, input):
        w_names = self._get_w_matrix_names()
        u_names = self._get_loop_matrix_names()
        all_hidden_activations = []
        if self.debug: util.colorprint("building architecture...", self.debug_model_color)
        # loop_inputs = a mapping from index to vector (dict), initialized to nothing
        # loop_outputs = a mapping from index to vector (dict), initialized to ones

        loop_inputs = {}
        loop_outputs = defaultdict(list)

        #======================================================
        # note that "output_layer" means output FROM the loop, and 
        # input_layer means input TO the loop.  Perhaps better names are in order.
        for input_layer, output_layer in self.loops:
            loop_inputs[input_layer] = None
            loop_output_shape = (self.all_layer_dims[output_layer], 1)
            loop_outputs[output_layer].append(np.ones(loop_output_shape))
            if self.debug:
                util.colorprint("creating loop output of shape %s from layer %s to layer %s"%(loop_output_shape, input_layer, output_layer), self.debug_model_color)

        for _ in range(self.n_unrolls):
            hidden_activation = input
            # hidden_activation = self._prepend_intercept(input)            
            #====================================================
            # put in all the weight matrices and populate the inputs to the loops
            for layer_i, w_name in enumerate(w_names):
                #optional addendum: terminate when the last loop is reached, unless it's the last unroll
                # b_name = self._b_name_from_w_name(w_name)
                if self.debug:
                    util.colorprint("Inserting parameters %s and (layer %s) into the graph..."%(w_name, layer_i), self.debug_model_color)

                if layer_i in loop_outputs:
                    for loop_output in loop_outputs[layer_i]:
                        if self.debug:
                            util.colorprint("\tInserting incoming loop activation...", self.debug_model_color)
                        hidden_activation *= loop_output
                    loop_outputs[layer_i] = []

                hidden_activation = T.dot(self.tparams[w_name], hidden_activation) 

                # print (1, hidden_activation.shape[1])
                # hidden_activation += T.tile(self.tparams[b_name], (1, hidden_activation.shape[1]))
                # hidden_activation += self.tparams[b_name]
                # hidden_activation = T.tanh(hidden_activation)
                hidden_activation = T.nnet.sigmoid(hidden_activation)
                # hidden_activation = self._prepend_intercept(hidden_activation)

                #---------------------------------------------------
                if layer_i in loop_inputs:
                    if self.debug:
                        util.colorprint("\tStoring outgoing loop activation...", self.debug_model_color)
                    loop_inputs[layer_i] = hidden_activation

                #---------------------------------------------------                    
                all_hidden_activations.append(hidden_activation)
            #====================================================
            # calculate the outputs 
            for u_i, u_name in enumerate(u_names):
                input_layer, output_layer = self.loops[u_i]
                # b_name = self._b_name_from_w_name(u_name)
                if self.debug:
                    util.colorprint("inserting %s and %s into the graph, ready to feed into layer %s"%(u_name, b_name, output_layer), self.debug_model_color)
                loop_output = T.dot(self.tparams[u_name], loop_inputs[input_layer])# +  self.tparams[b_name]
                loop_output = T.nnet.sigmoid(loop_output)
                loop_outputs[output_layer].append(loop_output)

        # final_activation = all_hidden_activations[-1]
        self.final_activation = T.nnet.softmax(all_hidden_activations[-1].T)
        all_hidden_activations.append(self.final_activation)

        self.all_hidden_activations = all_hidden_activations

        # self.final_activation = 1.0/(1.0 + T.nnet.sigmoid(final_activation))

        # off = 1e-8
        # if final_activation.dtype == 'float16':
        #     off = 1e-6

        # self.cost = -T.log(final_activation[self.y, 0] + off)        
       
        cost = self.loss_function(self.final_activation, self.y) + self.L1_reg * self.L1

        return cost
Exemple #17
0
def startWithDirectArgs(args: dict):
    root = None
    igxcFile = None
    igxcContent = None
    urlArgument = None
    basePath = None
    result = None
    outFileHashBase = ""
    outFileNameBase = "AO_result"

    resolutionValue = aoConfig["resolution"]
    if "resolution" in args and args["resolution"] is not None:
        resolutionValue = int(args["resolution"])

    if "url" in args and args["url"] is not None:
        print('fetch url', args["url"])
        urlArgument = URL(args["url"])
        igxcFile = fetch(urlArgument, '.igcx')
        colorprint("source: " + str(urlArgument) + '.igcx', 36)
    elif "file" in args and args["file"] is not None:
        igxcFile = Path(args["file"])
        colorprint("source: " + args["file"], 36)
    elif "igxcContent" in args and args["igxcContent"] is not None:
        igxcContent = args["igxcContent"]
        colorprint("source: bakeDirect POST parameters", 36)

    elif "test" in args and args["test"]:
        outFileNameBase = prepareOutFilename(str(random.randint(1, 1e9)),
                                             resolutionValue)
        root = test_scene()
        colorprint("source: test scene", 36)

    if igxcFile is not None:
        basePath = igxcFile.parent

    if "basePath" in args and args["basePath"] is not None:
        # print(args["basePath"])
        basePath = CachedFile(args["basePath"])
    print("basePath:", basePath)

    debug = "debug" in args and args["debug"]

    if igxcFile is not None:

        with igxcFile.open('r') as resource:
            igxcContent = json.load(resource)

        # if igxcContent is None:
        #     return None

    if igxcContent is None:
        print("No content in igxc")

    # check if configuration is already done
    if "Objects" in igxcContent and igxcContent["Objects"] is not None:
        outFileHashBase = outFileHashBase + json.dumps(igxcContent["Objects"],
                                                       sort_keys=True)
    if "Hashes" in igxcContent and igxcContent["Hashes"] is not None:
        outFileHashBase = outFileHashBase + json.dumps(igxcContent["Hashes"],
                                                       sort_keys=True)
    if outFileHashBase == "" and urlArgument is not None:
        outFileHashBase = urlArgument

    outFileNameBase = prepareOutFilename(outFileHashBase, resolutionValue)

    hasImage = os.path.isfile(joinOutputPath(outFileNameBase, 'png'))
    hasMapping = os.path.isfile(joinOutputPath(outFileNameBase, 'json'))
    if hasImage and hasMapping and not debug:
        colorprint("Taking from cache ({})".format(outFileNameBase), 32)
        mappingResult = None
        with open(joinOutputPath(outFileNameBase, 'json'),
                  'r') as mappingInFile:
            mappingResult = json.load(mappingInFile)
        modifyIgxc(igxcContent, outFileNameBase + '.png', mappingResult)
        result = {
            "urlAoMapImage": outFileNameBase + '.png',
            "urlAoMappingJson": outFileNameBase + '.json',
            "urlIgxcModified": outFileNameBase + '.igxc',
            "urlIgxcOriginal": outFileNameBase + '_original.igxc',
            "transforms": mappingResult,
            "igxcModified": igxcContent
        }
        return result

    # save unmodified version of igxc
    if igxcContent is not None:
        igxcOutfileName = joinOutputPath(outFileNameBase + "_original", 'igxc')
        with open(igxcOutfileName, 'w') as igxcOutfile:
            json.dump(igxcContent,
                      igxcOutfile,
                      indent=4,
                      separators=(',', ': '),
                      sort_keys=False)

    # result not in cache? proceed with baking
    root = None
    try:
        root = igxc.load(igxcContent, basePath)
    except AttributeError as e:
        errorMsg = "attributes missing in igxc ({})".format(" ".join(e.args))
        colorprint("startWithDirectArgs: " + errorMsg, 31)
        print(e)
        result = {
            "error": errorMsg,
            "urlIgxcOriginal": outFileNameBase + '_original.igxc'
        }
        return result
    except ConnectionError as e:
        errorMsg = "file referenced in igxc could not be fetched " + " ".join(
            e.args)
        colorprint("startWithDirectArgs: " + errorMsg, 31)
        print(e)
        result = {
            "error": errorMsg,
            "urlIgxcOriginal": outFileNameBase + '_original.igxc'
        }
        return result
    except Exception as e:
        errorMsg = "igxc couldn't be loaded"
        colorprint("startWithDirectArgs: " + errorMsg, 31)
        print(e)
        print(type(e))
        result = {
            "error": errorMsg,
            "urlIgxcOriginal": outFileNameBase + '_original.igxc'
        }
        return result

    triCounter = visitor.TriCounter()
    root.accept(triCounter)
    print('total triangles', triCounter.count)

    vertices = numpy.ndarray((triCounter.count, 3, 3), dtype=numpy.float)
    normals = numpy.ndarray((triCounter.count, 3, 3), dtype=numpy.float)
    texcoord = numpy.ndarray((triCounter.count, 3, 2), dtype=numpy.float)

    meshCounter = visitor.MeshCounter()
    root.accept(meshCounter)
    print('total meshes', meshCounter.count)

    amountBucketsX = math.ceil(math.sqrt(meshCounter.count))
    amountBucketsY = math.ceil(meshCounter.count / amountBucketsX)
    print('buckets: {}x{}'.format(amountBucketsX, amountBucketsY))
    uvPacker = visitor.SimplePacker(amountBucketsX, amountBucketsY,
                                    resolutionValue)
    triExtractor = visitor.TransformedTriExtractor(vertices,
                                                   normals,
                                                   texcoord,
                                                   packer=uvPacker)
    root.accept(triExtractor)

    # if face normals should be used, empty normals array
    if "face_normals" in args and args["face_normals"] == True:
        normals = numpy.zeros((0, 3, 3), dtype=numpy.float)

    # print(vertices)
    # print(texcoord)
    # print(triExtractor.mapping)
    # print("Packer:", uvPacker.i)

    img = generateMap(vertices, normals, texcoord,
                      (resolutionValue, resolutionValue))

    # save AO map image
    output = joinOutputPath(outFileNameBase, 'png')
    print("Save output at", joinOutputPath(outFileNameBase, 'png'))
    img.save(output)

    # save AO mapping
    mappingOutfileName = joinOutputPath(outFileNameBase, 'json')
    with open(mappingOutfileName, 'w') as outfile:
        json.dump(triExtractor.mapping,
                  outfile,
                  indent=4,
                  separators=(',', ': '),
                  sort_keys=True)

    # extend existing IGXC with AO entries
    if igxcContent is not None:
        modifyIgxc(igxcContent, outFileNameBase + '.png', triExtractor.mapping)
        igxcOutfileName = joinOutputPath(outFileNameBase, 'igxc')
        with open(igxcOutfileName, 'w') as igxcOutfile:
            json.dump(igxcContent,
                      igxcOutfile,
                      indent=4,
                      separators=(',', ': '),
                      sort_keys=False)

    if debug:
        import viewer
        viewer.debug_view(vertices, texcoord, image=img)

    result = {
        "urlAoMapImage": outFileNameBase + '.png',
        "urlAoMappingJson": outFileNameBase + '.json',
        "urlIgxcModified": outFileNameBase + '.igxc',
        "urlIgxcOriginal": outFileNameBase + '_original.igxc',
        "transforms": triExtractor.mapping,
        "igxcModified": igxcContent
    }
    return result