def build_dataset(self): dataset = None preprocessor = None if self.state.dataset.preprocessor.type is None else \ getattr(preproc, self.state.dataset.preprocessor.type)() if self.state.dataset.preprocessor.type == 'Scale': preprocessor.max = self.state.dataset.preprocessor.global_max preprocessor.min = self.state.dataset.preprocessor.global_min preprocessor.buffer = self.state.dataset.preprocessor.buffer preprocessor.scale_range = self.state.dataset.preprocessor.scale_range if self.state.dataset.type == 'Mnist': dataset = Mnist(train_valid_test_ratio=self.state.dataset. train_valid_test_ratio, preprocessor=preprocessor, batch_size=self.state.dataset.batch_size, num_batches=self.state.dataset.num_batches, iter_class=self.state.dataset.iter_class, rng=self.state.dataset.rng) if self.state.dataset.type[:11] == 'TransFactor': dataset = getattr(tf, self.state.dataset.type)( # feature_size = self.state.dataset.feature_size, # target_size = self.state.dataset.target_size, train_valid_test_ratio=self.state.dataset. train_valid_test_ratio, preprocessor=preprocessor, batch_size=self.state.dataset.batch_size, num_batches=self.state.dataset.num_batches, iter_class=self.state.dataset.iter_class, rng=self.state.dataset.rng) elif self.state.dataset.type[:12] == 'Mnist_Blocks': dataset = getattr(mnist, self.state.dataset.type)( feature_size=self.state.dataset.feature_size, target_size=self.state.dataset.feature_size, train_valid_test_ratio=self.state.dataset. train_valid_test_ratio, preprocessor=preprocessor, batch_size=self.state.dataset.batch_size, num_batches=self.state.dataset.num_batches, iter_class=self.state.dataset.iter_class, rng=self.state.dataset.rng) return dataset
import os os.environ['PYNET_DATA_PATH'] = '/Volumes/Storage/Dropbox/CodingProjects/pynet/data/' from pynet.datasets.mnist import Mnist from pynet.datasets.preprocessor import GCN data = Mnist() train = data.get_train() proc = GCN() out = proc.apply(train.X) inv = proc.invert(out)
def build_dataset(self): dataset = None preprocessor = None if self.state.dataset.preprocessor.type is None else \ getattr(preproc, self.state.dataset.preprocessor.type)() noise = None if self.state.dataset.dataset_noise.type is None else \ getattr(data_noise, self.state.dataset.dataset_noise.type)() if self.state.dataset.dataset_noise.type == 'Gaussian': noise.std = self.state.dataset.dataset_noise.std if self.state.dataset.preprocessor.type == 'Scale': preprocessor.max = self.state.dataset.preprocessor.global_max preprocessor.min = self.state.dataset.preprocessor.global_min preprocessor.buffer = self.state.dataset.preprocessor.buffer preprocessor.scale_range = self.state.dataset.preprocessor.scale_range if self.state.dataset.type == 'Mnist': dataset = Mnist(train_valid_test_ratio=self.state.dataset. train_valid_test_ratio, preprocessor=preprocessor, noise=noise, batch_size=self.state.dataset.batch_size, num_batches=self.state.dataset.num_batches, iter_class=self.state.dataset.iter_class, rng=self.state.dataset.rng) train = dataset.get_train() dataset.set_train(train.y, train.y) valid = dataset.get_valid() dataset.set_valid(valid.y, valid.y) test = dataset.get_test() dataset.set_test(test.y, test.y) elif self.state.dataset.type[:12] == 'Mnist_Blocks': dataset = getattr(mnist, self.state.dataset.type)( feature_size=self.state.dataset.feature_size, target_size=self.state.dataset.feature_size, train_valid_test_ratio=self.state.dataset. train_valid_test_ratio, preprocessor=preprocessor, noise=noise, batch_size=self.state.dataset.batch_size, num_batches=self.state.dataset.num_batches, iter_class=self.state.dataset.iter_class, rng=self.state.dataset.rng) elif self.state.dataset.type[:4] == 'P276': dataset = getattr(spec, self.state.dataset.type)( train_valid_test_ratio=self.state.dataset. train_valid_test_ratio, preprocessor=preprocessor, noise=noise, batch_size=self.state.dataset.batch_size, num_batches=self.state.dataset.num_batches, iter_class=self.state.dataset.iter_class, rng=self.state.dataset.rng) train = dataset.get_train() dataset.set_train(train.X, train.X) valid = dataset.get_valid() dataset.set_valid(valid.X, valid.X) test = dataset.get_test() dataset.set_test(test.X, test.X) elif self.state.dataset.type[:5] == 'Laura': dataset = getattr(spec, self.state.dataset.type)( feature_size=self.state.dataset.feature_size, target_size=self.state.dataset.feature_size, train_valid_test_ratio=self.state.dataset. train_valid_test_ratio, num_blocks=self.state.dataset.num_blocks, preprocessor=preprocessor, noise=noise, batch_size=self.state.dataset.batch_size, num_batches=self.state.dataset.num_batches, iter_class=self.state.dataset.iter_class, rng=self.state.dataset.rng) elif self.state.dataset.type[:18] == 'TransFactor_Blocks': dataset = getattr(tf, self.state.dataset.type)( feature_size=self.state.dataset.feature_size, target_size=self.state.dataset.feature_size, one_hot=self.state.dataset.one_hot, num_blocks=self.state.dataset.num_blocks, train_valid_test_ratio=self.state.dataset. train_valid_test_ratio, preprocessor=preprocessor, noise=noise, batch_size=self.state.dataset.batch_size, num_batches=self.state.dataset.num_batches, iter_class=self.state.dataset.iter_class, rng=self.state.dataset.rng) elif self.state.dataset.type[:11] == 'TransFactor': dataset = getattr(tf, self.state.dataset.type)( # feature_size = self.state.dataset.feature_size, # target_size = self.state.dataset.feature_size, train_valid_test_ratio=self.state.dataset. train_valid_test_ratio, preprocessor=preprocessor, noise=noise, batch_size=self.state.dataset.batch_size, num_batches=self.state.dataset.num_batches, iter_class=self.state.dataset.iter_class, rng=self.state.dataset.rng) train = dataset.get_train() dataset.set_train(train.X, train.X) valid = dataset.get_valid() dataset.set_valid(valid.X, valid.X) test = dataset.get_test() dataset.set_test(test.X, test.X) elif self.state.dataset.type[:13] == 'I2R_Posterior': dataset = getattr(i2r, self.state.dataset.type)( train_valid_test_ratio=self.state.dataset. train_valid_test_ratio, preprocessor=preprocessor, noise=noise, batch_size=self.state.dataset.batch_size, num_batches=self.state.dataset.num_batches, iter_class=self.state.dataset.iter_class, rng=self.state.dataset.rng) return dataset
def build_dataset(self): dataset = None preprocessor = None if self.state.dataset.preprocessor.type is None else \ getattr(preproc, self.state.dataset.preprocessor.type)() # if self.state.dataset.noise.type == 'BlackOut' or self.state.dataset.noise.type == 'MaskOut': # noise = None if self.state.dataset.noise.type is None else \ # getattr(noisy, self.state.dataset.noise.type)(ratio=self.state.dataset.noise.ratio) # else: # noise = getattr(noisy, self.state.dataset.noise.type)() noise = None if self.state.dataset.dataset_noise.type is None else \ getattr(noisy, self.state.dataset.dataset_noise.type)() if self.state.dataset.preprocessor.type == 'Scale': preprocessor.max = self.state.dataset.preprocessor.global_max preprocessor.min = self.state.dataset.preprocessor.global_min preprocessor.buffer = self.state.dataset.preprocessor.buffer preprocessor.scale_range = self.state.dataset.preprocessor.scale_range if self.state.dataset.type == 'Mnist': dataset = Mnist(train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, preprocessor = preprocessor, noise = noise, batch_size = self.state.dataset.batch_size, num_batches = self.state.dataset.num_batches, iter_class = self.state.dataset.iter_class, rng = self.state.dataset.rng) train = dataset.get_train() dataset.set_train(train.X, train.X) valid = dataset.get_valid() dataset.set_valid(valid.X, valid.X) test = dataset.get_test() dataset.set_test(test.X, test.X) elif self.state.dataset.type[:12] == 'Mnist_Blocks': dataset = getattr(mnist, self.state.dataset.type)( feature_size = self.state.dataset.feature_size, target_size = self.state.dataset.feature_size, train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, preprocessor = preprocessor, noise = noise, batch_size = self.state.dataset.batch_size, num_batches = self.state.dataset.num_batches, iter_class = self.state.dataset.iter_class, rng = self.state.dataset.rng) elif self.state.dataset.type[:4] == 'P276': dataset = getattr(spec, self.state.dataset.type)( train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, preprocessor = preprocessor, noise = noise, batch_size = self.state.dataset.batch_size, num_batches = self.state.dataset.num_batches, iter_class = self.state.dataset.iter_class, rng = self.state.dataset.rng) train = dataset.get_train() dataset.set_train(train.X, train.X) valid = dataset.get_valid() dataset.set_valid(valid.X, valid.X) test = dataset.get_test() dataset.set_test(test.X, test.X) elif self.state.dataset.type[:5] == 'Laura': dataset = getattr(spec, self.state.dataset.type)( feature_size = self.state.dataset.feature_size, target_size = self.state.dataset.feature_size, train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, preprocessor = preprocessor, noise = noise, batch_size = self.state.dataset.batch_size, num_batches = self.state.dataset.num_batches, iter_class = self.state.dataset.iter_class, rng = self.state.dataset.rng) elif self.state.dataset.type[:18] == 'TransFactor_Blocks': dataset = getattr(tf, self.state.dataset.type)( feature_size = self.state.dataset.feature_size, target_size = self.state.dataset.feature_size, train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, preprocessor = preprocessor, noise = noise, batch_size = self.state.dataset.batch_size, num_batches = self.state.dataset.num_batches, iter_class = self.state.dataset.iter_class, rng = self.state.dataset.rng) elif self.state.dataset.type[:11] == 'TransFactor': dataset = getattr(tf, self.state.dataset.type)( # feature_size = self.state.dataset.feature_size, # target_size = self.state.dataset.feature_size, train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, preprocessor = preprocessor, noise = noise, batch_size = self.state.dataset.batch_size, num_batches = self.state.dataset.num_batches, iter_class = self.state.dataset.iter_class, rng = self.state.dataset.rng) train = dataset.get_train() dataset.set_train(train.X, train.X) valid = dataset.get_valid() dataset.set_valid(valid.X, valid.X) test = dataset.get_test() dataset.set_test(test.X, test.X) return dataset
def build_dataset(self): dataset = None preprocessor = None if self.state.dataset.preprocessor.type is None else \ getattr(preproc, self.state.dataset.preprocessor.type)() noise = None if self.state.dataset.dataset_noise.type is None else \ getattr(data_noise, self.state.dataset.dataset_noise.type)() if self.state.dataset.dataset_noise.type == 'Gaussian': noise.std = self.state.dataset.dataset_noise.std if self.state.dataset.preprocessor.type == 'Scale': preprocessor.max = self.state.dataset.preprocessor.global_max preprocessor.min = self.state.dataset.preprocessor.global_min preprocessor.buffer = self.state.dataset.preprocessor.buffer preprocessor.scale_range = self.state.dataset.preprocessor.scale_range if self.state.dataset.type == 'Mnist': dataset = Mnist(train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, preprocessor = preprocessor, noise = noise, batch_size = self.state.dataset.batch_size, num_batches = self.state.dataset.num_batches, iter_class = self.state.dataset.iter_class, rng = self.state.dataset.rng) train = dataset.get_train() dataset.set_train(train.y, train.y) valid = dataset.get_valid() dataset.set_valid(valid.y, valid.y) test = dataset.get_test() dataset.set_test(test.y, test.y) elif self.state.dataset.type[:12] == 'Mnist_Blocks': dataset = getattr(mnist, self.state.dataset.type)( feature_size = self.state.dataset.feature_size, target_size = self.state.dataset.feature_size, train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, preprocessor = preprocessor, noise = noise, batch_size = self.state.dataset.batch_size, num_batches = self.state.dataset.num_batches, iter_class = self.state.dataset.iter_class, rng = self.state.dataset.rng) elif self.state.dataset.type[:4] == 'P276': dataset = getattr(spec, self.state.dataset.type)( train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, preprocessor = preprocessor, noise = noise, batch_size = self.state.dataset.batch_size, num_batches = self.state.dataset.num_batches, iter_class = self.state.dataset.iter_class, rng = self.state.dataset.rng) train = dataset.get_train() dataset.set_train(train.X, train.X) valid = dataset.get_valid() dataset.set_valid(valid.X, valid.X) test = dataset.get_test() dataset.set_test(test.X, test.X) elif self.state.dataset.type[:5] == 'Laura': dataset = getattr(spec, self.state.dataset.type)( feature_size = self.state.dataset.feature_size, target_size = self.state.dataset.feature_size, train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, num_blocks = self.state.dataset.num_blocks, preprocessor = preprocessor, noise = noise, batch_size = self.state.dataset.batch_size, num_batches = self.state.dataset.num_batches, iter_class = self.state.dataset.iter_class, rng = self.state.dataset.rng) elif self.state.dataset.type[:18] == 'TransFactor_Blocks': dataset = getattr(tf, self.state.dataset.type)( feature_size = self.state.dataset.feature_size, target_size = self.state.dataset.feature_size, one_hot = self.state.dataset.one_hot, num_blocks = self.state.dataset.num_blocks, train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, preprocessor = preprocessor, noise = noise, batch_size = self.state.dataset.batch_size, num_batches = self.state.dataset.num_batches, iter_class = self.state.dataset.iter_class, rng = self.state.dataset.rng) elif self.state.dataset.type[:11] == 'TransFactor': dataset = getattr(tf, self.state.dataset.type)( # feature_size = self.state.dataset.feature_size, # target_size = self.state.dataset.feature_size, train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, preprocessor = preprocessor, noise = noise, batch_size = self.state.dataset.batch_size, num_batches = self.state.dataset.num_batches, iter_class = self.state.dataset.iter_class, rng = self.state.dataset.rng) train = dataset.get_train() dataset.set_train(train.X, train.X) valid = dataset.get_valid() dataset.set_valid(valid.X, valid.X) test = dataset.get_test() dataset.set_test(test.X, test.X) elif self.state.dataset.type[:13] == 'I2R_Posterior': dataset = getattr(i2r, self.state.dataset.type)( train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, preprocessor = preprocessor, noise = noise, batch_size = self.state.dataset.batch_size, num_batches = self.state.dataset.num_batches, iter_class = self.state.dataset.iter_class, rng = self.state.dataset.rng) return dataset
def mlp(): # build dataset data = Mnist(preprocessor=None, train_valid_test_ratio=[5, 1, 1]) # build mlp mlp = MLP(input_dim=data.feature_size()) W1 = GaussianWeight(prev_dim=mlp.input_dim, this_dim=1000) hidden1 = PRELU(dim=1000, name='h1_layer', W=W1(mean=0, std=0.1), b=None, dropout_below=None) mlp.add_layer(hidden1) W2 = XavierWeight(prev_dim=hidden1.dim, this_dim=data.target_size()) output = Softmax(dim=data.target_size(), name='output_layer', W=W2(), b=None, dropout_below=None) mlp.add_layer(output) # build learning method learning_method = AdaGrad(learning_rate=0.1, momentum=0.9) # set the learning rules learning_rule = LearningRule(max_col_norm=10, L1_lambda=None, L2_lambda=None, training_cost=Cost(type='mse'), learning_rate_decay_factor=None, stopping_criteria={ 'max_epoch': 300, 'epoch_look_back': 10, 'cost': Cost(type='error'), 'percent_decrease': 0.01 }) # (optional) build the logging object log = Log(experiment_name='mnist', description='This is tutorial example', save_outputs=True, save_learning_rule=True, save_model=True, save_epoch_error=True, save_to_database={ 'name': 'Example.db', 'records': { 'Dataset': data.__class__.__name__, 'max_col_norm': learning_rule.max_col_norm, 'Weight_Init_Seed': mlp.rand_seed, 'Dropout_Below': str([layer.dropout_below for layer in mlp.layers]), 'Batch_Size': data.batch_size, 'Layer_Dim': str([layer.dim for layer in mlp.layers]), 'Layer_Types': str([layer.__class__.__name__ for layer in mlp.layers]), 'Preprocessor': data.preprocessor.__class__.__name__, 'Learning_Rate': learning_method.learning_rate, 'Momentum': learning_method.momentum, 'Training_Cost': learning_rule.cost.type, 'Stopping_Cost': learning_rule.stopping_criteria['cost'].type } }) # end log # put everything into the train object train_object = TrainObject(model=mlp, dataset=data, learning_rule=learning_rule, learning_method=learning_method, log=log) # finally run the code train_object.run()
def stacked_autoencoder(): name = "Stacked_AE" # =====[ Train First layer of stack autoencoder ]=====# print("Start training First Layer of AutoEncoder") # building dataset, batch_size and preprocessor data = Mnist(train_valid_test_ratio=[8, 1, 1], batch_size=100) # for AutoEncoder, the inputs and outputs must be the same train = data.get_train() data.set_train(train.X, train.X) valid = data.get_valid() data.set_valid(valid.X, valid.X) test = data.get_test() data.set_test(test.X, test.X) # building autoencoder ae = AutoEncoder(input_dim=data.feature_size(), rand_seed=123) h1_layer = RELU(dim=500, name="h1_layer", W=None, b=None) # adding encoding layer ae.add_encode_layer(h1_layer) # mirror layer has W = h1_layer.W.T h1_mirror = RELU(dim=ae.input_dim, name="h1_mirror", W=h1_layer.W.T, b=None) # adding decoding mirror layer ae.add_decode_layer(h1_mirror) # build learning method learning_method = SGD(learning_rate=0.001, momentum=0.9) # set the learning rules learning_rule = LearningRule( max_col_norm=10, L1_lambda=None, L2_lambda=None, training_cost=Cost(type="mse"), learning_rate_decay_factor=None, stopping_criteria={"max_epoch": 3, "epoch_look_back": 1, "cost": Cost(type="error"), "percent_decrease": 0.01}, ) # put all the components into a TrainObject train_object = TrainObject(model=ae, dataset=data, learning_rule=learning_rule, learning_method=learning_method) # finally run the training train_object.run() # =====[ Train Second Layer of autoencoder ]=====# print("Start training Second Layer of AutoEncoder") # fprop == forward propagation reduced_train_X = ae.encode(train.X) reduced_valid_X = ae.encode(valid.X) reduced_test_X = ae.encode(test.X) data.set_train(X=reduced_train_X, y=reduced_train_X) data.set_valid(X=reduced_valid_X, y=reduced_valid_X) data.set_test(X=reduced_test_X, y=reduced_test_X) # create a new mlp taking inputs from the encoded outputs of first autoencoder ae2 = AutoEncoder(input_dim=data.feature_size(), rand_seed=None) h2_layer = RELU(dim=100, name="h2_layer", W=None, b=None) ae2.add_encode_layer(h2_layer) h2_mirror = RELU(dim=h1_layer.dim, name="h2_mirror", W=h2_layer.W.T, b=None) ae2.add_decode_layer(h2_mirror) train_object = TrainObject(model=ae2, dataset=data, learning_rule=learning_rule, learning_method=learning_method) train_object.run() # =====[ Fine Tuning ]=====# print("Fine Tuning") data = Mnist() train = data.get_train() data.set_train(train.X, train.X) valid = data.get_valid() data.set_valid(valid.X, valid.X) test = data.get_test() data.set_test(test.X, test.X) ae3 = AutoEncoder(input_dim=data.feature_size(), rand_seed=None) ae3.add_encode_layer(h1_layer) ae3.add_encode_layer(h2_layer) ae3.add_decode_layer(h2_mirror) ae3.add_decode_layer(h1_mirror) train_object = TrainObject(model=ae3, dataset=data, learning_rule=learning_rule, learning_method=learning_method) train_object.run() print("Training Done")
def autoencoder(): # building dataset, batch_size and preprocessor data = Mnist(train_valid_test_ratio=[8, 1, 1], batch_size=100, preprocessor=GCN()) # for AutoEncoder, the inputs and outputs must be the same train = data.get_train() data.set_train(train.X, train.X) valid = data.get_valid() data.set_valid(valid.X, valid.X) test = data.get_test() data.set_test(test.X, test.X) # building autoencoder ae = AutoEncoder(input_dim=data.feature_size(), rand_seed=123) h1_layer = Tanh(dim=500, name="h1_layer", W=None, b=None) # adding encoding layer ae.add_encode_layer(h1_layer) # mirror layer has W = h1_layer.W.T h1_mirror = Tanh(dim=ae.input_dim, name="h1_mirror", W=h1_layer.W.T, b=None) # adding decoding mirror layer ae.add_decode_layer(h1_mirror) # build learning method learning_method = AdaGrad(learning_rate=0.1, momentum=0.9) # set the learning rules learning_rule = LearningRule( max_col_norm=10, L1_lambda=None, L2_lambda=None, training_cost=Cost(type="mse"), learning_rate_decay_factor=None, stopping_criteria={ "max_epoch": 300, "epoch_look_back": 10, "cost": Cost(type="error"), "percent_decrease": 0.01, }, ) # put all the components into a TrainObject train_object = TrainObject(model=ae, dataset=data, learning_rule=learning_rule, learning_method=learning_method) # finally run the training train_object.run()
import os os.environ[ 'PYNET_DATA_PATH'] = '/Volumes/Storage/Dropbox/CodingProjects/pynet/data/' from pynet.datasets.mnist import Mnist from pynet.datasets.preprocessor import GCN data = Mnist() train = data.get_train() proc = GCN() out = proc.apply(train.X) inv = proc.invert(out)
def mlp(): # build dataset data = Mnist(preprocessor=None, train_valid_test_ratio=[5,1,1]) # build mlp mlp = MLP(input_dim = data.feature_size()) W1 = GaussianWeight(prev_dim=mlp.input_dim, this_dim=1000) hidden1 = PRELU(dim=1000, name='h1_layer', W=W1(mean=0, std=0.1), b=None, dropout_below=None) mlp.add_layer(hidden1) W2 = XavierWeight(prev_dim=hidden1.dim, this_dim=data.target_size()) output = Softmax(dim=data.target_size(), name='output_layer', W=W2(), b=None, dropout_below=None) mlp.add_layer(output) # build learning method learning_method = AdaGrad(learning_rate=0.1, momentum=0.9) # set the learning rules learning_rule = LearningRule(max_col_norm = 10, L1_lambda = None, L2_lambda = None, training_cost = Cost(type='mse'), learning_rate_decay_factor = None, stopping_criteria = {'max_epoch' : 300, 'epoch_look_back' : 10, 'cost' : Cost(type='error'), 'percent_decrease' : 0.01} ) # (optional) build the logging object log = Log(experiment_name = 'mnist', description = 'This is tutorial example', save_outputs = True, save_learning_rule = True, save_model = True, save_epoch_error = True, save_to_database = {'name': 'Example.db', 'records' : {'Dataset' : data.__class__.__name__, 'max_col_norm' : learning_rule.max_col_norm, 'Weight_Init_Seed' : mlp.rand_seed, 'Dropout_Below' : str([layer.dropout_below for layer in mlp.layers]), 'Batch_Size' : data.batch_size, 'Layer_Dim' : str([layer.dim for layer in mlp.layers]), 'Layer_Types' : str([layer.__class__.__name__ for layer in mlp.layers]), 'Preprocessor' : data.preprocessor.__class__.__name__, 'Learning_Rate' : learning_method.learning_rate, 'Momentum' : learning_method.momentum, 'Training_Cost' : learning_rule.cost.type, 'Stopping_Cost' : learning_rule.stopping_criteria['cost'].type}} ) # end log # put everything into the train object train_object = TrainObject(model = mlp, dataset = data, learning_rule = learning_rule, learning_method = learning_method, log = log) # finally run the code train_object.run()