def network_test():
layer_dict = {'0':
{'n_neuron': 10,
'incoming_layer_list': [],
'incoming_weight_list': [],
'bias': None,
'loss': 'cross_entropy',
'act_func_name': 'sigmoid',
'value': None,
'layer_type': 'input',
'back_error' : 0,
'drop_rate': 0.0,
'link2input': X1,
'link2target': None
},
'1': {'n_neuron':5, 'layer_type':'input', 'link2input':X2},
'2': {'n_neuron': 20, 'incoming_layer_list': [0,1], 'act_func_name':'sigmoid','layer_type':'hidden', 'drop_rate':0.2},
'3': {'n_neuron':2, 'incoming_layer_list': [2,], 'loss':'mse', 'act_func_name':'linear',
'layer_type':'output', 'link2target':Y1},
'4': {'n_neuron':3, 'incoming_layer_list': [2,], 'act_func_name':'softmax',
'layer_type':'output', 'link2target':Y2}
}
network = NeuralNetwork(5,layer_dict = layer_dict)
#outputs = network.feed_forward()
#error = network.empirical_error()
return network.fit(batch_size = 20, learning_rate = step_iterator(0.1,0.01,0), isnorm = True,
weight_decay = step_iterator(1e-5,1e-5,0), n_iter = 500, random_state = 20, switch_point = 500)
def fit(self, batch_size=100, learning_rate=None, weight_decay=None, momentum=None,
n_iter=100, verbose = True, random_state=None, switch_point = None, isnorm = True, only_dropout = False):
learning_rate = step_iterator(0.1,0.1,0) if learning_rate is None else learning_rate
weight_decay = step_iterator(0,0,0) if weight_decay is None else weight_decay
momentum = step_iterator(0,0,0.) if momentum is None else momentum
switch_point = n_iter if switch_point is None else switch_point
rng = check_random_state(random_state)
for layer in self.layer_list:
layer.assign_random_state(rng)
fit_func = self._fit
for i_outer, iteration in enumerate(xrange(n_iter)):
pl = 0.
lr = learning_rate.next()
wd = weight_decay.next()
mom = momentum.next()
# r_index = np.arange(self.input_layer_list[0].link2input.shape[0])
# np.random.shuffle(r_index)
# for inp_layer in self.input_layer_list:
# inp_layer.link2input = inp_layer.link2input[r_index]
# for out_layer in self.output_layer_list:
# out_layer.link2target = out_layer.link2target[r_index]
if verbose:
begin = time()
fit_func = self._fit_dropout if only_dropout else fit_func
for batch_no in self.generate_batch(batch_size):
#print("batch no:%d"%batch_no)
pl_batch = fit_func(learning_rate = lr, weight_decay = wd, momentum = mom, max_lr_iter = 5, isnorm = isnorm)
if verbose:
pl += pl_batch.sum()
fit_func = self._fit if i_outer<switch_point else self._fit_with_minimize
if verbose:
#pl /= n_samples
end = time()
print("Iteration %d, pseudo-likelihood = %.2f, time = %.2fs"
% (iteration, pl, end-begin))
return self
RBMlayers_dict = {0: {
'dimension':X_train_bin.shape[1],
'bias': None,
'value': None,
'layer_type': 'binary',
'layer_name': 'input'
},
1: {
'dimension':200,
'bias': None,
'value': None,
'layer_type': 'binary',
'layer_name': 'hidden'
}
}
learning_rate, weight_decay, momentum= step_iterator(0.01,0.001,-0.002), step_iterator(2e-5,2e-5,0), step_iterator(0.5,0.9,0.05)
batch_func = batch_func_generator(X_train_bin, batch_size = 100)
rbm = BinRBM(layers_dict = RBMlayers_dict, weight_list = None, random_state = None)
print 'Training starts'
rbm.fit(batch_func, PCD = False, error_function = 'recon',learning_rate = learning_rate, momentum = momentum,
weight_decay = weight_decay, k = 1, perst_size = 100, n_iter = 500, verbose = True)
combined = rbm.transform(X_train_bin)
combined_test = rbm.transform(X_test_bin)
parameters = {'alpha':[1e-1,1e-2,1e-3,1e-4,1e-5], 'n_iter': [10, 50 ,100]}
clf = GridSearchCV(SGDClassifier(loss="hinge", penalty="l2", n_iter = 100, random_state = 500), parameters)
clf.fit(combined,labels_train)
y_pred = clf.predict(combined_test)
print 'toplam: ', labels_test.shape[0], 'dogru: ', (y_pred == labels_test).sum()
print clf.best_estimator_
RBM3layers_dict = {0: {
'dimension':RBM2layers_dict[1]['dimension'],
'bias': None,
'value': None,
'layer_type': 'binary',
'layer_name': 'input'
},
1: {
'dimension':2000,
'bias': None,
'value': None,
'layer_type': 'binary',
'layer_name': 'hidden'
}
}
learning_rate, weight_decay, momentum= step_iterator(0.1,0.1,0), step_iterator(2e-4,2e-4,0), step_iterator(0.5,0.9,0.05)
batch_func = batch_func_generator(X_train, batch_size = 100)
rbm0 = BinRBM(layers_dict = RBM1layers_dict, weight_list = None, random_state = None)
rbm0.fit(batch_func, PCD = False, error_function = 'recon',learning_rate = learning_rate, momentum = momentum,
weight_decay = weight_decay, k = 1, perst_size = 100, n_iter = 50, verbose = True)
X_train = rbm0.transform(X_train, sample = False)
learning_rate, weight_decay, momentum= step_iterator(0.1,0.1,0), step_iterator(2e-4,2e-4,0), step_iterator(0.5,0.9,0.05)
batch_func = batch_func_generator(X_train, batch_size = 100)
rbm1 = BinRBM(layers_dict = RBM2layers_dict, weight_list = None, random_state = None)
rbm1.fit(batch_func, PCD = False, error_function = 'recon',learning_rate = learning_rate, momentum = momentum,
weight_decay = weight_decay, k = 1, perst_size = 100, n_iter = 50, verbose = True)
X_train = rbm1.transform(X_train, sample = False)
learning_rate, weight_decay, momentum= step_iterator(0.1,0.1,0), step_iterator(2e-4,2e-4,0), step_iterator(0.5,0.9,0.05)
batch_func = batch_func_generator(X_train, batch_size = 100)
X_all = np.vstack((X_train, X_test))
scl = StandardScaler()
X_all = scl.fit_transform(X_all)
X_train = X_all[:50000]
X_test = X_all[50000:]
del X_all
X_train_copy = X_train.copy()
X_test_copy = X_test.copy()
## 3072-8192, 8192-4096, 4096-2048, 2048-1024, 1024-512
random_state = 500
RBMlayers_dict = {
0: {"dimension": X_train.shape[1], "bias": None, "value": None, "layer_type": "linear", "layer_name": "input"},
1: {"dimension": 8192, "bias": None, "value": None, "layer_type": "binary", "layer_name": "hidden"},
}
learning_rate, weight_decay, momentum = step_iterator(1, 1, 0), step_iterator(0, 0, 0), step_iterator(0, 0, 0)
batch_func = batch_func_generator(X_train, batch_size=100)
rbm = GaussRBM(layers_dict=RBMlayers_dict, weight_list=None, random_state=random_state)
print "Training starts"
rbm.fit(
batch_func,
PCD=False,
error_function="recon",
learning_rate=learning_rate,
momentum=momentum,
weight_decay=weight_decay,
k=1,
perst_size=100,
n_iter=50,
verbose=True,
)
RBMBlayers_dict = {0: {
'dimension':B_train.shape[1],
'bias': None,
'value': None,
'layer_type': 'linear',
'layer_name': 'input'
},
1: {
'dimension':500,
'bias': None,
'value': None,
'layer_type': 'binary',
'layer_name': 'hidden'
} }
learning_rate, weight_decay, momentum= step_iterator(1,1,0), step_iterator(0,0,0), step_iterator(0,0,0)
batch_func = batch_func_generator(R_train, batch_size = 50)
rbmR = GaussRBM(layers_dict = RBMRlayers_dict, weight_list = None, random_state = random_state)
print 'Training starts'
rbmR.fit(batch_func, PCD = False, error_function = 'recon',learning_rate = learning_rate, momentum = momentum,
weight_decay = weight_decay, k = 1, perst_size = 100, n_iter = 20, verbose = True)
# sparsity_cond = True, sparsity_target = 0.01, sparsity_lambda = 1e-6)
batch_func = batch_func_generator(G_train, batch_size = 50)
rbmG = GaussRBM(layers_dict = RBMGlayers_dict, weight_list = None, random_state = random_state)
print 'Training starts'
rbmG.fit(batch_func, PCD = False, error_function = 'recon',learning_rate = learning_rate, momentum = momentum,
weight_decay = weight_decay, k = 1, perst_size = 100, n_iter = 20, verbose = True)
# sparsity_cond = True, sparsity_target = 0.01, sparsity_lambda = 1e-6)
batch_func = batch_func_generator(B_train, batch_size = 50)
rbmB = GaussRBM(layers_dict = RBMBlayers_dict, weight_list = None, random_state = random_state)
print 'Training starts'