def model2():
#pdb.set_trace()
# train set X has dim (60,000, 784), y has dim (60,000, 10)
train_set = MNIST(which_set='train', one_hot=True)
# test set X has dim (10,000, 784), y has dim (10,000, 10)
test_set = MNIST(which_set='test', one_hot=True)
# =====<Create the MLP Model>=====
h1_layer = RectifiedLinear(layer_name='h1', dim=1000, irange=0.5)
#print h1_layer.get_params()
h2_layer = RectifiedLinear(layer_name='h2',
dim=1000,
sparse_init=15,
max_col_norm=1)
y_layer = Softmax(layer_name='y',
n_classes=train_set.y.shape[1],
irange=0.5)
mlp = MLP(batch_size=100,
input_space=VectorSpace(dim=train_set.X.shape[1]),
layers=[h1_layer, h2_layer, y_layer])
# =====<Create the SGD algorithm>=====
sgd = SGD(batch_size=100,
init_momentum=0.1,
learning_rate=0.01,
monitoring_dataset={
'valid': train_set,
'test': test_set
},
cost=SumOfCosts(costs=[
MethodCost('cost_from_X'),
WeightDecay(coeffs=[0.00005, 0.00005, 0.00005])
]),
termination_criterion=MonitorBased(
channel_name='valid_y_misclass', prop_decrease=0.0001, N=5))
#sgd.setup(model=mlp, dataset=train_set)
# =====<Extensions>=====
ext = [MomentumAdjustor(start=1, saturate=10, final_momentum=0.99)]
# =====<Create Training Object>=====
save_path = './mlp_model2.pkl'
train_obj = Train(dataset=train_set,
model=mlp,
algorithm=sgd,
extensions=ext,
save_path=save_path,
save_freq=0)
#train_obj.setup_extensions()
train_obj.main_loop()
def get_layer_rectifiedlinear(self, layer_id, layer_name):
row = self.db.executeSQL(
"""
SELECT dim,irange,istdev,sparse_init,sparse_stdev,include_prob,
init_bias,W_lr_scale,b_lr_scale,left_slope,max_row_norm,
max_col_norm,use_bias
FROM hps3.layer_rectifiedlinear
WHERE layer_id = %s
""", (layer_id, ), self.db.FETCH_ONE)
if not row or row is None:
raise HPSData("No rectifiedlinear layer for layer_id="\
+str(layer_id))
(dim, irange, istdev, sparse_init, sparse_stdev, include_prob,
init_bias, W_lr_scale, b_lr_scale, left_slope, max_row_norm,
max_col_norm, use_bias) = row
return RectifiedLinear(dim=dim,
irange=irange,
istdev=istdev,
sparse_init=sparse_init,
sparse_stdev=sparse_stdev,
include_prob=include_prob,
init_bias=init_bias,
W_lr_scale=W_lr_scale,
b_lr_scale=b_lr_scale,
left_slope=left_slope,
max_row_norm=max_row_norm,
max_col_norm=max_col_norm,
use_bias=use_bias,
layer_name=layer_name)
def generateNonConvRegressor(teacher_hintlayer, student_output_space):
dim = teacher_hintlayer.output_space.get_total_dimension()
layer_name = 'hint_regressor'
irng = 0.05
mcn = 0.9
if isinstance(teacher_hintlayer, MaxoutConvC01B):
hint_reg_layer = Maxout(layer_name,
dim,
teacher_hintlayer.num_pieces,
irange=irng,
max_col_norm=mcn)
elif isinstance(teacher_hintlayer, ConvRectifiedLinear):
hint_reg_layer = RectifiedLinear(dim=dim,
layer_name=layer_name,
irange=irng,
max_col_norm=mcn)
elif isinstance(teacher_hintlayer, ConvElemwise) or isinstance(
teacher_hintlayer, ConvElemwisePL2):
if isinstance(teacher_hintlayer.nonlinearity,
RectifierConvNonlinearity):
hint_reg_layer = RectifiedLinear(dim=dim,
layer_name=layer_name,
irange=irng,
max_col_norm=mcn)
elif isinstance(teacher_hintlayer.nonlinearity,
SigmoidConvNonlinearity):
hint_reg_layer = Sigmoid(dim=dim,
layer_name=layer_name,
irange=irng,
max_col_norm=mcn)
elif isinstance(teacher_hintlayer.nonlinearity, TanhConvNonlinearity):
hint_reg_layer = Tanh(dim=dim,
layer_name=layer_name,
irange=irng,
max_col_norm=mcn)
else:
raise AssertionError("Unknown layer type")
else:
raise AssertionError("Unknown fully-connected layer type")
return hint_reg_layer
def get_layer_rectifiedlinear(self, layer):
# TODO: left_slope is set to 0.0 It should be set by the user!
layer.left_slope = 0.0
return RectifiedLinear(layer_name=layer.layer_name,dim=layer.dim,irange=layer.irange,
istdev=layer.istdev,sparse_init=layer.sparse_init,
sparse_stdev=layer.sparse_stdev, include_prob=layer.include_prob,
init_bias=layer.init_bias,W_lr_scale=layer.W_lr_scale,
b_lr_scale=layer.b_lr_scale,max_col_norm=layer.max_col_norm,
max_row_norm=layer.max_row_norm,
left_slope=layer.left_slope,use_bias=layer.use_bias)
def get_conv2(dim_input):
config = {
'batch_size':
bsize,
'input_space':
Conv2DSpace(shape=dim_input[:2],
num_channels=dim_input[2],
axes=['c', 0, 1, 'b']),
'layers': [
ConvRectifiedLinear(layer_name='h0',
output_channels=20,
irange=.005,
max_kernel_norm=.9,
kernel_shape=[7, 7],
pool_shape=[4, 4],
pool_stride=[2, 2],
W_lr_scale=.1,
b_lr_scale=.1),
ConvRectifiedLinear(layer_name='h1',
output_channels=40,
irange=.005,
max_kernel_norm=0.9,
kernel_shape=[7, 7],
pool_shape=[4, 4],
pool_stride=[2, 2],
W_lr_scale=.1,
b_lr_scale=.1),
ConvRectifiedLinear(layer_name='h2',
output_channels=80,
irange=.005,
max_kernel_norm=0.9,
kernel_shape=[5, 5],
pool_shape=[2, 2],
pool_stride=[2, 2],
W_lr_scale=.1,
b_lr_scale=.1),
RectifiedLinear(layer_name='h3',
irange=.005,
dim=500,
max_col_norm=1.9),
Softmax(layer_name='y',
n_classes=nclass,
irange=.005,
max_col_norm=1.9)
]
}
return MLP(**config)
predAll = [np.zeros(y_valid.shape) for s in range(n_add)]
for i in range(nIter):
seed = i + 9198
R = col_k_ones_matrix(X.shape[1],
m,
k_min=k_min,
k_max=k_max,
seed=seed)
np.random.seed(seed + 33)
R.data = np.random.choice([1, -1], R.data.size)
X3 = X * R
X1 = np.sign(X3) * np.abs(X3)**po
X2 = scaler.fit_transform(X1)
training = DenseDesignMatrix(X=X2[train_idx], y=yMat[train_idx])
l1 = RectifiedLinear(layer_name='l1',
irange=ir,
dim=dim,
max_col_norm=1.)
l2 = RectifiedLinear(layer_name='l2',
irange=ir,
dim=dim,
max_col_norm=1.)
l3 = RectifiedLinear(layer_name='l3',
irange=ir,
dim=dim,
max_col_norm=1.)
output = Softmax(layer_name='y',
n_classes=9,
irange=ir,
max_col_norm=mcn_out)
mdl = MLP([l1, l2, l3, output], nvis=X2.shape[1])
trainer = sgd.SGD(learning_rate=lr,
def test_works():
load = True
if load == False:
ddmTrain = FacialKeypoint(which_set='train', start=0, stop=6000)
ddmValid = FacialKeypoint(which_set='train', start=6000, stop=7049)
# valid can_fit = false
pipeline = preprocessing.Pipeline()
stndrdz = preprocessing.Standardize()
stndrdz.apply(ddmTrain, can_fit=True)
#doubt, how about can_fit = False?
stndrdz.apply(ddmValid, can_fit=False)
GCN = preprocessing.GlobalContrastNormalization()
GCN.apply(ddmTrain, can_fit=True)
GCN.apply(ddmValid, can_fit=False)
pcklFile = open('kpd.pkl', 'wb')
obj = (ddmTrain, ddmValid)
pickle.dump(obj, pcklFile)
pcklFile.close()
return
else:
pcklFile = open('kpd.pkl', 'rb')
(ddmTrain, ddmValid) = pickle.load(pcklFile)
pcklFile.close()
#creating layers
#2 convolutional rectified layers, border mode valid
layer1 = ConvRectifiedLinear(layer_name='convRect1',
output_channels=64,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[3, 3],
pool_stride=[2, 2],
max_kernel_norm=1.9365)
layer2 = ConvRectifiedLinear(layer_name='convRect2',
output_channels=64,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[3, 3],
pool_stride=[2, 2],
max_kernel_norm=1.9365)
# Rectified linear units
layer3 = RectifiedLinear(dim=3000, sparse_init=15, layer_name='RectLin3')
#multisoftmax
n_groups = 30
n_classes = 98
irange = 0
layer_name = 'multisoftmax'
layerMS = MultiSoftmax(n_groups=n_groups,
irange=0.05,
n_classes=n_classes,
layer_name=layer_name)
#setting up MLP
MLPerc = MLP(batch_size=8,
input_space=Conv2DSpace(shape=[96, 96], num_channels=1),
layers=[layer1, layer2, layer3, layerMS])
#mlp_cost
missing_target_value = -1
mlp_cost = MLPCost(cost_type='default',
missing_target_value=missing_target_value)
#algorithm
# learning rate, momentum, batch size, monitoring dataset, cost, termination criteria
term_crit = MonitorBased(prop_decrease=0.00001,
N=30,
channel_name='validation_objective')
kpSGD = KeypointSGD(learning_rate=0.001,
init_momentum=0.5,
monitoring_dataset={
'validation': ddmValid,
'training': ddmTrain
},
batch_size=8,
batches_per_iter=750,
termination_criterion=term_crit,
train_iteration_mode='random_uniform',
cost=mlp_cost)
#train extension
train_ext = ExponentialDecayOverEpoch(decay_factor=0.998,
min_lr_scale=0.01)
#train object
train = Train(dataset=ddmTrain,
save_path='kpd_model2.pkl',
save_freq=1,
model=MLPerc,
algorithm=kpSGD,
extensions=[
train_ext,
MonitorBasedSaveBest(channel_name='validation_objective',
save_path='kpd_best.pkl'),
MomentumAdjustor(start=1, saturate=20, final_momentum=.9)
])
train.main_loop()
train.save()
predAll_test = np.zeros((test.shape[0], 9))
for i in range(nIter):
seed = i + 987654
R = col_k_ones_matrix(X.shape[1],
m,
k_min=k_min,
k_max=k_max,
seed=seed)
np.random.seed(seed + 34)
R.data = np.random.choice([1, -1], R.data.size)
X3 = X * R
X1 = np.sign(X3) * np.abs(X3)**po
X2 = scaler.fit_transform(X1)
training = DenseDesignMatrix(X=X2[:num_train], y=yMat)
l1 = RectifiedLinear(layer_name='l1',
irange=ir,
dim=dim,
max_col_norm=1.)
l2 = RectifiedLinear(layer_name='l2',
irange=ir,
dim=dim,
max_col_norm=1.)
l3 = RectifiedLinear(layer_name='l3',
irange=ir,
dim=dim,
max_col_norm=1.)
output = Softmax(layer_name='y',
n_classes=9,
irange=ir,
max_col_norm=mcn_out)
mdl = MLP([l1, l2, l3, output], nvis=X2.shape[1])
trainer = sgd.SGD(learning_rate=lr,