-
Notifications
You must be signed in to change notification settings - Fork 0
/
features_reduction.py
134 lines (103 loc) · 4.06 KB
/
features_reduction.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
import logging
import numpy as np
from blocks.algorithms import GradientDescent, Momentum
from blocks.bricks import Tanh, Softmax, Linear
from blocks.bricks.recurrent import LSTM
from blocks.dump import load_parameter_values
from blocks.dump import MainLoopDumpManager
from blocks.extensions import Printing
from blocks.extensions.monitoring import DataStreamMonitoring
from blocks.graph import ComputationGraph
from blocks.initialization import IsotropicGaussian, Constant
from blocks.main_loop import MainLoop
from blocks.model import Model
from theano import tensor
from datastream import prepare_data, RandomTransposeIt, LogregOrderTransposeIt
logging.basicConfig(level='INFO')
logger = logging.getLogger(__name__)
def construct_model(activation_function, r_dim, hidden_dim, out_dim):
# Construct the model
r = tensor.fmatrix('r')
x = tensor.fmatrix('x')
y = tensor.ivector('y')
nx = x.shape[0]
nj = x.shape[1] # also is r.shape[0]
nr = r.shape[1]
# r is nj x nr
# x is nx x nj
# y is nx
# Get a representation of r of size r_dim
r = DAE(r)
# r is now nj x r_dim
# r_rep is nx x nj x r_dim
r_rep = r[None, :, :].repeat(axis=0, repeats=nx)
# x3 is nx x nj x 1
x3 = x[:, :, None]
# concat is nx x nj x (r_dim + 1)
concat = tensor.concatenate([r_rep, x3], axis=2)
# Change concat from Batch x Time x Features to T X B x F
rnn_input = concat.dimshuffle(1, 0, 2)
linear = Linear(input_dim=r_dim + 1, output_dim=4 * hidden_dim,
name="input_linear")
lstm = LSTM(dim=hidden_dim, activation=activation_function,
name="hidden_recurrent")
top_linear = Linear(input_dim=hidden_dim, output_dim=out_dim,
name="out_linear")
pre_rnn = linear.apply(rnn_input)
states = lstm.apply(pre_rnn)[0]
activations = top_linear.apply(states)
activations = tensor.mean(activations, axis=0)
cost = Softmax().categorical_cross_entropy(y, activations)
pred = activations.argmax(axis=1)
error_rate = tensor.neq(y, pred).mean()
# Initialize parameters
for brick in (linear, lstm, top_linear):
brick.weights_init = IsotropicGaussian(0.1)
brick.biases_init = Constant(0.)
brick.initialize()
return cost, error_rate
def train_model(cost, error_rate, train_stream,
load_location=None, save_location=None):
cost.name = "Cross_entropy"
error_rate.name = 'Error_rate'
# Define the model
model = Model(cost)
# Load the parameters from a dumped model
if load_location is not None:
logger.info('Loading parameters...')
model.set_param_values(load_parameter_values(load_location))
cg = ComputationGraph(cost)
step_rule = Momentum(learning_rate=0.1, momentum=0.9)
algorithm = GradientDescent(cost=cost, step_rule=step_rule,
params=cg.parameters)
main_loop = MainLoop(
model=model,
data_stream=train_stream,
algorithm=algorithm,
extensions=[
# DataStreamMonitoring([cost], test_stream, prefix='test',
# after_epoch=False, every_n_epochs=10),
DataStreamMonitoring([cost], train_stream, prefix='train',
after_epoch=True),
Printing(after_epoch=True)
]
)
main_loop.run()
# Save the main loop
if save_location is not None:
logger.info('Saving the main loop...')
dump_manager = MainLoopDumpManager(save_location)
dump_manager.dump(main_loop)
logger.info('Saved')
if __name__ == "__main__":
train_ex = 100
r_dim = 50
# Build model
cost, error_rate = construct_model(Tanh(), r_dim, 30, 2)
# Build datastream
train_stream = prepare_data("ARCENE", "train",
LogregOrderTransposeIt(10, True,
'model_param/logreg_param.pkl', 500))
# Train the model
train_model(cost, error_rate, train_stream, load_location=None,
save_location=None)