def add_inputs(self, p):
try:
b = self.buffer[p]
except KeyError:
self.add_buffer(p)
b = self.buffer[p]
self.inputs[p] = In(p, value=b.container, implicit=False)
def test_extend_inplace(self):
mySymbolicMatricesList1 = TypedListType(
tt.TensorType(theano.config.floatX, (False, False)))()
mySymbolicMatricesList2 = TypedListType(
tt.TensorType(theano.config.floatX, (False, False)))()
z = Extend()(mySymbolicMatricesList1, mySymbolicMatricesList2)
m = theano.compile.mode.get_default_mode().including(
"typed_list_inplace_opt")
f = theano.function(
[
In(mySymbolicMatricesList1, borrow=True, mutable=True),
mySymbolicMatricesList2,
],
z,
mode=m,
)
assert f.maker.fgraph.toposort()[0].op.inplace
x = rand_ranged(-1000, 1000, [100, 101])
y = rand_ranged(-1000, 1000, [100, 101])
assert np.array_equal(f([x], [y]), [x, y])
def test_remove_inplace(self):
mySymbolicMatricesList = TypedListType(T.TensorType(
theano.config.floatX, (False, False)))()
mySymbolicMatrix = T.matrix()
z = Remove()(mySymbolicMatricesList, mySymbolicMatrix)
m = theano.compile.mode.get_default_mode().including("typed_list_inplace_opt")
f = theano.function([In(mySymbolicMatricesList, borrow=True,
mutable=True), In(mySymbolicMatrix, borrow=True,
mutable=True)], z, accept_inplace=True, mode=m)
self.assertTrue(f.maker.fgraph.toposort()[0].op.inplace)
x = rand_ranged_matrix(-1000, 1000, [100, 101])
y = rand_ranged_matrix(-1000, 1000, [100, 101])
self.assertTrue(numpy.array_equal(f([x, y], y), [x]))
def __init__(self,
init_params,
inf_params=None,
X=T.matrix(),
S=T.matrix(),
P=T.matrix(),
batch_size=1,
rng=None):
super().__init__(init_params, inf_params, X, S, P, batch_size, rng)
self.type = 'FOOPSI'
start = T.zeros([self.S.shape[0]])
def one_step(s, C_tm1, g):
C = s + T.nnet.sigmoid(g) * C_tm1
return C
C, updates = theano.scan(fn=one_step,
sequences=[self.S.T],
outputs_info=[start],
non_sequences=[self.pars['gamma']])
self.C = C.T
self.F = (softp(self.pars['alpha']) * self.C.T + self.pars['beta']).T
self.genfunc = theano.function(
[self.S,
In(self.P, value=np.zeros([1, 1]).astype(config.floatX))],
self.F,
on_unused_input='ignore')
def test_insert_inplace(self):
mySymbolicMatricesList = TypedListType(
tt.TensorType(theano.config.floatX, (False, False)))()
mySymbolicIndex = tt.scalar(dtype="int64")
mySymbolicMatrix = tt.matrix()
z = Insert()(mySymbolicMatricesList, mySymbolicIndex, mySymbolicMatrix)
m = theano.compile.mode.get_default_mode().including(
"typed_list_inplace_opt")
f = theano.function(
[
In(mySymbolicMatricesList, borrow=True, mutable=True),
mySymbolicIndex,
mySymbolicMatrix,
],
z,
accept_inplace=True,
mode=m,
)
assert f.maker.fgraph.toposort()[0].op.inplace
x = rand_ranged(-1000, 1000, [100, 101])
y = rand_ranged(-1000, 1000, [100, 101])
assert np.array_equal(f([x], np.asarray(1, dtype="int64"), y), [x, y])
def main():
x = T.dmatrix('x')
# T.exp
s = 1 / (1 + T.exp(-x))
logistic = function([x], s)
# 0 is 0.5, negative < 0.5...
print(logistic([[0, 1], [-1, -2]]))
# logistic function can be expressed with hyperbolic tan term
s2 = (1 + T.tanh(x / 2)) / 2
logistic2 = function([x], s2)
print(
np.allclose(logistic([[0, 1], [-1, -2]]), logistic2([[0, 1], [-1,
-2]])))
# do more things at a time
a, b = T.dmatrices('a', 'b')
diff = a - b
abs_diff = abs(diff)
diff_squared = diff**2
f = function([a, b], [diff, abs_diff, diff_squared])
print(f([[1, 1], [1, 1]], [[0, 1], [2, 3]]))
# default value
x, y = T.dscalars('x', 'y')
z = x + y
f = function([x, In(y, value=1)], z)
print(f(33))
print(f(33, 2))
# Inputs with default values must follow inputs without default
# values (like Python’s functions). There can be multiple inputs
# with default values. These parameters can be set positionally
# or by name, as in standard Python
x, y, w = T.dscalars('x', 'y', 'w')
z = (x + y) * w
f = function([x, In(y, value=1), In(w, value=2, name='w_by_name')], z)
print(f(33))
print(f(33, 2))
print(f(33, 0, 1))
print(f(33, w_by_name=1))
print(f(33, w_by_name=1, y=0))
def makeFunc(inList, outList, updates):
inputs = []
for i in inList:
inputs.append(In(i, borrow=True, allow_downcast=True))
outputs = []
for o in outList:
outputs.append(Out(o, borrow=True))
return function(inputs=inputs,
outputs=outputs,
updates=updates,
allow_input_downcast=True)
def test_reverse_inplace(self):
mySymbolicMatricesList = TypedListType(
T.TensorType(theano.config.floatX, (False, False))
)()
z = Reverse()(mySymbolicMatricesList)
m = theano.compile.mode.get_default_mode().including("typed_list_inplace_opt")
f = theano.function(
[In(mySymbolicMatricesList, borrow=True, mutable=True)],
z,
accept_inplace=True,
mode=m,
)
assert f.maker.fgraph.toposort()[0].op.inplace
x = rand_ranged_matrix(-1000, 1000, [100, 101])
y = rand_ranged_matrix(-1000, 1000, [100, 101])
assert np.array_equal(f([x, y]), [y, x])
def __init__(self,
init_params,
inf_params=None,
X=T.matrix(),
S=T.matrix(),
P=T.matrix(),
batch_size=1,
rng=None):
super().__init__(init_params, inf_params, X, S, P, batch_size, rng)
self.type = 'ML_phys'
start = T.zeros([self.S.shape[0]])
def one_step(s, C_tm1, D_tm1, delta_t, tau, eta, c0, kappa, gamma):
C = s + T.exp(-delta_t / tau) * C_tm1
D = T.minimum(1 / softp(gamma),
(D_tm1 * T.maximum(
0, 1. - delta_t / softp(kappa) *
(1 + softp(gamma) *
((softp(c0) + C)**eta - softp(c0)**eta))) +
delta_t / softp(kappa) *
((softp(c0) + C)**eta - softp(c0)**eta)))
return C, D
C_D, updates = theano.scan(fn=one_step,
sequences=[self.S.T],
outputs_info=[start, start],
non_sequences=[
self.pars['delta_t'], self.pars['tau'],
self.pars['eta'], self.pars['c0'],
self.pars['kappa'], self.pars['gamma']
])
self.C = C_D[0].T
self.D = C_D[1].T
self.F = (softp(self.pars['alpha']) * self.D.T + self.pars['beta']).T
self.genfunc = theano.function(
[self.S,
In(self.P, value=np.zeros([1, 1]).astype(config.floatX))],
self.F,
on_unused_input='ignore')
def sample_biases(self, data, traces, dt):
# Gamma prior parameters
alpha = self.priors['glm']['bias']['alpha']
beta = self.priors['glm']['bias']['beta']
# could try to grad the following also from the existing graph?
x = tt.as_tensor_variable(data[0], 'x')
y = tt.as_tensor_variable(data[1], 'y')
η = self.filter(x, self.updates)
α = theano.function([],
tt.sum(y) + alpha,
on_unused_input='warn',
allow_input_downcast=True)
# get binsize without adding it to self.inputs
Δ = theano.shared(empty(self.emt.binsize.ndim), self.emt.binsize.name)
in_Δ = In(self.emt.binsize, value=Δ.container, implicit=False)
Δ.set_value(dt)
i = list(self.inputs.values()) + [in_Δ]
β = theano.function(i,
self.emt.binsize * tt.sum(tt.exp(η)) + beta,
on_unused_input='warn',
allow_input_downcast=True)
# sample exp(bias) = λo given all other parameters and the data
nsamp = list(traces.values())[0].shape[0]
traces['λo'] = np.zeros(nsamp)
for i in tqdm(range(nsamp)):
for v in self.inputs:
self.buffer[v].set_value(traces[self.buffer[v].name][i])
traces['λo'][i] = np.random.gamma(shape=α(), scale=1. / β())
traces['bias'] = np.log(traces['λo'])
return traces
def __init__(self,
init_params,
inf_params=None,
X=T.matrix(),
S=T.matrix(),
P=T.matrix(),
batch_size=1,
rng=None):
super().__init__(init_params, inf_params, X, S, P, batch_size, rng)
self.type = 'SCDF'
start = T.zeros([self.S.shape[0]])
def one_step(s, C_tm1, D_tm1, g, eta, zeta, D_m, kappa):
C = s + T.nnet.sigmoid(g) * C_tm1
D = softp(eta) * (C**(1 + softp(zeta))) * (D_m - T.minimum(
D_tm1, D_m)) + T.nnet.sigmoid(kappa) * T.minimum(D_tm1, D_m)
return C, D
C_D, updates = theano.scan(fn=one_step,
sequences=[self.S.T],
outputs_info=[start, start],
non_sequences=[
self.pars['gamma'], self.pars['eta'],
self.pars['zeta'], self.pars['d_max'],
self.pars['kappa']
])
self.C = C_D[0].T
self.D = C_D[1].T
self.F = (softp(self.pars['alpha']) * self.D.T + self.pars['beta']).T
self.genfunc = theano.function(
[self.S,
In(self.P, value=np.zeros([1, 1]).astype(config.floatX))],
self.F,
on_unused_input='ignore')
from theano import In
from theano import function
import theano.tensor as T
x, y = T.dscalars('x', 'y')
z = x + y
f = function([x, In(y, value=1)], z)
# In class allows you to specify properties of your function's params with greater detail
print f(33)
print f(33, 2)
x, y, w = T.dscalars('x', 'y', 'w')
z = (x + y) * w
# The symbolic variable objects(ex. discalar) have name attributes
# and these are the names of the keyword params in the functions
#
# We can override the symbolic variable's name attribute with a name to be used for this function
f = function([x, In(y, value=1), In(w, value=2, name='w_by_name')], z)
print f(33)
print f(33, 2)
print f(33, 0, 1)
print f(33, w_by_name=1)
print f(33, w_by_name=1, y=0)
import numpy
import theano.tensor as T
import pygpu
from theano import function, pp, In
from matplotlib import pyplot
x = T.dscalar('x')
y = T.dscalar('y')
z = x + y
t = x * y
f = function([x, In(y, value=1)], [z, t])
print("device=", T.config.device)
print(f(2, 3))
print(numpy.allclose(f(16.3, 12.1), 28.4))
print(x.type)
print(type(x))
print(pp(z))
print(pp(x))
# x = T.dmatrix('x')
# s = 1 / (1 + T.exp(-x))
# logistic = function([x], s)
# print(pp(s))
# x = (logistic([[0, 1], [-1, -2]]))
# print(x)
#
import numpy
import theano.tensor as T
from theano import function
from theano import In
x, y, w = T.dscalars('x', 'y', 'w')
z = (x + 2 * y) * w
f = function([In(x, value=0),
In(y, value=0, name='y_name'),
In(w, value=1)], z)
print 'f(): ' + str(f())
print 'f(4): ' + str(f(4))
print 'f(y_name=3): ' + str(f(y_name=3))
print 'f(4, 3, 2): ' + str(f(4, 3, 2))
print 'f(4, 3): ' + str(f(4, 3))
print 'f(y_name=3, w = 4): ' + str(f(y_name=3, w=4))
def __init__(self,
rng,
n_in,
n_per_base,
n_out,
n_layer=1,
basefuncs1=None,
basefuncs2=None,
gradient=None,
with_shortcuts=False):
"""Initialize the parameters for the multilayer function graph
:type rng: numpy.random.RandomState
:param rng: a random number generator used to initialize weights
:type n_in: int
:param n_in: number of input units, the dimension of the space in
which the datapoints lie
:type n_layer: int
:param n_layer: number of hidden layers
:type n_per_base: int
:param n_per_base: number of nodes per basis function see FGLayer
:type n_out: int
:param n_out: number of output units, the dimension of the space in
which the labels lie
:type basefuncs1: [int]
:param basefuncs1: see FGLayer
:type basefuncs2: [int]
:param basefuncs2: see FGLayer
:type gradient: string
:param gradient: type of gradient descent algo (None=="sgd+","adagrad","adadelta","nag")
:type with_shortcuts: bool
:param with_shortcuts: whether to use shortcut connections (output is connected to all units)
"""
self.input = T.matrix('input') # the data is presented as vector input
self.labels = T.matrix(
'labels') # the labels are presented as vector of continous values
self.rng = rng
self.n_layers = n_layer
self.hidden_layers = []
self.params = []
self.n_in = n_in
self.n_out = n_out
self.with_shortcuts = with_shortcuts
self.fixL0 = False
for l in xrange(n_layer):
if l == 0:
layer_input = self.input
n_input = n_in
else:
layer_input = self.hidden_layers[l - 1].output
n_input = self.hidden_layers[l - 1].n_out
hiddenLayer = FGLayer(
rng=rng,
inp=layer_input,
n_in=n_input,
n_per_base=n_per_base,
basefuncs1=basefuncs1,
basefuncs2=basefuncs2,
layer_idx=l,
)
self.hidden_layers.append(hiddenLayer)
self.params.extend(hiddenLayer.params)
div_thresh = T.scalar("div_thresh")
# The linear output layer, either it gets as input the output of ALL previous layers
if self.with_shortcuts:
output_layer_inp = T.concatenate(
[l.output for l in reversed(self.hidden_layers)], axis=1)
output_layer_n_in = sum([l.n_out for l in self.hidden_layers])
else: # or just of the last hidden layer
output_layer_inp = self.hidden_layers[-1].output
output_layer_n_in = self.hidden_layers[-1].n_out
self.output_layer = DivisionRegression(rng=rng,
inp=output_layer_inp,
n_in=output_layer_n_in,
n_out=n_out,
div_thresh=div_thresh)
self.params.extend(self.output_layer.params)
self.evalfun = theano.function(
inputs=[self.input, In(div_thresh, value=0.0001)],
outputs=self.output_layer.output)
L1_reg = T.scalar('L1_reg')
L2_reg = T.scalar('L2_reg')
fixL0 = T.bscalar('fixL0')
self.L1 = self.output_layer.L1 + sum(
[l.L1 for l in self.hidden_layers])
self.L2_sqr = self.output_layer.L2_sqr + sum(
[l.L2_sqr for l in self.hidden_layers])
self.penalty = self.output_layer.penalty
self.loss = self.output_layer.loss
self.errors = self.loss
self.cost = (self.loss(self.labels) + L1_reg * self.L1 +
L2_reg * self.L2_sqr + self.penalty)
#Extrapol penalty
self.extrapol_cost = self.output_layer.extrapol_loss
learning_rate = T.scalar('learning_rate')
def process_updates(par, newp):
# print par.name
if par.name == "W":
# if fixL0 is True, then keep small weights at 0
return par, ifelse(
fixL0, T.switch(T.abs_(par) < 0.001, par * 0, newp), newp)
return par, newp
print "Gradient:", gradient
update = None
if gradient == 'sgd+' or gradient == 'sgd' or gradient == None:
gparams = [T.grad(self.cost, param) for param in self.params]
update = OrderedDict([
(param, param - (learning_rate * gparam).clip(-1.0, 1.0))
for param, gparam in zip(self.params, gparams)
])
elif gradient == 'adam':
update = Lupdates.adam(self.cost,
self.params,
learning_rate,
epsilon=1e-04)
elif gradient == 'adadelta':
update = Lupdates.adadelta(self.cost, self.params, learning_rate)
elif gradient == 'rmsprop':
update = Lupdates.rmsprop(self.cost, self.params, learning_rate)
elif gradient == 'nag':
update = Lupdates.nesterov_momentum(self.cost, self.params,
learning_rate)
else:
assert ("unknown gradient " + gradient)
#Extrapol sanity gradient computation:
extrapol_updates = Lupdates.adam(self.extrapol_cost,
self.params,
learning_rate,
epsilon=1e-04)
updates = [process_updates(*up) for up in update.items()]
self.train_model = theano.function(
inputs=[
self.input, self.labels, L1_reg, L2_reg, fixL0, learning_rate,
div_thresh
],
outputs=self.cost,
updates=updates,
)
# avoid too large outputs in extrapolation domain
self.remove_extrapol_error = theano.function(
inputs=[self.input, learning_rate, div_thresh],
outputs=self.extrapol_cost,
updates=extrapol_updates,
)
self.test_model = theano.function(
inputs=[self.input, self.labels,
In(div_thresh, value=0.0001)],
outputs=self.errors(self.labels),
)
self.validate_model = theano.function(
inputs=[self.input, self.labels,
In(div_thresh, value=0.0001)],
outputs=self.errors(self.labels),
)
self.L1_loss = theano.function(
inputs=[],
outputs=self.L1,
)
self.MSE = theano.function(
inputs=[self.input, self.labels,
In(div_thresh, value=0.0001)],
outputs=self.errors(self.labels),
)
it isn't that hard as it looks, just use the "In" class
"In" from theano accepts a variable and a "value=" which initializes default variable
if not present
You can think of a "In(variable_name, value=default_value)" class as an "Input" to a function
We will also use a "T.dscalars" macro, which creates multiple variables in one line, unlike the
"T.dscalar"
'''
from theano import In
x5, y5 = T.dscalars('x5', 'y5')
z5 = x5 + y5
func5 = function([x5, In(y5, value=4)], z5)
print(func5(32))
print(func5(20, 56.7))
assert func5(25) == 29
'''
and how to share a value between theano functions?
you can allocate memory space that will be accessible from any theano function, even after
it finishes work
the additional "set_value()" and "get_value()" help show and modify shared varaible's contents without a need of creting
a function
def predict(self,
X,
prediction_sample_size=250,
batchsize=360,
return_distrib=False,
train_mode=False,
return_std=False):
'''
:param prediction_sample_size: size of prediction sample of variables
:param return_distrib: whether to return a whole set of samples of only the mean value
'''
if not train_mode:
bar = tqdm(total=100)
# exception handling required for tqdm to work correctly
try:
pred_op = tt.mean(self.output.output, axis=0)
std_op = tt.sum(tt.sqrt(
tt.mean((self.output.output - pred_op)**2, axis=0)),
axis=-1)
pred_distrib = th.function([
self.input.input,
In(self.input.sample_size, value=prediction_sample_size)
], self.output.output)
pred = th.function([
self.input.input,
In(self.input.sample_size, value=prediction_sample_size)
], pred_op)
predstd = th.function([
self.input.input,
In(self.input.sample_size, value=prediction_sample_size)
], [pred_op, std_op])
# prepare data for feeding into the NN
nbatch = int(len(X) / batchsize) + 1
temp = []
stds = []
for i in range(nbatch):
if not train_mode:
bar.update(100. / nbatch)
if (i + 1) * batchsize > len(X) - 1:
batch = X[i * batchsize:].astype(dtype)
else:
batch = X[i * batchsize:(i + 1) * batchsize].astype(dtype)
if return_distrib:
preds = pred_distrib(batch)
else:
if return_std:
preds, std = predstd(batch)
stds.append(std)
else:
preds = pred(batch)
temp.append(preds)
if (i + 1) * batchsize > len(X) - 1:
break
finally:
if not train_mode:
bar.close()
if return_distrib:
return np.concatenate(temp, axis=1)
else:
if return_std:
return np.concatenate(temp, axis=0), np.concatenate(stds,
axis=0)
else:
return np.concatenate(temp, axis=0)
def fit(self,
X,
y,
nepoch,
batchsize,
log_freq=100,
valid_set=None,
shuffle_freq=1,
running_backup_dir=None,
scale_var_grad=1,
logfile=None):
if logfile:
logs = open(logfile, 'w')
sample_size = self.sample_size
# create input suitable for feeding into the input node
in_tens = X.astype(dtype)
in_tens_y = y.astype(dtype)
nbatch = int(len(X) / batchsize)
init_val = self.batch_iterated
batch_iterated_ph = th.shared(
np.array(self.batch_iterated, dtype=dtype),
'batch number placeholder')
repar_speed = th.shared(np.array(self.repar_speed, dtype=dtype),
'repar speed constant')
loss_scaler = 1 / (
1 + tt.exp(-(batch_iterated_ph - np.array(init_val, dtype)) *
repar_speed - np.array(init_val, dtype)))
if not self.loss_final:
loss = loss_scaler * self.match_loss + self.var_loss / (nbatch *
1.)
loss = loss / sample_size
self.loss = loss
self.loss_final = True
# remember batchsize in case of change
self.batchsize = batchsize
# reconfigure loss in case of batch size change
if self.loss_final and self.batchsize != batchsize:
loss = loss_scaler * self.match_loss + self.var_loss / (nbatch *
1.)
loss /= sample_size
self.loss = loss
self.batchsize = batchsize
obj_fun = th.function([
self.input.input, self.y,
In(self.input.sample_size, value=sample_size)
], self.objective / self.input.sample_size)
grad = th.grad(self.loss, self.weights)
# grad_scaler = np.ones(shape=(len(self.weights),), dtype=dtype)
for i in range(len(self.weights)):
if i % 2 == 1:
grad[i] *= scale_var_grad
# grad_scaler_th = tt.constant(grad_scaler, name='gradient scaler')
# grad *= grad_scaler_th
train = th.function([
self.input.input, self.y,
In(self.input.sample_size, value=sample_size)
],
updates=self.updates(grad, self.weights))
to_write = None
try:
for epoch in range(nepoch):
# update the number of passed epochs
self.batch_iterated += 1
if loss_scaler.eval() < 1 - 0.0001:
batch_iterated_ph.set_value(
np.array(self.batch_iterated, dtype=dtype))
# print logs every log_freq epochs:
if epoch % log_freq == 0:
preds = self.predict(in_tens,
prediction_sample_size=100,
train_mode=True)
train_mse = self.loss_func(preds, in_tens_y)
obj = obj_fun(in_tens, in_tens_y)
if valid_set is not None:
preds, std = self.predict(valid_set[0].astype(dtype),
prediction_sample_size=100,
train_mode=True,
return_std=True)
valid_mse = self.loss_func(preds, valid_set[1])
losses = self.loss_func_nf(preds, valid_set[1])
corr = np.sum(
(losses - np.mean(losses)) * (std - np.mean(std)) /
(np.std(std) * np.std(losses)))
logstr = 'epoch: {} \n train error: {} \n valid_error: {} \n objective: {}\n loss_scale: {}\n loss-std corr: {}\n\n'.format(
epoch, train_mse, valid_mse, obj,
loss_scaler.eval(), corr)
else:
logstr = 'epoch: {} \n train error: {} \n objective: {}\n loss_scale: {}\n\n'.format(
epoch, train_mse, obj, loss_scaler.eval())
#print('epoch: {} \n objective: {}\n\n\n'.format(epoch, obj))
print(logstr)
if logfile:
logs.write(logstr)
# record NN weights if the backup dir is set:
if running_backup_dir is not None:
if valid_set is not None:
self.save(running_backup_dir +
'runnung_tr{}_test{}.npy'.format(
train_mse, valid_mse))
else:
self.save(running_backup_dir +
'runnung_tr{}.npy'.format(train_mse))
for i in range(nbatch):
train(in_tens[batchsize * i:batchsize * (i + 1), :],
in_tens_y[batchsize * i:batchsize * (i + 1), :])
# shuffle data every shuffle_freq epochs
if shuffle_freq is not None:
if epoch % shuffle_freq == 0:
shuffle = np.random.permutation(in_tens.shape[0])
# not running gc right after shuffle causes memory leak
gc.collect()
in_tens = in_tens[shuffle, :]
in_tens_y = in_tens_y[shuffle, :]
except (Exception, BaseException) as exc:
to_write = traceback.format_exc(exc)
raise exc
finally:
if logfile:
if to_write:
logs.write(to_write)
logs.close()
import numpy as np
from theano import function as fn
from theano import In
import theano.tensor as T
import matplotlib.pyplot as plt
a,b = T.dmatrices('a','b')
diff = a-b
abs_diff = abs(diff)
diff_sr = diff ** 2
f = fn( [a,b], [diff, abs_diff, diff_sr])
n = [ [11,22], [22,11] ]
m = [ [1,2], [3,4] ]
print( f(m,n) )
x,y,z = f(n,m)
# we coul try out a default value.
f2 = fn( [a, In(b, value = [ [0,0], [0,0] ]) ], diff_sr)
def __init__(self, config, testMode):
self.config = config
batch_size = config['batch_size']
lib_conv = config['lib_conv']
useLayers = config['useLayers']
#imgWidth = config['imgWidth']
#imgHeight = config['imgHeight']
initWeights = config['initWeights'] #if we wish to initialize alexnet with some weights. #need to make changes in layers.py to accept initilizing weights
if initWeights:
weightsDir = config['weightsDir']
weightFileTag = config['weightFileTag']
prob_drop = config['prob_drop']
# ##################### BUILD NETWORK ##########################
x = T.ftensor4('x')
mean = T.ftensor4('mean')
#y = T.lvector('y')
print '... building the model'
self.layers = []
params = []
weight_types = []
if useLayers >= 1:
convpool_layer1 = ConvPoolLayer(input=x-mean,
image_shape=(3, None, None, batch_size),
filter_shape=(3, 11, 11, 96),
convstride=4, padsize=0, group=1,
poolsize=3, poolstride=2,
bias_init=0.0, lrn=True,
lib_conv=lib_conv,
initWeights=initWeights, weightsDir=weightsDir, weightFiles=['W_0'+weightFileTag, 'b_0'+weightFileTag]
)
self.layers.append(convpool_layer1)
params += convpool_layer1.params
weight_types += convpool_layer1.weight_type
if useLayers >= 2:
convpool_layer2 = ConvPoolLayer(input=convpool_layer1.output,
image_shape=(96, None, None, batch_size), #change from 27 to appropriate value sbased on conv1's output
filter_shape=(96, 5, 5, 256),
convstride=1, padsize=2, group=2,
poolsize=3, poolstride=2,
bias_init=0.1, lrn=True,
lib_conv=lib_conv,
initWeights=initWeights, weightsDir=weightsDir, weightFiles=['W0_1'+weightFileTag, 'W1_1'+weightFileTag, 'b0_1'+weightFileTag, 'b1_1'+weightFileTag]
)
self.layers.append(convpool_layer2)
params += convpool_layer2.params
weight_types += convpool_layer2.weight_type
if useLayers >= 3:
convpool_layer3 = ConvPoolLayer(input=convpool_layer2.output,
image_shape=(256, None, None, batch_size),
filter_shape=(256, 3, 3, 384),
convstride=1, padsize=1, group=1,
poolsize=1, poolstride=0,
bias_init=0.0, lrn=False,
lib_conv=lib_conv,
initWeights=initWeights, weightsDir=weightsDir, weightFiles=['W_2'+weightFileTag, 'b_2'+weightFileTag]
)
self.layers.append(convpool_layer3)
params += convpool_layer3.params
weight_types += convpool_layer3.weight_type
if useLayers >= 4:
convpool_layer4 = ConvPoolLayer(input=convpool_layer3.output,
image_shape=(384, None, None, batch_size),
filter_shape=(384, 3, 3, 384),
convstride=1, padsize=1, group=2,
poolsize=1, poolstride=0,
bias_init=0.1, lrn=False,
lib_conv=lib_conv,
initWeights=initWeights, weightsDir=weightsDir, weightFiles=['W0_3'+weightFileTag, 'W1_3'+weightFileTag, 'b0_3'+weightFileTag, 'b1_3'+weightFileTag]
)
self.layers.append(convpool_layer4)
params += convpool_layer4.params
weight_types += convpool_layer4.weight_type
if useLayers >= 5:
convpool_layer5 = ConvPoolLayer(input=convpool_layer4.output,
image_shape=(384, None, None, batch_size),
filter_shape=(384, 3, 3, 256),
convstride=1, padsize=1, group=2,
poolsize=3, poolstride=2,
bias_init=0.0, lrn=False,
lib_conv=lib_conv,
initWeights=initWeights, weightsDir=weightsDir, weightFiles=['W0_4'+weightFileTag, 'W1_4'+weightFileTag, 'b0_4'+weightFileTag, 'b1_4'+weightFileTag]
)
self.layers.append(convpool_layer5)
params += convpool_layer5.params
weight_types += convpool_layer5.weight_type
if useLayers >= 6:
fc_layer6_input = T.flatten(convpool_layer5.output.dimshuffle(3, 0, 1, 2), 2)
fc_layer6 = FCLayer(input=fc_layer6_input, n_in=9216, n_out=4096, initWeights=initWeights, weightsDir=weightsDir, weightFiles=['W_5'+weightFileTag, 'b_5'+weightFileTag])
self.layers.append(fc_layer6)
params += fc_layer6.params
weight_types += fc_layer6.weight_type
if testMode:
dropout_layer6 = fc_layer6
else:
dropout_layer6 = DropoutLayer(fc_layer6.output, n_in=4096, n_out=4096, prob_drop=prob_drop)
if useLayers >= 7:
fc_layer7 = FCLayer(input=dropout_layer6.output, n_in=4096, n_out=4096, initWeights=initWeights, weightsDir=weightsDir, weightFiles=['W_6'+weightFileTag, 'b_6'+weightFileTag])
self.layers.append(fc_layer7)
params += fc_layer7.params
weight_types += fc_layer7.weight_type
if testMode:
dropout_layer6 = fc_layer7
else:
dropout_layer7 = DropoutLayer(fc_layer7.output, n_in=4096, n_out=4096, prob_drop=prob_drop)
if useLayers >= 8:
softmax_layer8 = SoftmaxLayer(input=dropout_layer7.output, n_in=4096, n_out=1000, initWeights=initWeights, weightsDir=weightsDir, weightFiles=['W_7'+weightFileTag, 'b_7'+weightFileTag])
self.layers.append(softmax_layer8)
params += softmax_layer8.params
weight_types += softmax_layer8.weight_type
# #################### NETWORK BUILT #######################
self.output = self.layers[useLayers-1]
self.params = params
self.x = x
self.mean = mean
self.weight_types = weight_types
self.batch_size = batch_size
self.useLayers = useLayers
self.outLayer = self.layers[useLayers-1]
meanVal = np.load(config['mean_file'])
meanVal = meanVal[:, :, :, np.newaxis].astype('float32') #x is 4d, with 'batch' number of images. meanVal has only '1' in the 'batch' dimension. subtraction wont work.
meanVal = np.tile(meanVal,(1,1,1,batch_size))
self.meanVal = meanVal
#meanVal = np.zeros([3,imgHeight,imgWidth,2], dtype='float32')
if useLayers >= 8: #if last layer is softmax, then its output is y_pred
finalOut = self.outLayer.y_pred
else:
finalOut = self.outLayer.output
self.forwardFunction = theano.function([self.x, In(self.mean, value=meanVal)], [finalOut])
z_tanh1 = (T.exp(x) - T.exp(-x)) / (T.exp(x) + T.exp(-x))
# tanh to sigmoid
z_tanh2sig = (1 + T.tanh(x / 2)) / 2
sig_tanh = function([x], [z_sig, z_tanh0, z_tanh1, z_tanh2sig])
a = np.random.randint(-2, 2, (2, 2))
print(a)
print(
sig_tanh(a)[0], '\n',
sig_tanh(a)[1], '\n',
sig_tanh(a)[2], '\n',
sig_tanh(a)[3])
# 为参数设置默认值
x, y, w = T.dscalars('x', 'y', 'w')
z = (x + y) * w
f = function([x, In(y, value=1), In(w, value=2, name='w_default')], z)
print(f(3)) # (3+1)*2
print(f(3, 4)) # (3+4)*2
print(f(3, 4, 3)) # (3+4)*3
# 通过变量名称修改默认
print(f(3, 4, w_default=5)) # (3+4)*5
# 使用共享变量(shared variable)
state = shared(0)
# 判断输入是否为标量
inc = T.iscalar('inc')
# 如果是标量，执行状态更新
accumulator = function([inc], state, updates=[(state, state + inc)])
# 更新状态
print(state.get_value()) # 0
accumulator(1)
"""
2016-07-04-15:52:19
arhik
"""
"""
The aim of this script is to show you how to set the default values of an
Argument
"""
import theano.tensor as T
from theano import In, function
x, y = T.dscalars('x', 'y')
z = x + y
f = function([x, In(y, value=1)], z)
print(f(23))
from theano import In
import numpy as np
x = T.dscalar('x')
y = T.dscalar('y')
z = x + y
fn = f([x,y], z)
#Returns True if two arrays are element-wise equal within a tolerance.
print(np.allclose(fn(33.0003, 33.0003), 66.0006))
print ( 'This will not work now: print(fn([1,2], [3,4]))')
print('\n because those are scalars, not matrices')
x = T.dmatrix('x')
y = T.dmatrix('y')
z = x+y
fn = f([x, y], z)
# default values
x,y = T.dscalars('x','y')
z = x + y
fn = f( [x, In(y,value=0)],z)