def score(self, x):
prehidden = t.dot(
x, self.parameters.hidden_weights) + self.parameters.hidden_biases
hidden = t.clip(prehidden, -1, 1)
score = t.dot(
hidden,
self.parameters.output_weights) + self.parameters.output_biases
return score, prehidden
def functions(self, sequence_length):
key = (sequence_length)
if key not in self.cache:
logging.info("Need to construct graph for sequence_length=%d..." % (sequence_length))
# creating network input variable nodes
correct_inputs = t.ftensor3("correct input")
noise_inputs = t.ftensor3("noise input")
learning_rate = t.fscalar("learning rate")
# creating op nodes for firing the network
correct_score, correct_prehidden = self.score(correct_inputs)
noise_score, noise_prehidden = self.score(noise_inputs)
# creating op nodes for the pairwise ranking cost function
loss = t.clip(1 - correct_score + noise_score, 0, 1e999)
total_loss = t.sum(loss)
# the necessary cost function gradients
parameters_gradient = grad(total_loss, list(self.parameters))
correct_inputs_gradient = grad(total_loss, correct_inputs)
noise_inputs_gradient = grad(total_loss, noise_inputs)
# setting network inputs
predict_inputs = [correct_inputs]
train_inputs = [correct_inputs, noise_inputs, learning_rate]
verbose_predict_inputs = predict_inputs
# setting network outputs
predict_outputs = [correct_score]
train_outputs = [correct_inputs_gradient, noise_inputs_gradient, loss, correct_score, noise_score]
verbose_predict_outputs = [correct_score, correct_prehidden]
nnodes = len(theano.gof.graph.ops(predict_inputs, predict_outputs))
logging.info("About to compile prediction function over %d ops [nodes]..." % nnodes)
predict = theano.function(predict_inputs, predict_outputs, mode=COMPILE_MODE)
logging.info("...done constructing graph for sequence_length=%d" % (sequence_length))
nnodes = len(theano.gof.graph.ops(verbose_predict_inputs, verbose_predict_outputs))
logging.info("About to compile verbose prediction function over %d ops [nodes]..." % nnodes)
verbose_predict = theano.function(verbose_predict_inputs, verbose_predict_outputs, mode=COMPILE_MODE)
logging.info("...done constructing graph for sequence_length=%d" % (sequence_length))
nnodes = len(theano.gof.graph.ops(train_inputs, train_outputs))
logging.info("About to compile training function over %d ops [nodes]..." % nnodes)
train = theano.function(train_inputs, train_outputs, mode=COMPILE_MODE, updates=[(p, p - learning_rate * gp) for p, gp in zip(list(self.parameters), parameters_gradient)])
logging.info("...done constructing graph for sequence_length=%d" % (sequence_length))
self.cache[key] = (predict, train, verbose_predict)
return self.cache[key]
def hard_sigmoid(x):
"""An approximation of sigmoid.
More approximate and faster than ultra_fast_sigmoid.
Approx in 3 parts: 0, scaled linear, 1
Removing the slope and shift does not make it faster.
"""
slope = 0.2
shift = 0.5
x = (x * slope) + shift
x = tensor.clip(x, 0, 1)
return x
def hard_sigmoid(x):
"""An approximation of sigmoid.
More approximate and faster than ultra_fast_sigmoid.
Approx in 3 parts: 0, scaled linear, 1
Removing the slope and shift does not make it faster.
"""
slope = 0.2
shift = 0.5
x = (x * slope) + shift
x = tensor.clip(x, 0, 1)
return x
def hard_sigmoid(x):
"""An approximation of sigmoid.
More approximate and faster than ultra_fast_sigmoid.
Approx in 3 parts: 0, scaled linear, 1
Removing the slope and shift does not make it faster.
"""
# Use the same dtype as determined by "upgrade_to_float",
# and perform computation in that dtype.
out_dtype = scalar.upgrade_to_float(scalar.Scalar(dtype=x.dtype))[0].dtype
slope = tensor.constant(0.2, dtype=out_dtype)
shift = tensor.constant(0.5, dtype=out_dtype)
x = (x * slope) + shift
x = tensor.clip(x, 0, 1)
return x
def functions(self, sequence_length):
key = (sequence_length)
if key not in self.cache:
logging.info("Need to construct graph for sequence_length=%d..." %
(sequence_length))
# creating network input variable nodes
correct_inputs = t.ftensor3("correct input")
noise_inputs = t.ftensor3("noise input")
learning_rate = t.fscalar("learning rate")
# creating op nodes for firing the network
correct_score, correct_prehidden = self.score(correct_inputs)
noise_score, noise_prehidden = self.score(noise_inputs)
# creating op nodes for the pairwise ranking cost function
loss = t.clip(1 - correct_score + noise_score, 0, 1e999)
total_loss = t.sum(loss)
# the necessary cost function gradients
parameters_gradient = grad(total_loss, list(self.parameters))
correct_inputs_gradient = grad(total_loss, correct_inputs)
noise_inputs_gradient = grad(total_loss, noise_inputs)
# setting network inputs
predict_inputs = [correct_inputs]
train_inputs = [correct_inputs, noise_inputs, learning_rate]
verbose_predict_inputs = predict_inputs
# setting network outputs
predict_outputs = [correct_score]
train_outputs = [
correct_inputs_gradient, noise_inputs_gradient, loss,
correct_score, noise_score
]
verbose_predict_outputs = [correct_score, correct_prehidden]
nnodes = len(theano.gof.graph.ops(predict_inputs, predict_outputs))
logging.info(
"About to compile prediction function over %d ops [nodes]..." %
nnodes)
predict = theano.function(predict_inputs,
predict_outputs,
mode=COMPILE_MODE)
logging.info("...done constructing graph for sequence_length=%d" %
(sequence_length))
nnodes = len(
theano.gof.graph.ops(verbose_predict_inputs,
verbose_predict_outputs))
logging.info(
"About to compile verbose prediction function over %d ops [nodes]..."
% nnodes)
verbose_predict = theano.function(verbose_predict_inputs,
verbose_predict_outputs,
mode=COMPILE_MODE)
logging.info("...done constructing graph for sequence_length=%d" %
(sequence_length))
nnodes = len(theano.gof.graph.ops(train_inputs, train_outputs))
logging.info(
"About to compile training function over %d ops [nodes]..." %
nnodes)
train = theano.function(
train_inputs,
train_outputs,
mode=COMPILE_MODE,
updates=[(p, p - learning_rate * gp) for p, gp in zip(
list(self.parameters), parameters_gradient)])
logging.info("...done constructing graph for sequence_length=%d" %
(sequence_length))
self.cache[key] = (predict, train, verbose_predict)
return self.cache[key]
def score(self,x):
prehidden = t.dot(x, self.parameters.hidden_weights) + self.parameters.hidden_biases
hidden = t.clip(prehidden, -1, 1)
score = t.dot(hidden, self.parameters.output_weights) + self.parameters.output_biases
return score, prehidden