def main():
train_images, train_labels, test_images, test_labels = load_mnist()
X = normalize(train_images)
label_size = len(np.unique(train_labels))
y = one_hot_vector(train_labels, label_size)
print("Total training example:", X.shape[0])
nn = NN(epoch=20, batch_size=256)
nn.add_layer(Layer(784))
nn.add_layer(Layer(200, activation_fn=relu))
nn.add_layer(Layer(100, activation_fn=relu))
nn.add_layer(Layer(10, activation_fn=softmax))
nn.fit(X, y)
print("Train Accuracy is:", nn.accuracy(X, y))
X_test = normalize(test_images)
Y_test = one_hot_vector(test_labels, label_size)
print("Test Accuracy is:", nn.accuracy(X_test, Y_test))
nn.plot_learning_curve()
def ready(self):
args = self.args
index = self.index = T.lscalar()
x = self.x = T.fmatrix()
y = self.y = T.ivector()
dropout = self.dropout = theano.shared(
np.float64(args.dropout).astype("float32"))
n_d = args.hidden_dim
layers = self.layers = []
for i in xrange(args.depth):
l = Layer(n_in=28 * 28 if i == 0 else n_d,
n_out=n_d,
activation=ReLU)
layers.append(l)
output_layer = self.output_layer = Layer(n_in=n_d,
n_out=10,
activation=softmax)
h = x
for l in layers:
h = l.forward(h)
h = apply_dropout(h, dropout)
self.h_final = h
# batch * 10
probs = self.probs = output_layer.forward(h)
# batch
preds = self.preds = T.argmax(probs, axis=1)
err = self.err = T.mean(T.cast(T.neq(preds, y), dtype="float32"))
#
loss = self.loss = -T.mean(T.log(probs[T.arange(y.shape[0]), y]))
#loss = self.loss = T.mean( T.nnet.categorical_crossentropy(
# probs,
# y
# ))
params = self.params = []
for l in layers + [output_layer]:
for p in l.params:
params.append(p)
l2_cost = None
for p in params:
if l2_cost is None:
l2_cost = T.sum(p**2)
else:
l2_cost += T.sum(p**2)
l2_cost = l2_cost * args.l2_reg
self.l2_cost = l2_cost
self.cost = loss + l2_cost
print "cost.dtype", self.cost.dtype
def ready(self, args, train):
# len * batch
self.idxs = T.imatrix()
self.idys = T.imatrix()
self.init_state = T.matrix(dtype=theano.config.floatX)
dropout_prob = np.float64(args["dropout"]).astype(theano.config.floatX)
self.dropout = theano.shared(dropout_prob)
self.n_d = args["hidden_dim"]
embedding_layer = EmbeddingLayer(n_d=self.n_d,
vocab=set(w for w in train))
self.n_V = embedding_layer.n_V
say("Vocab size: {}\tHidden dim: {}\n".format(self.n_V, self.n_d))
activation = get_activation_by_name(args["activation"])
rnn_layer = LSTM(n_in=self.n_d, n_out=self.n_d, activation=activation)
output_layer = Layer(
n_in=self.n_d,
n_out=self.n_V,
activation=T.nnet.softmax,
)
# (len*batch) * n_d
x_flat = embedding_layer.forward(self.idxs.ravel())
# len * batch * n_d
x = apply_dropout(x_flat, self.dropout)
x = x.reshape((self.idxs.shape[0], self.idxs.shape[1], self.n_d))
# len * batch * (n_d+n_d)
h = rnn_layer.forward_all(x, self.init_state, return_c=True)
self.last_state = h[-1]
h = h[:, :, self.n_d:]
h = apply_dropout(h, self.dropout)
self.p_y_given_x = output_layer.forward(h.reshape(x_flat.shape))
idys = self.idys.ravel()
self.nll = -T.log(self.p_y_given_x[T.arange(idys.shape[0]), idys])
#self.nll = T.nnet.categorical_crossentropy(
# self.p_y_given_x,
# idys
# )
self.layers = [embedding_layer, rnn_layer, output_layer]
#self.params = [ x_flat ] + rnn_layer.params + output_layer.params
self.params = embedding_layer.params + rnn_layer.params + output_layer.params
self.num_params = sum(
len(x.get_value(borrow=True).ravel()) for l in self.layers
for x in l.params)
say("# of params in total: {}\n".format(self.num_params))
def build_model(self):
args = self.args
weights = self.weights
meta_emb = self.meta_emb = self.embs[0]
golden_embs = self.embs[1:]
n_m_d = meta_emb.n_d
dropout = self.dropout = theano.shared(
np.float64(args.dropout_rate).astype(theano.config.floatX))
batch_ids = self.batch_ids = T.ivector('batch_d_char')
batch_masks = self.batch_masks = T.fmatrix('batch_d_char_mask')
layers = self.layers = [meta_emb]
slices_embs = meta_emb.forward(batch_ids.ravel())
slices_embs = slices_embs.reshape((batch_ids.shape[0], n_m_d))
prev_output = apply_dropout(slices_embs, dropout, v2=True)
self.all_loss = 0.0
for i in range(len(weights)):
mask, weight, golden_emb = batch_masks[i], weights[i], golden_embs[
i]
n_o_d = golden_emb.n_d
layer = Layer(n_m_d, n_o_d, linear)
layers.append(layer)
mapped_output = layer.forward(prev_output)
slices_embs = golden_emb.forward(batch_ids.ravel())
slices_embs = slices_embs.reshape((batch_ids.shape[0], n_o_d))
self.all_loss += weight * T.sum(
T.sum((mapped_output - slices_embs) *
(mapped_output - slices_embs),
axis=1) * mask) / (1e-8 + T.sum(mask))
for i, l in enumerate(layers[1:]):
say("layer {}: n_in={}\tn_out={}\n".format(i, l.n_in, l.n_out))
self.l2_sqr = None
self.params = []
for layer in layers:
self.params += layer.params
for p in self.params:
if self.l2_sqr is None:
self.l2_sqr = args.l2_reg * T.sum(p**2)
else:
self.l2_sqr += args.l2_reg * T.sum(p**2)
self.all_loss += self.l2_sqr
n_params = sum(
len(x.get_value(borrow=True).ravel()) for x in self.params)
say("total # parameters: {}\n".format(n_params))
def __init__(self,
n_in,
n_out,
activation,
highway=True,
dropout=None,
combine_c=True):
self.n_in, self.n_out = n_in, n_out
self.highway = highway
self.activation = activation
self.dropout = dropout
self.combine_c = combine_c
self.lambda_gate = RecurrentLayer(n_in, n_out, sigmoid)
self.input_layer_1 = Layer(n_in, n_out, linear, has_bias=False)
self.input_layer_2 = Layer(n_in, n_out, linear, has_bias=False)
if highway:
self.highway_layer = HighwayLayer(n_out)
class HighwayLayer(object):
def __init__(self, n_d):
self.n_d = n_d
self.gate = Layer(n_d, n_d, sigmoid)
def forward(self, x, h):
t = self.gate.forward(x)
return h * t + x * (1 - t)
@property
def params(self):
return self.gate.params
@params.setter
def params(self, param_list):
self.gate.params = param_list
def ready(self):
index = self.index = T.lscalar()
x = self.x = T.fmatrix()
y = self.y = T.ivector()
layer = self.layer = Layer(
n_in = 28*28,
n_out = 10,
activation = softmax
)
# batch * 10
probs = self.probs = layer.forward(x)
# batch
preds = self.preds = T.argmax(probs, axis=1)
err = self.err = T.mean(T.cast(T.neq(preds, y), dtype="float32"))
#
loss = self.loss = - T.mean( T.log(probs[T.arange(y.shape[0]),y]) )
#loss = self.loss = T.mean( T.nnet.categorical_crossentropy(
# probs,
# y
# ))
self.params = layer.params
l2_cost = None
for p in self.params:
if l2_cost is None:
l2_cost = T.sum(p**2)
else:
l2_cost += T.sum(p**2)
l2_cost = l2_cost * args.l2_reg
self.l2_cost = l2_cost
self.cost = loss + l2_cost
print "cost.dtype", self.cost.dtype
# import matplotlib.pyplot as plt
import numpy as np
def col(a):
"""Make sure the array is a column."""
return np.atleast_2d(a).T
N = 100_000 # Create N points inside the square [-2,2]×[-2,2]
data = np.random.uniform(low=-2, high=2, size=(2, N))
ts = np.zeros(shape=(2, N))
ts[0, data[0,]*data[1,]>0] = 1
ts[1, :] = 1 - ts[0, :]
net = NeuralNetwork([
Layer(2, 3, LeakyReLU()),
Layer(3, 2, LeakyReLU()),
], MSELoss(), 0.05)
def assess(net, data, ts):
correct = 0
cs = []
for i in range(data.shape[1]):
out = net.forward_pass(col(data[:, i]))
guess = np.argmax(np.ndarray.flatten(out))
if ts[guess, i]:
correct += 1
cs.append(guess)
# fig = plt.figure()
# plt.scatter(data[0, :1000], data[1, :1000], c=cs)
# fig.show()
def ready(self):
args = self.args
#n_domain = 2
accum_dict = self.accum_dict = {}
# len(sent) * len(doc) * batch
s_idxs = self.s_idxs = T.itensor3()
t_idxs = self.t_idxs = T.itensor3()
# batch
s_idys = self.s_idys = T.ivector()
t_idys = self.t_idys = T.ivector()
# batch
s_dom_ids = self.s_dom_ids = T.ivector()
t_dom_ids = self.t_dom_ids = T.ivector()
# len(doc) * batch, 0: negative, 1: positive, -1: REL_UNK, -2, REL_PAD
s_gold_rels = self.s_gold_rels = T.imatrix()
t_gold_rels = self.t_gold_rels = T.imatrix()
# has label flag, 0: no, 1: yes
s_has_lab = self.s_has_lab = T.iscalar()
t_has_lab = self.t_has_lab = T.iscalar()
self.dropout = theano.shared(np.float64(args.dropout).astype(
theano.config.floatX))
embedding_layer = self.embedding_layer
if not embedding_layer.fix_init_embs:
accum_dict[embedding_layer] = self.create_accumulators(embedding_layer)
activation = get_activation_by_name(args.activation)
n_d = self.n_d = args.hidden_dim
n_e = self.n_e = embedding_layer.n_d
n_c = self.nclasses
self.rho = theano.shared(np.float64(0.0).astype(theano.config.floatX))
self.source_k = 2
# CNN to encode sentence into embedding
cnn_layer = self.cnn_layer = LeCNN(
n_in = n_e,
n_out = n_d,
activation=activation,
order = args.cnn_window_size,
BN = True,
)
accum_dict[cnn_layer] = self.create_accumulators(cnn_layer)
# softmax layer to predict the label of the document
self.lab_hid_layer = lab_hid_layer = Layer(
n_in = n_d,
n_out = n_d,
activation = activation,
)
accum_dict[lab_hid_layer] = self.create_accumulators(lab_hid_layer)
self.lab_out_layer = lab_out_layer = Layer(
n_in = n_d,
n_out = n_c,
activation = logsoftmax,
)
accum_dict[lab_out_layer] = self.create_accumulators(lab_out_layer)
# hidden layer to predict the domain of the document
dom_hid_layer = self.dom_hid_layer = Layer(
n_in = n_d,
n_out = n_d,
activation = activation,
)
accum_dict[dom_hid_layer] = self.create_accumulators(dom_hid_layer)
# softmax layer to predict the domain of the document
dom_out_layer = self.dom_out_layer = Layer(
n_in = n_d,
n_out = 2,
activation = logsoftmax,
)
accum_dict[dom_out_layer] = self.create_accumulators(dom_out_layer)
# for each domain, a vector parameter to compute the relevance score
rel_hid_layer = self.rel_hid_layer = Layer(
n_in = n_d,
n_out = n_d,
activation = activation,
)
accum_dict[rel_hid_layer] = self.create_accumulators(rel_hid_layer)
s_rel_out_layer = self.s_rel_out_layer = Layer(
n_in = n_d,
n_out = 1,
activation = sigmoid,
)
accum_dict[s_rel_out_layer] = self.create_accumulators(s_rel_out_layer)
t_rel_out_layer = self.t_rel_out_layer = Layer(
n_in = n_d,
n_out = 1,
activation = sigmoid,
)
accum_dict[t_rel_out_layer] = self.create_accumulators(t_rel_out_layer)
# transformation to domain independent layer
trans_layer = self.trans_layer = Layer(
n_in = n_d,
n_out = n_d,
activation = activation,
has_bias=False,
init_zero=True,
)
accum_dict[trans_layer] = self.create_accumulators(trans_layer)
val = np.eye(n_d, dtype=theano.config.floatX)
identity_mat = theano.shared(val)
trans_layer.W.set_value(val)
# reconstruction layer
recon_layer = self.recon_layer = Layer(
n_in = n_d,
n_out = n_e,
activation = tanh,
)
accum_dict[recon_layer] = self.create_accumulators(recon_layer)
# construct network
s_lab_loss, s_rel_loss, s_dom_loss, s_adv_loss, s_lab_prob, s_recon_loss = self.ready_one_domain(
s_idxs, s_idys, s_dom_ids, s_gold_rels, \
cnn_layer, rel_hid_layer, s_rel_out_layer, trans_layer, \
dom_hid_layer, dom_out_layer, lab_hid_layer, lab_out_layer)
self.s_lab_loss, self.s_rel_loss, self.s_dom_loss, self.s_adv_loss, self.s_lab_prob, self.s_recon_loss = \
s_lab_loss, s_rel_loss, s_dom_loss, s_adv_loss, s_lab_prob, s_recon_loss
t_lab_loss, t_rel_loss, t_dom_loss, t_adv_loss, t_lab_prob, t_recon_loss = self.ready_one_domain(
t_idxs, t_idys, t_dom_ids, t_gold_rels, \
cnn_layer, rel_hid_layer, t_rel_out_layer, trans_layer, \
dom_hid_layer, dom_out_layer, lab_hid_layer, lab_out_layer)
self.t_lab_loss, self.t_rel_loss, self.t_dom_loss, self.t_adv_loss, self.t_lab_prob, self.t_recon_loss = \
t_lab_loss, t_rel_loss, t_dom_loss, t_adv_loss, t_lab_prob, t_recon_loss
# transformation regularization
trans_reg = self.trans_reg = args.trans_reg * T.sum((trans_layer.W - identity_mat) ** 2)
# domain cost
layers = [ dom_out_layer, dom_hid_layer ]
self.dom_params = self.get_params(layers)
self.dom_accums = self.get_accumulators(layers, accum_dict)
self.dom_cost = s_dom_loss + t_dom_loss + args.l2_reg * self.get_l2_cost(self.dom_params)
# label cost
lab_layers = [ lab_out_layer, lab_hid_layer ]
lab_params = self.get_params(lab_layers)
lab_cost = s_has_lab * self.source_k * s_lab_loss + t_has_lab * t_lab_loss \
+ args.l2_reg * (s_has_lab + t_has_lab) * self.get_l2_cost(lab_params)
# total cost
other_layers = [ cnn_layer, s_rel_out_layer, t_rel_out_layer, rel_hid_layer, trans_layer, recon_layer ]
other_params = self.get_params(other_layers)
self.other_cost_except_dom = lab_cost + s_rel_loss + t_rel_loss + s_adv_loss + t_adv_loss + trans_reg \
+ s_recon_loss + t_recon_loss \
+ args.l2_reg * self.get_l2_cost(other_params)
self.other_params_except_dom = lab_params + other_params
self.other_accums_except_dom = self.get_accumulators(lab_layers + other_layers, accum_dict)
if not embedding_layer.fix_init_embs:
self.other_params_except_dom += embedding_layer.params
self.add_accumulators(self.other_accums_except_dom, embedding_layer, accum_dict)
# info
layers = lab_layers + other_layers + [ dom_out_layer, dom_hid_layer ]
params = self.params = self.get_params(layers)
if not embedding_layer.fix_init_embs:
self.params += embedding_layer.params
say("num of parameters: {}\n".format(
sum(len(x.get_value(borrow=True).ravel()) for x in params)
))
for digit in range(10):
t = np.zeros((10, 1))
t[digit] = 1
ts[digit] = t
for i, row in enumerate(data):
if i % 1000 == 0:
print(i)
digit = row[0]
x = row[1:].reshape((784, 1))
net.train(x, ts[digit])
if __name__ == "__main__":
layers = [
Layer(784, 16, LeakyReLU()),
Layer(16, 16, LeakyReLU()),
Layer(16, 10, LeakyReLU()),
]
net = NeuralNetwork(layers, MSELoss(), 0.001)
# CrossEntropyLoss ← um pouco mais chato
# Sigmoid ← wikipedia
print("Loading data...")
train_data = load_data("mnistdata/mnist_train.csv")
print("Done.")
print("Training network...")
train(net, train_data)
print("Done.")
def ready(self):
global total_generate_time
#say("in generator ready: \n")
#start_generate_time = time.time()
embedding_layer = self.embedding_layer
args = self.args
padding_id = embedding_layer.vocab_map["<padding>"]
dropout = self.dropout = theano.shared(
np.float64(args.dropout).astype(theano.config.floatX))
# len*batch
x = self.x = T.imatrix()
n_d = args.hidden_dimension
n_e = embedding_layer.n_d
activation = get_activation_by_name(args.activation)
layers = self.layers = []
layer_type = args.layer.lower()
for i in xrange(2):
if layer_type == "rcnn":
l = RCNN(n_in=n_e,
n_out=n_d,
activation=activation,
order=args.order)
elif layer_type == "lstm":
l = LSTM(n_in=n_e, n_out=n_d, activation=activation)
l = Layer(n_in=n_e, n_out=n_d, activation=sigmoid)
layers.append(l)
# len * batch
#masks = T.cast(T.neq(x, padding_id), theano.config.floatX)
masks = T.cast(T.neq(x, padding_id), theano.config.floatX).dimshuffle(
(0, 1, "x"))
# (len*batch)*n_e
embs = embedding_layer.forward(x.ravel())
# len*batch*n_e
embs = embs.reshape((x.shape[0], x.shape[1], n_e))
embs = apply_dropout(embs, dropout)
self.word_embs = embs
flipped_embs = embs[::-1]
# len*bacth*n_d
h1 = layers[0].forward(embs)
h2 = layers[1].forward(flipped_embs)
h_final = T.concatenate([h1, h2[::-1]], axis=2)
h_final = apply_dropout(h_final, dropout)
size = n_d * 2
#size = n_e
output_layer = self.output_layer = Layer(n_in=size,
n_out=1,
activation=sigmoid)
# len*batch*1
probs = output_layer.forward(h_final)
#probs = output_layer.forward(embs)
#probs1 = probs.reshape(x.shape)
#probs_rev = output_layer.forward(flipped_embs)
#probs1_rev = probs.reshape(x.shape)
#probs = T.concatenate([probs1, probs1_rev[::-1]], axis=2)
# len*batch
probs2 = probs.reshape(x.shape)
if self.args.seed is not None:
self.MRG_rng = MRG_RandomStreams(self.args.seed)
else:
self.MRG_rng = MRG_RandomStreams()
z_pred = self.z_pred = T.cast(
self.MRG_rng.binomial(size=probs2.shape, p=probs2),
theano.config.floatX) #"int8")
# we are computing approximated gradient by sampling z;
# so should mark sampled z not part of the gradient propagation path
#
z_pred = self.z_pred = theano.gradient.disconnected_grad(z_pred)
#self.sample_updates = sample_updates
print "z_pred", z_pred.ndim
z2 = z_pred.dimshuffle((0, 1, "x"))
logpz = -T.nnet.binary_crossentropy(probs, z2) * masks
logpz = self.logpz = logpz.reshape(x.shape)
probs = self.probs = probs.reshape(x.shape)
# batch
z = z_pred
self.zsum = T.sum(z, axis=0, dtype=theano.config.floatX)
self.zdiff = T.sum(T.abs_(z[1:] - z[:-1]),
axis=0,
dtype=theano.config.floatX)
params = self.params = []
for l in layers + [output_layer]:
for p in l.params:
params.append(p)
nparams = sum(len(x.get_value(borrow=True).ravel()) \
for x in params)
say("total # parameters: {}\n".format(nparams))
l2_cost = None
for p in params:
if l2_cost is None:
l2_cost = T.sum(p**2)
else:
l2_cost = l2_cost + T.sum(p**2)
l2_cost = l2_cost * args.l2_reg
self.l2_cost = l2_cost
def ready(self):
embedding_layer = self.embedding_layer
args = self.args
padding_id = embedding_layer.vocab_map["<padding>"]
dropout = self.dropout = theano.shared(
np.float64(args.dropout).astype(theano.config.floatX)
)
# len*batch
x = self.x = T.imatrix()
n_d = args.hidden_dimension
n_e = embedding_layer.n_d
activation = get_activation_by_name(args.activation)
layers = self.layers = [ ]
layer_type = args.layer.lower()
for i in xrange(1):
l = CNN(
n_in = n_e,
n_out = n_d,
activation = activation,
order = args.order
)
layers.append(l)
# len * batch
masks = T.cast(T.neq(x, padding_id), "int8").dimshuffle((0,1,'x'))
# (len*batch)*n_e
embs = embedding_layer.forward(x.ravel())
# len*batch*n_e
embs = embs.reshape((x.shape[0], x.shape[1], n_e))
embs = apply_dropout(embs, dropout)
self.word_embs = embs
# len*bacth*n_d
h1 = layers[0].forward_all(embs)
h_final = h1
size = n_d
h_final = apply_dropout(h_final, dropout)
output_layer = self.output_layer = Layer(
n_in = size,
n_out = 1,
activation = sigmoid
)
# len*batch*1
probs = output_layer.forward(h_final)
# len*batch
self.MRG_rng = MRG_RandomStreams()
z_pred_dim3 = self.MRG_rng.binomial(size=probs.shape, p=probs, dtype="int8")
z_pred = z_pred_dim3.reshape(x.shape)
# we are computing approximated gradient by sampling z;
# so should mark sampled z not part of the gradient propagation path
#
z_pred = self.z_pred = theano.gradient.disconnected_grad(z_pred)
print "z_pred", z_pred.ndim
#logpz = - T.nnet.binary_crossentropy(probs, z_pred_dim3) * masks
logpz = - T.nnet.binary_crossentropy(probs, z_pred_dim3)
logpz = self.logpz = logpz.reshape(x.shape)
probs = self.probs = probs.reshape(x.shape)
# batch
z = z_pred
self.zsum = T.sum(z, axis=0, dtype=theano.config.floatX)
self.zdiff = T.sum(T.abs_(z[1:]-z[:-1]), axis=0, dtype=theano.config.floatX)
params = self.params = [ ]
for l in layers + [ output_layer ]:
for p in l.params:
params.append(p)
nparams = sum(len(x.get_value(borrow=True).ravel()) \
for x in params)
say("total # parameters: {}\n".format(nparams))
l2_cost = None
for p in params:
if l2_cost is None:
l2_cost = T.sum(p**2)
else:
l2_cost = l2_cost + T.sum(p**2)
l2_cost = l2_cost * args.l2_reg
self.l2_cost = l2_cost
def ready(self):
encoder = self.encoder
embedding_layer = self.embedding_layer
args = self.args
padding_id = embedding_layer.vocab_map["<padding>"]
dropout = self.dropout = encoder.dropout
# len*batch
x = self.x = encoder.x
z = self.z = encoder.z
n_d = args.hidden_dimension
n_e = embedding_layer.n_d
activation = get_activation_by_name(args.activation)
layers = self.layers = []
layer_type = args.layer.lower()
for i in range(2):
if layer_type == "rcnn":
l = RCNN(
n_in=n_e, # if i == 0 else n_d,
n_out=n_d,
activation=activation,
order=args.order)
elif layer_type == "lstm":
l = LSTM(
n_in=n_e, # if i == 0 else n_d,
n_out=n_d,
activation=activation)
layers.append(l)
# len * batch
#masks = T.cast(T.neq(x, padding_id), theano.config.floatX)
masks = T.cast(T.neq(x, padding_id), "int8").dimshuffle((0, 1, "x"))
# (len*batch)*n_e
embs = embedding_layer.forward(x.ravel())
# len*batch*n_e
embs = embs.reshape((x.shape[0], x.shape[1], n_e))
embs = apply_dropout(embs, dropout)
flipped_embs = embs[::-1]
# len*bacth*n_d
h1 = layers[0].forward_all(embs)
h2 = layers[1].forward_all(flipped_embs)
h_final = T.concatenate([h1, h2[::-1]], axis=2)
h_final = apply_dropout(h_final, dropout)
size = n_d * 2
output_layer = self.output_layer = Layer(n_in=size,
n_out=1,
activation=sigmoid)
# len*batch*1
probs = output_layer.forward(h_final)
# len*batch
probs2 = probs.reshape(x.shape)
self.MRG_rng = MRG_RandomStreams()
z_pred = self.z_pred = T.cast(
self.MRG_rng.binomial(size=probs2.shape, p=probs2), "int8")
# we are computing approximated gradient by sampling z;
# so should mark sampled z not part of the gradient propagation path
#
self.z_pred = theano.gradient.disconnected_grad(z_pred)
z2 = z.dimshuffle((0, 1, "x"))
logpz = -T.nnet.binary_crossentropy(probs, z2) * masks
logpz = self.logpz = logpz.reshape(x.shape)
probs = self.probs = probs.reshape(x.shape)
# batch
zsum = T.sum(z, axis=0, dtype=theano.config.floatX)
zdiff_pre = (z[1:] - z[:-1]) * 1.0
zdiff = T.sum(abs(zdiff_pre), axis=0, dtype=theano.config.floatX)
loss_mat = encoder.loss_mat
if args.aspect < 0:
loss_vec = T.mean(loss_mat, axis=1)
else:
assert args.aspect < self.nclasses
loss_vec = loss_mat[:, args.aspect]
self.loss_vec = loss_vec
coherent_factor = args.sparsity * args.coherent
loss = self.loss = T.mean(loss_vec)
sparsity_cost = self.sparsity_cost = T.mean(zsum) * args.sparsity + \
T.mean(zdiff) * coherent_factor
cost_vec = loss_vec + zsum * args.sparsity + zdiff * coherent_factor
cost_logpz = T.mean(cost_vec * T.sum(logpz, axis=0))
self.obj = T.mean(cost_vec)
params = self.params = []
for l in layers + [output_layer]:
for p in l.params:
params.append(p)
nparams = sum(len(x.get_value(borrow=True).ravel()) \
for x in params)
say("total # parameters: {}\n".format(nparams))
l2_cost = None
for p in params:
if l2_cost is None:
l2_cost = T.sum(p**2)
else:
l2_cost = l2_cost + T.sum(p**2)
l2_cost = l2_cost * args.l2_reg
cost = self.cost = cost_logpz * 10 + l2_cost
print("cost.dtype", cost.dtype)
self.cost_e = loss * 10 + encoder.l2_cost
def ready(self):
args = self.args
embedding_layer = self.embedding_layer
self.n_hidden = args.hidden_dim
self.n_in = embedding_layer.n_d
dropout = self.dropout = theano.shared(
np.float64(args.dropout_rate).astype(theano.config.floatX))
# x is length * batch_size
# y is batch_size
self.x = T.imatrix('x')
self.y = T.ivector('y')
x = self.x
y = self.y
n_hidden = self.n_hidden
n_in = self.n_in
# fetch word embeddings
# (len * batch_size) * n_in
slices = embedding_layer.forward(x.ravel())
self.slices = slices
# 3-d tensor, len * batch_size * n_in
slices = slices.reshape((x.shape[0], x.shape[1], n_in))
# stacking the feature extraction layers
pooling = args.pooling
depth = args.depth
layers = self.layers = []
prev_output = slices
prev_output = apply_dropout(prev_output, dropout, v2=True)
size = 0
softmax_inputs = []
activation = get_activation_by_name(args.act)
for i in range(depth):
if args.layer.lower() == "lstm":
layer = LSTM(n_in=n_hidden if i > 0 else n_in, n_out=n_hidden)
elif args.layer.lower() == "strcnn":
layer = StrCNN(n_in=n_hidden if i > 0 else n_in,
n_out=n_hidden,
activation=activation,
decay=args.decay,
order=args.order)
elif args.layer.lower() == "rcnn":
layer = RCNN(n_in=n_hidden if i > 0 else n_in,
n_out=n_hidden,
activation=activation,
order=args.order,
mode=args.mode)
else:
raise Exception("unknown layer type: {}".format(args.layer))
layers.append(layer)
prev_output = layer.forward_all(prev_output)
if pooling:
softmax_inputs.append(T.sum(prev_output,
axis=0)) # summing over columns
else:
softmax_inputs.append(prev_output[-1])
prev_output = apply_dropout(prev_output, dropout)
size += n_hidden
# final feature representation is the concatenation of all extraction layers
if pooling:
softmax_input = T.concatenate(softmax_inputs, axis=1) / x.shape[0]
else:
softmax_input = T.concatenate(softmax_inputs, axis=1)
softmax_input = apply_dropout(softmax_input, dropout, v2=True)
# feed the feature repr. to the softmax output layer
layers.append(
Layer(n_in=size,
n_out=self.nclasses,
activation=softmax,
has_bias=False))
for l, i in zip(layers, range(len(layers))):
say("layer {}: n_in={}\tn_out={}\n".format(i, l.n_in, l.n_out))
# unnormalized score of y given x
self.p_y_given_x = layers[-1].forward(softmax_input)
self.pred = T.argmax(self.p_y_given_x, axis=1)
self.nll_loss = T.mean(
T.nnet.categorical_crossentropy(self.p_y_given_x, y))
# adding regularizations
self.l2_sqr = None
self.params = []
for layer in layers:
self.params += layer.params
for p in self.params:
if self.l2_sqr is None:
self.l2_sqr = args.l2_reg * T.sum(p**2)
else:
self.l2_sqr += args.l2_reg * T.sum(p**2)
nparams = sum(len(x.get_value(borrow=True).ravel()) \
for x in self.params)
say("total # parameters: {}\n".format(nparams))
def ready(self, args, train):
# len * batch
depth = args["depth"]
self.args = args
self.idxs = T.imatrix()
self.idys = T.imatrix()
self.init_state = [
T.matrix(dtype=theano.config.floatX) for i in xrange(depth * 2)
]
dropout_prob = np.float64(args["dropout"]).astype(theano.config.floatX)
self.dropout = theano.shared(dropout_prob)
rnn_dropout_prob = np.float64(args["rnn_dropout"]).astype(
theano.config.floatX)
self.rnn_dropout = theano.shared(rnn_dropout_prob)
self.n_d = args["hidden_dim"]
embedding_layer = EmbeddingLayer(n_d=self.n_d,
vocab=set(w for w in train))
self.n_V = embedding_layer.n_V
say("Vocab size: {}\tHidden dim: {}\n".format(self.n_V, self.n_d))
activation = get_activation_by_name(args["activation"])
layers = self.layers = []
for i in xrange(depth):
rnn_layer = KernelNN(n_in=self.n_d,
n_out=self.n_d,
activation=activation,
highway=args["highway"],
dropout=self.rnn_dropout)
layers.append(rnn_layer)
output_layer = Layer(
n_in=self.n_d,
n_out=self.n_V,
activation=T.nnet.softmax,
)
output_layer.W = embedding_layer.embeddings.T
# (len*batch) * n_d
x_flat = embedding_layer.forward(self.idxs.ravel())
# len * batch * n_d
x = apply_dropout(x_flat, self.dropout)
#x = x_flat
x = x.reshape((self.idxs.shape[0], self.idxs.shape[1], self.n_d))
# len * batch * (n_d+n_d)
self.last_state = []
prev_h = x
for i in xrange(depth):
hidden = self.init_state[i * 2:i * 2 + 2]
c, h = layers[i].forward_all(prev_h, hidden, return_c=True)
self.last_state += [c[-1], h[-1]]
prev_h = h
prev_h = apply_dropout(prev_h, self.dropout)
self.p_y_given_x = output_layer.forward(prev_h.reshape(x_flat.shape))
idys = self.idys.ravel()
self.nll = T.nnet.categorical_crossentropy(self.p_y_given_x, idys)
self.params = [x for l in layers for x in l.params]
self.params += [embedding_layer.embeddings, output_layer.b]
self.num_params = sum(
len(x.get_value(borrow=True).ravel()) for x in self.params)
say("# of params in total: {}\n".format(self.num_params))
layers += [embedding_layer, output_layer]
guess = np.argmax(out)
if t == guess:
correct += 1
return correct / test_data.shape[0]
def train(net, train_data):
for i, train_row in enumerate(train_data):
if not i % 1000:
print(i)
net.train(to_col(train_row[1:]) / 255, train_row[0])
if __name__ == "__main__":
layers = [
Layer(784, 10, LeakyReLU()),
]
net = NeuralNetwork(layers, CrossEntropyLoss(), 0.001)
test_data = load_data(TEST_FILE, delimiter=",", dtype=int)
accuracy = test(net, test_data)
print(f"Accuracy is {100*accuracy:.2f}%") # Expected to be around 10%
train_data = load_data(TRAIN_FILE, delimiter=",", dtype=int)
train(net, train_data)
accuracy = test(net, test_data)
print(f"Accuracy is {100*accuracy:.2f}%")
def train_student(student, teacher, train_data):
"""Train a student network to behave like the teacher network."""
for i, train_row in enumerate(train_data):
if not i % 1000:
print(i)
x = to_col(train_row[1:])
teacher_out = teacher.forward_pass(x)
student.train(x, teacher_out)
if __name__ == "__main__":
teacher_layers = [
Layer(784, 16, LeakyReLU()),
Layer(16, 16, LeakyReLU()),
Layer(16, 10, LeakyReLU()),
]
teacher_net = NeuralNetwork(teacher_layers, CrossEntropyLoss(), 0.001)
train_data = load_data("mnistdata/mnist_train.csv",
delimiter=",",
dtype=int)
train(teacher_net, train_data)
test_data = load_data("mnistdata/mnist_test.csv", delimiter=",", dtype=int)
accuracy = test(teacher_net, test_data)
print(f"Accuracy of the teacher net is {100*accuracy:.2f}")
student_layers = [
def ready(self):
args = self.args
embedding_layer = self.embedding_layer
user_embedding_layer = self.user_embedding_layer
self.n_hidden = args.hidden_dim
self.n_in = embedding_layer.n_d
dropout = self.dropout = theano.shared(
np.float64(args.dropout_rate).astype(theano.config.floatX)
)
# x is length * batch_size
# y is batch_size
self.x = T.imatrix('x')
self.w_masks = T.fmatrix('mask')
self.w_lens = T.fvector('lens')
self.s_ml = T.iscalar('sent_maxlen')
self.s_num = T.iscalar('sent_num')
self.y = T.ivector('y')
self.usr = T.ivector('users')
x = self.x
y = self.y
usr = self.usr
w_masks = self.w_masks
w_lens = self.w_lens
s_ml = self.s_ml
s_num = self.s_num
n_hidden = self.n_hidden
n_emb = n_in = self.n_in
layers = self.layers = []
slicesu = user_embedding_layer.forward(usr)
slices = embedding_layer.forward(x.ravel())
self.slices = slices # important for updating word embeddings
# 3-d tensor, len * batch_size * n_in
slices = slices.reshape((x.shape[0], x.shape[1], n_in))
pooling = args.pooling
prev_output = slices
prev_output = apply_dropout(prev_output, dropout, v2=True)
size = 0
n_hidden_t = n_hidden
if args.direction == "bi":
n_hidden_t = 2 * n_hidden
softmax_inputs = []
activation = get_activation_by_name(args.act)
if args.layer.lower() == "lstm":
layer = LSTM(n_in=n_in,
n_out=n_hidden_t,
direction=args.direction
)
elif args.layer.lower() == "cnn":
layer = CNN(n_in=n_in,
n_out=n_hidden_t,
activation=activation,
order=args.order
)
else:
raise Exception("unknown layer type: {}".format(args.layer))
layers.append(layer)
prev_output = layer.forward_all(prev_output, masks=w_masks)
prev_output = apply_dropout(prev_output, dropout)
# final feature representation is the concatenation of all extraction layers
if args.user_atten:
layer = IterAttentionLayer(
n_in=n_emb,
n_out=n_hidden_t
)
layers.append(layer)
if args.user_atten_base:
slicesu = None
softmax_input = layers[-1].multi_hop_forward(
prev_output, user_embs=slicesu, isWord=True, masks=w_masks)
else:
if pooling:
softmax_input = T.sum(prev_output, axis=0) / w_lens.dimshuffle(0, 'x')
else:
ind = T.cast(w_lens - T.ones_like(w_lens), 'int32')
softmax_input = prev_output[T.arange(ind.shape[0]), ind]
softmax_input = apply_dropout(softmax_input, dropout, v2=True)
n_in = n_hidden_t
size = 0
softmax_inputs = []
[sentlen, emblen] = T.shape(softmax_input)
prev_output = softmax_input.reshape(
(sentlen / s_num, s_num, emblen)).dimshuffle(1, 0, 2)
if args.layer.lower() == "lstm":
layer = LSTM(n_in=n_in,
n_out=n_hidden_t,
direction=args.direction
)
elif args.layer.lower() == "cnn":
layer = CNN(n_in=n_in,
n_out=n_hidden_t,
activation=activation,
order=args.order,
)
else:
raise Exception("unknown layer type: {}".format(args.layer))
layers.append(layer)
prev_output = layer.forward_all(prev_output)
prev_output = apply_dropout(prev_output, dropout)
if args.user_atten:
layer = IterAttentionLayer(
n_in=n_emb,
n_out=n_hidden_t
)
layers.append(layer)
if args.user_atten_base:
slicesu = None
softmax_input = layers[-1].multi_hop_forward(
prev_output, user_embs=slicesu, isWord=False)
else:
if pooling:
softmax_input = T.sum(prev_output, axis=0) / \
T.cast(s_num, 'float32')
else:
softmax_input = prev_output[-1]
softmax_input = apply_dropout(softmax_input, dropout, v2=True)
size = n_hidden_t
layers.append(Layer(
n_in=size,
n_out=self.nclasses,
activation=softmax,
has_bias=False
))
if not args.fix_emb:
for l, i in zip(layers, range(len(layers))):
say("layer {}: n_in={}\tn_out={}\n".format(
i, l.n_in, l.n_out
))
else:
for l, i in zip(layers[1:], range(len(layers[1:]))):
say("layer {}: n_in={}\tn_out={}\n".format(
i, l.n_in, l.n_out
))
# unnormalized score of y given x
self.p_y_given_x = layers[-1].forward(softmax_input)
self.pred = T.argmax(self.p_y_given_x, axis=1)
self.nll_loss = T.mean(T.nnet.categorical_crossentropy(
self.p_y_given_x,
y
))
# adding regularizations
self.l2_sqr = None
self.params = []
for layer in layers:
self.params += layer.params
for p in self.params:
if self.l2_sqr is None:
self.l2_sqr = args.l2_reg * T.sum(p**2)
else:
self.l2_sqr += args.l2_reg * T.sum(p**2)
nparams = sum(len(x.get_value(borrow=True).ravel())
for x in self.params)
say("total # parameters: {}\n".format(nparams))
# tv = np.zeros((10, 1))
# tv[t] = 1
# ts[t] = tv
for i, train_row in enumerate(train_data):
if not i % 1000:
print(i)
t = train_row[0]
x = to_col(train_row[1:])
net.train(x, t)
if __name__ == "__main__":
layers = [
Layer(784, 16, LeakyReLU()),
Layer(16, 16, LeakyReLU()),
Layer(16, 10, LeakyReLU()),
]
net = NeuralNetwork(layers, CrossEntropyLoss(), 0.001)
test_data = load_data("mnistdata/mnist_test.csv", delimiter=",", dtype=int)
accuracy = test(net, test_data)
print(f"Accuracy is {100*accuracy:.2f}%") # Expected to be around 10%
train_data = load_data("mnistdata/mnist_train.csv",
delimiter=",",
dtype=int)
train(net, train_data)
logging.basicConfig(level=logging.NOTSET)
def see(layer: Layer):
for n in layer.cells:
print(n)
for link in n.output_list:
print(link)
for it in range(10):
s = Sequential()
s.add_layer(InputLayer(2, 'input'))
s.add_layer(
Layer(2, 'hidden', Function.ReLU(), Initializer.uniform(0, 0.1)),
Initializer.uniform(-0.1, 0.1))
s.add_layer(Layer(1, 'output', initializer=Initializer.uniform(0, 0.1)),
Initializer.uniform(-0.1, 0.1))
logging.info('TEST %i' % it)
error_val = [1]
error_val_max_length = 50
while np.mean(error_val) > 0.05:
a, b = np.random.randint(2), np.random.randint(2)
c = int(a == b)
input_data = np.array([a, b])
target_output = np.array([c])
s.commit(input_data, target_output, 0.1)
error = s.forward(input_data)[0] - target_output
def ready(self):
args = self.args
w_emb_layer = self.w_emb_layer
c_emb_layer = self.c_emb_layer
r_emb_layers = self.r_emb_layers
r_matrix_layers = self.r_matrix_layers
char_dim = self.char_dim = args.char_dim
char_lstm_dim = self.char_lstm_dim = args.char_lstm_dim
word_dim = self.word_dim = args.word_dim
word_lstm_dim = self.word_lstm_dim = args.word_lstm_dim
dropout = self.dropout = theano.shared(
np.float64(args.dropout).astype(theano.config.floatX)
)
word_ids = self.word_ids = T.ivector('word_ids')
char_ids = self.char_ids = T.imatrix('char_ids')
char_lens = self.char_lens = T.fvector('char_lens')
char_masks = self.char_masks = T.imatrix('char_masks')
up_ids = self.up_ids = T.imatrix('up_ids')
up_rels = self.up_rels = T.imatrix('up_rels')
up_id_masks = self.up_id_masks = T.imatrix('up_id_masks')
down_ids = self.down_ids = T.imatrix('down_ids')
down_rels = self.down_rels = T.imatrix('down_rels')
down_id_masks = self.down_id_masks = T.imatrix('down_id_masks')
tag_ids = self.tag_ids = T.ivector('tag_ids')
layers = self.layers = [w_emb_layer, c_emb_layer]
layers.extend(r_emb_layers)
layers.extend(r_matrix_layers)
inputs = self.inputs = []
inputs.append(self.word_ids)
inputs.append(self.char_ids)
inputs.append(self.char_lens)
inputs.append(self.char_masks)
inputs.append(self.up_ids)
inputs.append(self.up_rels)
inputs.append(self.up_id_masks)
inputs.append(self.down_ids)
inputs.append(self.down_rels)
inputs.append(self.down_id_masks)
inputs.append(self.tag_ids)
wslices = w_emb_layer.forward(word_ids)
cslices = c_emb_layer.forward(char_ids.ravel())
cslices = cslices.reshape((char_ids.shape[0], char_ids.shape[1], char_dim))
cslices = cslices.dimshuffle(1, 0, 2)
bv_ur_slicess = []
bv_dr_slicess = []
b_ur_slicess = []
b_dr_slicess = []
bv_ur_matrixss = []
bv_dr_matrixss = []
b_ur_matrixss = []
b_dr_matrixss = []
for r_matrix_layer in r_matrix_layers:
bv_ur_matrixs = r_matrix_layer.forward1(up_rels.ravel())
bv_dr_matrixs = r_matrix_layer.forward1(down_rels.ravel())
b_ur_matrixs = r_matrix_layer.forward2(up_rels.ravel())
b_dr_matrixs = r_matrix_layer.forward2(down_rels.ravel())
bv_ur_matrixss.append(bv_ur_matrixs.reshape((up_rels.shape[0], up_rels.shape[1], word_dim, word_dim)))
bv_dr_matrixss.append(bv_dr_matrixs.reshape((down_rels.shape[0], down_rels.shape[1], word_dim, word_dim)))
b_ur_matrixss.append(b_ur_matrixs.reshape((up_rels.shape[0], up_rels.shape[1], word_dim, word_dim)))
b_dr_matrixss.append(b_dr_matrixs.reshape((down_rels.shape[0], down_rels.shape[1], word_dim, word_dim)))
for r_emb_layer in r_emb_layers:
bv_ur_slices = r_emb_layer.forward(up_rels.ravel())
bv_dr_slices = r_emb_layer.forward(down_rels.ravel())
b_ur_slices = r_emb_layer.forward2(up_rels.ravel())
b_dr_slices = r_emb_layer.forward2(down_rels.ravel())
bv_ur_slicess.append(bv_ur_slices.reshape((up_rels.shape[0], up_rels.shape[1], word_dim)))
bv_dr_slicess.append(bv_dr_slices.reshape((down_rels.shape[0], down_rels.shape[1], word_dim)))
b_ur_slicess.append(b_ur_slices.reshape((up_rels.shape[0], up_rels.shape[1], word_dim)))
b_dr_slicess.append(b_dr_slices.reshape((down_rels.shape[0], down_rels.shape[1], word_dim)))
char_masks = char_masks.dimshuffle(1, 0)
prev_output = wslices
prev_size = word_dim
if char_dim:
layers.append(LSTM(
n_in = char_dim,
n_out = char_lstm_dim,
direction = 'bi' if args.char_bidirect else 'si'
))
prev_output_2 = cslices
prev_output_2 = apply_dropout(prev_output_2, dropout, v2 = True)
prev_output_2 = layers[-1].forward_all(cslices, char_masks)
prev_output_2 = T.sum(prev_output_2, axis = 0)
prev_output_2 = prev_output_2 / (1e-6 * T.ones_like(char_lens) + char_lens).dimshuffle(0, 'x')
prev_size += char_lstm_dim
prev_output = T.concatenate([prev_output, prev_output_2], axis = 1)
prev_output = apply_dropout(prev_output, dropout)
if args.conv != 0:
for i in range(args.clayer):
layers.append(GKNNMultiHeadGate(
n_in = prev_size,
n_out = prev_size,
n_head = args.head
))
prev_output = layers[-1].forward_all(prev_output, up_ids, up_id_masks, bv_ur_slicess[0], down_ids, down_id_masks, bv_dr_slicess[0])
prev_output = apply_dropout(prev_output, dropout)
#prev_size *= 2
#layers.append(LSTM(
# n_in = prev_size,
# n_out = word_lstm_dim,
# direction = 'bi' if args.word_bidirect else 'si'
#))
#prev_output = prev_output.dimshuffle(0, 'x', 1)
#prev_output = layers[-1].forward_all(prev_output)
#prev_output = prev_output.reshape((prev_output.shape[0], prev_output.shape[-1]))
#prev_size = word_lstm_dim
layers.append(Layer(
n_in = prev_size,
n_out = args.classes,
activation = linear, #ReLU,
has_bias = False
))
n_tags = args.classes
s_len = char_ids.shape[0]
tags_scores = layers[-1].forward(prev_output)
transitions = shared((n_tags + 2, n_tags + 2), 'transitions')
small = -1000
b_s = np.array([[small] * n_tags + [0, small]]).astype(np.float32)
e_s = np.array([[small] * n_tags + [small, 0]]).astype(np.float32)
observations = T.concatenate(
[tags_scores, small * T.ones((s_len, 2))],
axis=1
)
observations = T.concatenate(
[b_s, observations, e_s],
axis=0
)
real_path_score = tags_scores[T.arange(s_len), tag_ids].sum()
b_id = theano.shared(value=np.array([n_tags], dtype=np.int32))
e_id = theano.shared(value=np.array([n_tags + 1], dtype=np.int32))
padded_tags_ids = T.concatenate([b_id, tag_ids, e_id], axis=0)
pre_ids = T.arange(s_len + 1)
s_ids = T.arange(s_len + 1) + 1
real_path_score += transitions[
padded_tags_ids[pre_ids],
padded_tags_ids[s_ids]
].sum()
all_paths_scores = CRFForward(observations, transitions)
self.nll_loss = nll_loss = - (real_path_score - all_paths_scores)
preds = CRFForward(observations, transitions, viterbi = True,
return_alpha = False, return_best_sequence=True)
self.pred = preds[1:-1]
self.l2_sqr = None
params = self.params = [transitions]
for layer in layers:
self.params += layer.params
for p in self.params:
if self.l2_sqr is None:
self.l2_sqr = args.l2_reg * T.sum(p**2)
else:
self.l2_sqr += args.l2_reg * T.sum(p**2)
#for l, i in zip(layers[3:], range(len(layers[3:]))):
for l, i in zip(layers[2+len(r_emb_layers)+len(r_matrix_layers):], range(len(layers[2+len(r_emb_layers)+len(r_matrix_layers):]))):
say("layer {}: n_in={}\tn_out={}\n".format(
i, l.n_in, l.n_out
))
nparams = sum(len(x.get_value(borrow=True).ravel()) \
for x in self.params)
say("total # parameters: {}\n".format(nparams))
cost = self.nll_loss + self.l2_sqr
lr_method_name = args.learning
lr_method_parameters = {}
lr_method_parameters['lr'] = args.learning_rate
updates = Optimization(clip=5.0).get_updates(lr_method_name, cost, params, **lr_method_parameters)
f_train = theano.function(
inputs = self.inputs,
outputs = [cost, nll_loss],
updates = updates,
allow_input_downcast = True
)
f_eval = theano.function(
inputs = self.inputs[:-1],
outputs = self.pred,
allow_input_downcast = True
)
return f_train, f_eval
def ready(self):
args = self.args
weights = self.weights
# len(source) * batch
idxs = self.idxs = T.imatrix()
# len(target) * batch
idys = self.idys = T.imatrix()
idts = idys[:-1]
idgs = idys[1:]
dropout = self.dropout = theano.shared(
np.float64(args.dropout).astype(theano.config.floatX))
embedding_layer = self.embedding_layer
activation = get_activation_by_name(args.activation)
n_d = self.n_d = args.hidden_dim
n_e = self.n_e = embedding_layer.n_d
n_V = self.n_V = embedding_layer.n_V
if args.layer.lower() == "rcnn":
LayerType = RCNN
elif args.layer.lower() == "lstm":
LayerType = LSTM
elif args.layer.lower() == "gru":
LayerType = GRU
depth = self.depth = args.depth
layers = self.layers = []
for i in range(depth * 2):
if LayerType != RCNN:
feature_layer = LayerType(n_in=n_e if i / 2 == 0 else n_d,
n_out=n_d,
activation=activation)
else:
feature_layer = LayerType(n_in=n_e if i / 2 == 0 else n_d,
n_out=n_d,
activation=activation,
order=args.order,
mode=args.mode,
has_outgate=args.outgate)
layers.append(feature_layer)
self.output_layer = output_layer = Layer(
n_in=n_d,
n_out=n_V,
activation=T.nnet.softmax,
)
# feature computation starts here
# (len*batch)*n_e
xs_flat = embedding_layer.forward(idxs.ravel())
xs_flat = apply_dropout(xs_flat, dropout)
if weights is not None:
xs_w = weights[idxs.ravel()].dimshuffle((0, 'x'))
xs_flat = xs_flat * xs_w
# len*batch*n_e
xs = xs_flat.reshape((idxs.shape[0], idxs.shape[1], n_e))
# (len*batch)*n_e
xt_flat = embedding_layer.forward(idts.ravel())
xt_flat = apply_dropout(xt_flat, dropout)
if weights is not None:
xt_w = weights[idts.ravel()].dimshuffle((0, 'x'))
xt_flat = xt_flat * xt_w
# len*batch*n_e
xt = xt_flat.reshape((idts.shape[0], idts.shape[1], n_e))
prev_hs = xs
prev_ht = xt
for i in range(depth):
# len*batch*n_d
hs = layers[i * 2].forward_all(prev_hs, return_c=True)
ht = layers[i * 2 + 1].forward_all(prev_ht, hs[-1])
hs = hs[:, :, -n_d:]
ht = ht[:, :, -n_d:]
prev_hs = hs
prev_ht = ht
prev_hs = apply_dropout(hs, dropout)
prev_ht = apply_dropout(ht, dropout)
self.p_y_given_x = output_layer.forward(
prev_ht.reshape((xt_flat.shape[0], n_d)))
h_final = hs[-1]
self.scores2 = -(h_final[1:] - h_final[0]).norm(2, axis=1)
h_final = self.normalize_2d(h_final)
self.scores = T.dot(h_final[1:], h_final[0])
# (len*batch)
nll = T.nnet.categorical_crossentropy(self.p_y_given_x, idgs.ravel())
nll = nll.reshape(idgs.shape)
self.nll = nll
self.mask = mask = T.cast(T.neq(idgs, self.padding_id),
theano.config.floatX)
nll = T.sum(nll * mask, axis=0)
#layers.append(embedding_layer)
layers.append(output_layer)
params = []
for l in self.layers:
params += l.params
self.params = params
say("num of parameters: {}\n".format(
sum(len(x.get_value(borrow=True).ravel()) for x in params)))
l2_reg = None
for p in params:
if l2_reg is None:
l2_reg = p.norm(2)
else:
l2_reg = l2_reg + p.norm(2)
l2_reg = l2_reg * args.l2_reg
self.loss = T.mean(nll)
self.cost = self.loss + l2_reg
def ready(self):
args = self.args
embedding_layer = self.embedding_layer
num_aspects = self.num_aspects
self.n_emb = embedding_layer.n_d
dropout = self.dropout = theano.shared(
np.float64(args.dropout_rate).astype(theano.config.floatX)
)
self.x = T.imatrix('x')
self.w_masks = T.fmatrix('mask')
self.w_lens = T.fvector('sent_len')
self.s_maxlen = T.iscalar('sent_max_len')
self.s_num = T.iscalar('sent_num')
self.y = T.ivector('y')
self.ay = T.imatrix('ay')
self.ay_mask = T.fmatrix('ay_mask')
self.aay = T.itensor3('aay')
x = self.x
query = self.query
w_masks = self.w_masks
w_lens = self.w_lens
s_ml = self.s_maxlen
s_num = self.s_num
n_emb = self.n_emb
y = self.y
ay = self.ay
ay_mask = self.ay_mask
aay = self.aay
layers = self.layers = [embedding_layer]
slices = embedding_layer.forward(x.ravel())
self.slices = slices = slices.reshape( (x.shape[0], x.shape[1], n_emb) )
slices_query = embedding_layer.forward(query.flatten(), is_node = False)
slices_query = slices_query.reshape( (query.shape[0], query.shape[1], n_emb))
layers.append(Query_Repr_Layer(slices_query))
slices_query_tmp = slices_query = layers[-1].forward()
layer = LSTM(n_in = n_emb, n_out = n_emb)
layers.append(layer)
prev_output = slices
prev_output = apply_dropout(prev_output, dropout, v2=True)
prev_output = layers[-1].forward_all(prev_output, w_masks)
layer = Layer(n_in = n_emb, n_out = n_emb, activation = tanh)
layers.append(layer)
self.slices_query = slices_query = layers[-1].forward(slices_query)
maskss = []
w_lenss = []
for i in range(num_aspects):
maskss.append(w_masks)
w_lenss.append(w_lens)
maskss = T.concatenate(maskss, axis = 1)
w_lenss = T.concatenate(w_lenss)
layer = IterAttentionLayer(n_in = n_emb, n_out = n_emb)
layers.append(layer)
prev_output = layers[-1].forward(prev_output, slices_query, is_word = True, hop = args.hop_word, masks = w_masks, aspect_num = num_aspects)
prev_output = prev_output.reshape((prev_output.shape[0] * prev_output.shape[1], prev_output.shape[2]))
prev_output = apply_dropout(prev_output, dropout, v2=True)
prev_output = prev_output.reshape((num_aspects, prev_output.shape[0] / (num_aspects * s_num), s_num, prev_output.shape[1]))
prev_output = prev_output.dimshuffle(2, 0, 1, 3)
prev_output = prev_output.reshape((prev_output.shape[0], prev_output.shape[1] * prev_output.shape[2], prev_output.shape[3]))
layer = LSTM(n_in = n_emb * args.hop_word, n_out = n_emb)
layers.append(layer)
prev_output = layers[-1].forward_all(prev_output)
#layers.append(Query_Repr_Layer(slices_query))
#slices_query = layers[-1].forward()
layer = Layer(n_in = n_emb, n_out = n_emb, activation = tanh)
layers.append(layer)
slices_query = layers[-1].forward(slices_query_tmp) # bug
layer = IterAttentionLayer(n_in = n_emb, n_out = n_emb)
layers.append(layer)
prev_output = layers[-1].forward(prev_output, slices_query, is_word = False, hop = args.hop_sent, aspect_num = num_aspects)
prev_output = prev_output.reshape((prev_output.shape[0] * prev_output.shape[1], prev_output.shape[2]))
prev_output = apply_dropout(prev_output, dropout, v2=True)
prev_output = prev_output.reshape((num_aspects, prev_output.shape[0] / num_aspects, prev_output.shape[1]))
softmax_inputs = []
for i in range(num_aspects):
softmax_inputs.append(prev_output[i])
size = n_emb * args.hop_sent
p_y_given_a = []
pred_ay = []
nll_loss_ay = []
for i in range(num_aspects):
layers.append(Layer(n_in = size,
n_out = args.score_scale,
activation = softmax,
has_bias = False,))
p_y_given_a.append(layers[-1].forward(softmax_inputs[i]))
nll_loss_ay.append( T.mean(T.sum( -T.log(p_y_given_a[-1]) * aay[:, i, :] * ay_mask[:, i].dimshuffle(0, 'x'))))
pred_ay.append(T.argmax(p_y_given_a[-1], axis = 1))
self.p_y_given_a = p_y_given_a
self.nll_loss_ay = T.sum(nll_loss_ay)
self.pred_ay = T.stack(pred_ay).dimshuffle(1, 0)
for l,i in zip(layers[4:], range(len(layers[3:]))):
say("layer {}: n_in={}\tn_out={}\n".format(
i, l.n_in, l.n_out
))
self.l2_sqr = None
self.params = [ ]
for layer in layers:
self.params += layer.params
for p in self.params:
if self.l2_sqr is None:
self.l2_sqr = args.l2_reg * T.sum(p**2)
else:
self.l2_sqr += args.l2_reg * T.sum(p**2)
nparams = sum(len(x.get_value(borrow=True).ravel()) \
for x in self.params)
say("total # parameters: {}\n".format(nparams))
def ready(self):
embedding_layer = self.embedding_layer
args = self.args
padding_id = embedding_layer.vocab_map["<padding>"]
dropout = self.dropout = theano.shared(
np.float64(args.dropout).astype(theano.config.floatX))
# len*batch
x = self.x = T.imatrix()
z = self.z = T.bmatrix()
z = z.dimshuffle((0, 1, "x"))
# batch*nclasses
y = self.y = T.fmatrix()
n_d = args.hidden_dimension
n_e = embedding_layer.n_d
activation = get_activation_by_name(args.activation)
layers = self.layers = []
depth = args.depth
layer_type = args.layer.lower()
for i in range(depth):
if layer_type == "rcnn":
l = ExtRCNN(n_in=n_e if i == 0 else n_d,
n_out=n_d,
activation=activation,
order=args.order)
elif layer_type == "lstm":
l = ExtLSTM(n_in=n_e if i == 0 else n_d,
n_out=n_d,
activation=activation)
layers.append(l)
# len * batch * 1
masks = T.cast(
T.neq(x, padding_id).dimshuffle((0, 1, "x")) * z,
theano.config.floatX)
# batch * 1
cnt_non_padding = T.sum(masks, axis=0) + 1e-8
# (len*batch)*n_e
embs = embedding_layer.forward(x.ravel())
# len*batch*n_e
embs = embs.reshape((x.shape[0], x.shape[1], n_e))
embs = apply_dropout(embs, dropout)
pooling = args.pooling
lst_states = []
h_prev = embs
for l in layers:
# len*batch*n_d
h_next = l.forward_all(h_prev, z)
if pooling:
# batch * n_d
masked_sum = T.sum(h_next * masks, axis=0)
lst_states.append(masked_sum / cnt_non_padding) # mean pooling
else:
lst_states.append(h_next[-1]) # last state
h_prev = apply_dropout(h_next, dropout)
if args.use_all:
size = depth * n_d
# batch * size (i.e. n_d*depth)
h_final = T.concatenate(lst_states, axis=1)
else:
size = n_d
h_final = lst_states[-1]
h_final = apply_dropout(h_final, dropout)
output_layer = self.output_layer = Layer(n_in=size,
n_out=self.nclasses,
activation=sigmoid)
# batch * nclasses
preds = self.preds = output_layer.forward(h_final)
# batch
loss_mat = self.loss_mat = (preds - y)**2
loss = self.loss = T.mean(loss_mat)
pred_diff = self.pred_diff = T.mean(
T.max(preds, axis=1) - T.min(preds, axis=1))
params = self.params = []
for l in layers + [output_layer]:
for p in l.params:
params.append(p)
nparams = sum(len(x.get_value(borrow=True).ravel()) \
for x in params)
say("total # parameters: {}\n".format(nparams))
l2_cost = None
for p in params:
if l2_cost is None:
l2_cost = T.sum(p**2)
else:
l2_cost = l2_cost + T.sum(p**2)
l2_cost = l2_cost * args.l2_reg
self.l2_cost = l2_cost
cost = self.cost = loss * 10 + l2_cost
net.train(to_col(train_row[1:])/255, train_row[0])
def train_students(teacher, students, train_data):
for i, train_row in enumerate(train_data):
if not i%1000:
print(i)
x = to_col(train_row[1:])/255
out = teacher.forward_pass(x)
for student in students:
student.train(x, out)
if __name__ == "__main__":
layers = [
Layer(784, 16, LeakyReLU()),
Layer(16, 16, LeakyReLU()),
Layer(16, 10, LeakyReLU()),
]
teacher = NeuralNetwork(layers, CrossEntropyLoss(), 0.03)
students = [
NeuralNetwork([Layer(784, 10, LeakyReLU())], MSELoss(), 0.001),
NeuralNetwork([Layer(784, 10, LeakyReLU())], MSELoss(), 0.003),
NeuralNetwork([Layer(784, 10, LeakyReLU())], MSELoss(), 0.01),
NeuralNetwork([Layer(784, 10, LeakyReLU())], MSELoss(), 0.03),
NeuralNetwork([Layer(784, 10, LeakyReLU())], MSELoss(), 0.1),
NeuralNetwork([Layer(784, 10, LeakyReLU())], MSELoss(), 0.3),
]
test_data = load_data(TEST_FILE, delimiter=",", dtype=int)
accuracy = test(teacher, test_data)
def ready(self):
generator = self.generator
embedding_layer = self.embedding_layer
args = self.args
padding_id = embedding_layer.vocab_map["<padding>"]
unk_id = embedding_layer.vocab_map["<unk>"]
unk_vec = embedding_layer.embeddings[unk_id]
dropout = generator.dropout
# len*batch
x = generator.x
z = generator.z_pred
z = z.dimshuffle((0,1,"x"))
# batch*nclasses
y = self.y = T.fmatrix()
n_d = args.hidden_dimension
n_e = embedding_layer.n_d
activation = get_activation_by_name(args.activation)
layers = self.layers = [ ]
depth = args.depth
layer_type = args.layer.lower()
for i in xrange(depth):
l = CNN(
n_in = n_e if i == 0 else n_d,
n_out = n_d,
activation = activation,
order = args.order
)
layers.append(l)
# len * batch * 1
masks = T.cast(T.neq(x, padding_id).dimshuffle((0,1,"x")) * z, theano.config.floatX)
# batch * 1
cnt_non_padding = T.sum(masks, axis=0) + 1e-8
# len*batch*n_e
embs = generator.word_embs*z + unk_vec.dimshuffle(('x','x',0))*(1-z)
pooling = args.pooling
lst_states = [ ]
h_prev = embs
for l in layers:
# len*batch*n_d
h_next = l.forward_all(h_prev)
if pooling:
# batch * n_d
masked_sum = T.sum(h_next * masks, axis=0)
lst_states.append(masked_sum/cnt_non_padding) # mean pooling
else:
lst_states.append(T.max(h_next, axis=0))
h_prev = apply_dropout(h_next, dropout)
if args.use_all:
size = depth * n_d
# batch * size (i.e. n_d*depth)
h_final = T.concatenate(lst_states, axis=1)
else:
size = n_d
h_final = lst_states[-1]
h_final = apply_dropout(h_final, dropout)
output_layer = self.output_layer = Layer(
n_in = size,
n_out = self.nclasses,
activation = sigmoid
)
# batch * nclasses
p_y_given_x = self.p_y_given_x = output_layer.forward(h_final)
preds = self.preds = p_y_given_x > 0.5
print preds, preds.dtype
print self.nclasses
# batch
loss_mat = T.nnet.binary_crossentropy(p_y_given_x, y)
if args.aspect < 0:
loss_vec = T.mean(loss_mat, axis=1)
else:
assert args.aspect < self.nclasses
loss_vec = loss_mat[:,args.aspect]
self.loss_vec = loss_vec
self.true_pos = T.sum(preds*y)
self.tot_pos = T.sum(preds)
self.tot_true = T.sum(y)
zsum = generator.zsum
zdiff = generator.zdiff
logpz = generator.logpz
coherent_factor = args.sparsity * args.coherent
loss = self.loss = T.mean(loss_vec)
sparsity_cost = self.sparsity_cost = T.mean(zsum) * args.sparsity + \
T.mean(zdiff) * coherent_factor
cost_vec = loss_vec + zsum * args.sparsity + zdiff * coherent_factor
cost_logpz = T.mean(cost_vec * T.sum(logpz, axis=0))
self.obj = T.mean(cost_vec)
params = self.params = [ ]
for l in layers + [ output_layer ]:
for p in l.params:
params.append(p)
if not args.fix_emb:
params += embedding_layer.params
nparams = sum(len(x.get_value(borrow=True).ravel()) \
for x in params)
say("total # parameters: {}\n".format(nparams))
l2_cost = None
for p in params:
if l2_cost is None:
l2_cost = T.sum(p**2)
else:
l2_cost = l2_cost + T.sum(p**2)
l2_cost = l2_cost * args.l2_reg
self.l2_cost = l2_cost
self.cost_g = cost_logpz + generator.l2_cost
self.cost_e = loss + l2_cost
def __init__(self, n_d):
self.n_d = n_d
self.gate = Layer(n_d, n_d, sigmoid)
def ready(self):
global total_encode_time
#say("in encoder ready: \n")
#start_encode_time = time.time()
generator = self.generator
embedding_layer = self.embedding_layer
args = self.args
padding_id = embedding_layer.vocab_map["<padding>"]
dropout = generator.dropout
# len*batch
x = generator.x
z = generator.z_pred
z = z.dimshuffle((0, 1, "x"))
# batch*nclasses
y = self.y = T.fmatrix()
n_d = args.hidden_dimension
n_e = embedding_layer.n_d
activation = get_activation_by_name(args.activation)
layers = self.layers = []
depth = args.depth
layer_type = args.layer.lower()
for i in xrange(depth):
if layer_type == "rcnn":
l = ExtRCNN(n_in=n_e if i == 0 else n_d,
n_out=n_d,
activation=activation,
order=args.order)
elif layer_type == "lstm":
l = ExtLSTM(n_in=n_e if i == 0 else n_d,
n_out=n_d,
activation=activation)
layers.append(l)
# len * batch * 1
masks = T.cast(
T.neq(x, padding_id).dimshuffle((0, 1, "x")) * z,
theano.config.floatX)
# batch * 1
cnt_non_padding = T.sum(masks, axis=0) + 1e-8
# len*batch*n_e
embs = generator.word_embs
pooling = args.pooling
lst_states = []
h_prev = embs
for l in layers:
# len*batch*n_d
h_next = l.forward_all(h_prev, z)
if pooling:
# batch * n_d
masked_sum = T.sum(h_next * masks, axis=0)
lst_states.append(masked_sum / cnt_non_padding) # mean pooling
else:
lst_states.append(h_next[-1]) # last state
h_prev = apply_dropout(h_next, dropout)
if args.use_all:
size = depth * n_d
# batch * size (i.e. n_d*depth)
h_final = T.concatenate(lst_states, axis=1)
else:
size = n_d
h_final = lst_states[-1]
h_final = apply_dropout(h_final, dropout)
output_layer = self.output_layer = Layer(n_in=size,
n_out=self.nclasses,
activation=sigmoid)
# batch * nclasses
preds = self.preds = output_layer.forward(h_final)
# batch
loss_mat = self.loss_mat = (preds - y)**2
pred_diff = self.pred_diff = T.mean(
T.max(preds, axis=1) - T.min(preds, axis=1))
if args.aspect < 0:
loss_vec = T.mean(loss_mat, axis=1)
else:
assert args.aspect < self.nclasses
loss_vec = loss_mat[:, args.aspect]
self.loss_vec = loss_vec
zsum = generator.zsum
zdiff = generator.zdiff
logpz = generator.logpz
coherent_factor = args.sparsity * args.coherent
loss = self.loss = T.mean(loss_vec)
sparsity_cost = self.sparsity_cost = T.mean(zsum) * args.sparsity + \
T.mean(zdiff) * coherent_factor
cost_vec = loss_vec + zsum * args.sparsity + zdiff * coherent_factor
cost_logpz = T.mean(cost_vec * T.sum(logpz, axis=0))
self.obj = T.mean(cost_vec)
params = self.params = []
for l in layers + [output_layer]:
for p in l.params:
params.append(p)
nparams = sum(len(x.get_value(borrow=True).ravel()) \
for x in params)
say("total # parameters: {}\n".format(nparams))
l2_cost = None
for p in params:
if l2_cost is None:
l2_cost = T.sum(p**2)
else:
l2_cost = l2_cost + T.sum(p**2)
l2_cost = l2_cost * args.l2_reg
self.l2_cost = l2_cost
self.cost_g = cost_logpz * 10 + generator.l2_cost
self.cost_e = loss * 10 + l2_cost
def ready(self, args, train):
# len * batch
self.idxs = T.imatrix()
self.idys = T.imatrix()
self.init_state = T.matrix(dtype=theano.config.floatX)
dropout_prob = np.float64(args["dropout"]).astype(theano.config.floatX)
self.dropout = theano.shared(dropout_prob)
self.n_d = args["hidden_dim"]
embedding_layer = EmbeddingLayer(
n_d = self.n_d,
vocab = set(w for w in train)
)
self.n_V = embedding_layer.n_V
say("Vocab size: {}\tHidden dim: {}\n".format(
self.n_V, self.n_d
))
activation = get_activation_by_name(args["activation"])
rnn_layer = LSTM(
n_in = self.n_d,
n_out = self.n_d,
activation = activation
)
output_layer = Layer(
n_in = self.n_d,
n_out = self.n_V,
activation = T.nnet.softmax,
)
# (len*batch) * n_d
x_flat = embedding_layer.forward(self.idxs.ravel())
# len * batch * n_d
x = apply_dropout(x_flat, self.dropout)
x = x.reshape( (self.idxs.shape[0], self.idxs.shape[1], self.n_d) )
# len * batch * (n_d+n_d)
h = rnn_layer.forward_all(x, self.init_state, return_c=True)
self.last_state = h[-1]
h = h[:,:,self.n_d:]
h = apply_dropout(h, self.dropout)
self.p_y_given_x = output_layer.forward(h.reshape(x_flat.shape))
idys = self.idys.ravel()
self.nll = -T.log(self.p_y_given_x[T.arange(idys.shape[0]), idys])
#self.nll = T.nnet.categorical_crossentropy(
# self.p_y_given_x,
# idys
# )
self.layers = [ embedding_layer, rnn_layer, output_layer ]
#self.params = [ x_flat ] + rnn_layer.params + output_layer.params
self.params = embedding_layer.params + rnn_layer.params + output_layer.params
self.num_params = sum(len(x.get_value(borrow=True).ravel())
for l in self.layers for x in l.params)
say("# of params in total: {}\n".format(self.num_params))
import numpy as np
from nn import NN, Layer
X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
Y = np.array([[0], [1], [1], [0]])
nn = NN(2)
nn.add(Layer(3))
nn.add(Layer(1))
print("Predictions before training")
print(nn.feed_forward(X))
print()
nn.train(X, Y)
print()
print("Predictions after training")
print(nn.feed_forward(X))
def ready(self):
args = self.args
index = self.index = T.lscalar()
x = self.x = T.fmatrix()
y = self.y = T.ivector()
dropout = self.dropout = theano.shared(np.float64(args.dropout).astype(
"float32"))
n_d = args.hidden_dim
layers = self.layers = [ ]
for i in xrange(args.depth):
l = Layer(
n_in = 28*28 if i == 0 else n_d,
n_out = n_d,
activation = ReLU
)
layers.append(l)
output_layer = self.output_layer = Layer(
n_in = n_d,
n_out = 10,
activation = softmax
)
h = x
for l in layers:
h = l.forward(h)
h = apply_dropout(h, dropout)
self.h_final = h
# batch * 10
probs = self.probs = output_layer.forward(h)
# batch
preds = self.preds = T.argmax(probs, axis=1)
err = self.err = T.mean(T.cast(T.neq(preds, y), dtype="float32"))
#
loss = self.loss = -T.mean( T.log(probs[T.arange(y.shape[0]), y]) )
#loss = self.loss = T.mean( T.nnet.categorical_crossentropy(
# probs,
# y
# ))
params = self.params = [ ]
for l in layers + [ output_layer ]:
for p in l.params:
params.append(p)
l2_cost = None
for p in params:
if l2_cost is None:
l2_cost = T.sum(p**2)
else:
l2_cost += T.sum(p**2)
l2_cost = l2_cost * args.l2_reg
self.l2_cost = l2_cost
self.cost = loss + l2_cost
print "cost.dtype", self.cost.dtype
