class DataBlock(object):
def __init__(self, char_to_idx, device_id):
self.context = Context(device_id)
device_id = self.context.device_id
self.char_idx = Connector(Matrix.empty(1, 1, 'int', device_id))
self.char_to_idx = char_to_idx
self.char = None
def fprop(self):
char_npa = np.zeros((1, 1), np.int32, 'F')
char_npa[0][0] = self.char_to_idx[self.char] if self.char in self.char_to_idx else self.char_to_idx['<unk>']
self.char_idx.assign_npa(self.context, char_npa)
self.char_idx.fprop()
class DataBlock(object):
def __init__(self, word_to_idx, device_id):
self.context = Context(device_id)
device_id = self.context.device_id
self.word_idx = Connector(Matrix.empty(1, 1, 'int', device_id))
self.word_to_idx = word_to_idx
self.word = None
def fprop(self):
word_npa = np.zeros((1, 1), np.int32, 'F')
word_npa[0][0] = self.word_to_idx[self.word] if self.word in self.word_to_idx else self.word_to_idx['<UNK>']
self.word_idx.assign_npa(self.context, word_npa)
self.word_idx.fprop()
class DataBlock(object):
def __init__(self, data, char_to_idx, batch_size, x_device_id,
y_device_id):
self.data = HomogeneousDataIterator(data, char_to_idx, batch_size,
True, True)
self.data_iterator = iter(self.data)
self.x_context = Context(x_device_id)
self.y_context = Context(y_device_id)
max_len = 0
for sub_line in data:
cur_len = len(sub_line)
if cur_len > max_len:
max_len = cur_len
print max_len
self.x = Connector(
Matrix.empty(batch_size, max_len - 1, 'int', x_device_id))
self._y = Matrix.empty(batch_size, max_len - 1, 'int', y_device_id)
self.y = List([Connector(self._y[:, i]) for i in xrange(max_len - 1)],
self.x.ncols)
self.lengths = Matrix.empty(self.x.nrows, 1, 'int', x_device_id)
self._mask = Matrix.empty(self.x.nrows, self.x.ncols, 'float',
x_device_id)
self.mask = List(
[Connector(self._mask[:, i]) for i in xrange(max_len)],
self.x.ncols)
self.blocking_contexts = None
def fprop(self):
self.x_context.wait(*self.blocking_contexts)
self.y_context.wait(*self.blocking_contexts)
data = next(self.data_iterator)
lengths_npa = np.array([[len(e) - 1] for e in data],
np.int32,
order='F')
x_npa = np.zeros((len(data), int(np.max(lengths_npa))), np.int32, 'F')
for k, e in enumerate(data):
x_npa[k, :len(e) - 1] = e[:-1]
self.x.assign_npa(self.x_context, x_npa)
y_npa = np.zeros((len(data), int(np.max(lengths_npa))), np.int32, 'F')
for k, e in enumerate(data):
y_npa[k, :len(e) - 1] = e[1:]
self._y.assign_npa(self.y_context, y_npa)
for e in self.y:
e.last_modification_context = self.y_context
self.lengths.assign_npa(self.x_context, lengths_npa)
self._mask.mask_column_numbers_row_wise(self.x_context, self.lengths)
for e in self.mask:
e.last_modification_context = self.x_context
self.x.fprop()
self.y.fprop()
self.mask.fprop()
class DataBlock(object):
def __init__(self, char_to_idx, device_id):
self.context = Context(device_id)
device_id = self.context.device_id
self.char_idx = Connector(Matrix.empty(1, 1, 'int', device_id))
self.char_to_idx = char_to_idx
self.char = None
def fprop(self):
char_npa = np.zeros((1, 1), np.int32, 'F')
char_npa[0][0] = self.char_to_idx[
self.char] if self.char in self.char_to_idx else self.char_to_idx[
'<unk>']
self.char_idx.assign_npa(self.context, char_npa)
self.char_idx.fprop()
class DataBlock(object):
def __init__(self, word_to_idx, device_id):
self.context = Context(device_id)
device_id = self.context.device_id
self.word_idx = Connector(Matrix.empty(1, 1, 'int', device_id))
self.word_to_idx = word_to_idx
self.word = None
def fprop(self):
word_npa = np.zeros((1, 1), np.int32, 'F')
word_npa[0][0] = self.word_to_idx[
self.word] if self.word in self.word_to_idx else self.word_to_idx[
'<UNK>']
self.word_idx.assign_npa(self.context, word_npa)
self.word_idx.fprop()
class DataBlock(object):
def __init__(self, data, char_to_idx, batch_size, x_device_id, y_device_id):
self.data = HomogeneousDataIterator(data, char_to_idx, batch_size, True, True)
self.data_iterator = iter(self.data)
self.x_context = Context(x_device_id)
self.y_context = Context(y_device_id)
max_len = 0
for sub_line in data:
cur_len = len(sub_line)
if cur_len > max_len:
max_len = cur_len
print max_len
self.x = Connector(Matrix.empty(batch_size, max_len - 1, 'int', x_device_id))
self._y = Matrix.empty(batch_size, max_len - 1, 'int', y_device_id)
self.y = List([Connector(self._y[:, i]) for i in xrange(max_len - 1)], self.x.ncols)
self.lengths = Matrix.empty(self.x.nrows, 1, 'int', x_device_id)
self._mask = Matrix.empty(self.x.nrows, self.x.ncols, 'float', x_device_id)
self.mask = List([Connector(self._mask[:, i]) for i in xrange(max_len)], self.x.ncols)
self.blocking_contexts = None
def fprop(self):
self.x_context.wait(*self.blocking_contexts)
self.y_context.wait(*self.blocking_contexts)
data = next(self.data_iterator)
lengths_npa = np.array([[len(e) - 1] for e in data], np.int32, order='F')
x_npa = np.zeros((len(data), int(np.max(lengths_npa))), np.int32, 'F')
for k, e in enumerate(data):
x_npa[k, :len(e) - 1] = e[:-1]
self.x.assign_npa(self.x_context, x_npa)
y_npa = np.zeros((len(data), int(np.max(lengths_npa))), np.int32, 'F')
for k, e in enumerate(data):
y_npa[k, :len(e) - 1] = e[1:]
self._y.assign_npa(self.y_context, y_npa)
for e in self.y:
e.last_modification_context = self.y_context
self.lengths.assign_npa(self.x_context, lengths_npa)
self._mask.mask_column_numbers_row_wise(self.x_context, self.lengths)
for e in self.mask:
e.last_modification_context = self.x_context
self.x.fprop()
self.y.fprop()
self.mask.fprop()
class DataBlock(object):
def __init__(self, train_data, valid_data, batch_size, word_dropout_prob, device_id):
self.train_data = HomogeneousDataIterator(train_data, batch_size, randomize=True, infinite=True)
self.valid_data = HomogeneousDataIterator(valid_data, batch_size)
self.train_data_iterator = iter(self.train_data)
self.valid_data_iterator = iter(self.valid_data)
self.word_keep_prob = 1.0 - word_dropout_prob
self.rnd = RandomState(47571)
self.unk_idx = word_to_idx['<UNK>']
self.context = Context(device_id)
c = Counter([len(line) for line in chain(train_data, valid_data)])
print c.most_common()
max_len = max([len(line) for line in chain(train_data, valid_data)])
self.enc_x = Connector(Matrix.empty(batch_size, max_len, 'int', device_id))
self.enc_lengths = Matrix.empty(self.enc_x.nrows, 1, 'int', device_id)
self._enc_mask = Matrix.empty(self.enc_x.nrows, self.enc_x.ncols, 'float', device_id)
self.enc_mask = List([Connector(self._enc_mask[:, i]) for i in xrange(max_len)], self.enc_x.ncols)
self.dec_x = Connector(Matrix.empty(batch_size, max_len + 1, 'int', device_id))
self._dec_y = Matrix.empty(batch_size, max_len + 1, 'int', device_id)
self.dec_y = List([Connector(self._dec_y[:, i]) for i in xrange(max_len + 1)], self._dec_y.ncols)
self.dec_lengths = Matrix.empty(self.dec_x.nrows, 1, 'int', device_id)
self._dec_mask = Matrix.empty(self.dec_x.nrows, self.dec_x.ncols, 'float', device_id)
self.dec_mask = List([Connector(self._dec_mask[:, i]) for i in xrange(max_len + 1)], self.dec_x.ncols)
self.blocking_contexts = None
self.training_mode = True
def set_training_mode(self):
self.training_mode = True
def set_testing_mode(self):
self.training_mode = False
def fprop(self):
if self.training_mode:
data = next(self.train_data_iterator)
else:
try:
data = next(self.valid_data_iterator)
except StopIteration as e:
self.valid_data_iterator = iter(self.valid_data)
raise e
lengths_npa = np.array([[len(e)] for e in data], np.int32, order='F')
max_len = int(np.max(lengths_npa))
self.enc_lengths.assign_npa(self.context, lengths_npa)
self._enc_mask.mask_column_numbers_row_wise(self.context, self.enc_lengths)
for e in self.enc_mask:
e.last_modification_context = self.context
lengths_npa += 1
self.dec_lengths.assign_npa(self.context, lengths_npa)
self._dec_mask.mask_column_numbers_row_wise(self.context, self.dec_lengths)
for e in self.dec_mask:
e.last_modification_context = self.context
enc_x_npa = np.zeros((len(data), max_len), np.int32, 'F')
dec_x_npa = np.zeros((len(data), max_len + 1), np.int32, 'F')
dec_y_npa = np.zeros((len(data), max_len + 1), np.int32, 'F')
for k, e in enumerate(data):
enc_x_npa[k, :len(e)] = e
if self.training_mode:
new_e = [_ if self.rnd.rand() < self.word_keep_prob else self.unk_idx for _ in e]
else:
new_e = e
dec_x_npa[k, :len(e) + 1] = [word_to_idx['<<S>>']] + new_e
dec_y_npa[k, :len(e) + 1] = e + [word_to_idx['<<S>>']]
self.enc_x.assign_npa(self.context, enc_x_npa)
self.dec_x.assign_npa(self.context, dec_x_npa)
self._dec_y.assign_npa(self.context, dec_y_npa)
for e in self.dec_y:
e.last_modification_context = self.context
self.enc_mask.fprop()
self.dec_mask.fprop()
self.enc_x.fprop()
self.dec_x.fprop()
self.dec_y.fprop()
def test_theano_grad(self):
quagga.processor_type = 'gpu'
r = []
for i in xrange(self.N):
max_input_sequence_len = self.rng.random_integers(300)
sequence_len = max_input_sequence_len if i == 0 else self.rng.random_integers(
max_input_sequence_len)
batch_size = self.rng.random_integers(128)
input_dim, hidden_dim, class_num = self.rng.random_integers(1500,
size=3)
x = [
self.rng.randn(batch_size, input_dim).astype(np.float32)
for _ in xrange(max_input_sequence_len)
]
true_labels = [
self.rng.randint(class_num,
size=(batch_size, 1)).astype(np.int32)
for _ in xrange(max_input_sequence_len)
]
mask = (self.rng.rand(batch_size, sequence_len) < 0.8).astype(
np.float32)
h_0 = self.rng.randn(batch_size, hidden_dim).astype(np.float32)
c_0 = self.rng.randn(batch_size, hidden_dim).astype(np.float32)
W_z = self.get_orthogonal_matrix(input_dim, hidden_dim)
W_i = self.get_orthogonal_matrix(input_dim, hidden_dim)
W_f = self.get_orthogonal_matrix(input_dim, hidden_dim)
W_o = self.get_orthogonal_matrix(input_dim, hidden_dim)
W = np.hstack((W_z, W_i, W_f, W_o))
R_z = self.get_orthogonal_matrix(hidden_dim, hidden_dim)
R_i = self.get_orthogonal_matrix(hidden_dim, hidden_dim)
R_f = self.get_orthogonal_matrix(hidden_dim, hidden_dim)
R_o = self.get_orthogonal_matrix(hidden_dim, hidden_dim)
R = np.hstack((R_z, R_i, R_f, R_o))
lr_W = self.get_orthogonal_matrix(hidden_dim, class_num)
lr_b = self.rng.rand(1, class_num).astype(dtype=np.float32)
device_id = 0
for reverse in [False, True]:
for with_mask in [False, True]:
for learn_inital_states in [False, True]:
# quagga model
context = Context()
qx = List([
Connector(Matrix.from_npa(e), device_id) for e in x
])
qtrue_labels = List([
Connector(Matrix.from_npa(e)) for e in true_labels
], qx.length)
qmask = Matrix.empty(batch_size, qx.length, 'float')
qmask_list = [
Connector(qmask[:, i]) for i in xrange(qmask.ncols)
]
qmask = Connector(qmask)
qh_0 = Connector(
Matrix.from_npa(h_0),
device_id if learn_inital_states else None)
qc_0 = Connector(
Matrix.from_npa(c_0),
device_id if learn_inital_states else None)
qW = Connector(Matrix.from_npa(W), device_id)
qR = Connector(Matrix.from_npa(R), device_id)
qlr_W = Connector(Matrix.from_npa(lr_W), device_id)
qlr_b = Connector(Matrix.from_npa(lr_b), device_id)
lstm = SequencerBlock(
block_class=LstmBlock,
params=[qW, qR],
sequences=[
qx,
qmask_list if with_mask else [None] * len(qx)
],
output_names=['h'],
prev_names=['c', 'h'],
paddings=[qc_0, qh_0],
reverse=reverse)
seq_dot_block = SequencerBlock(block_class=DotBlock,
params=[qlr_W, qlr_b],
sequences=[lstm.h],
output_names=['output'])
seq_sce_block = SequencerBlock(
block_class=SoftmaxCeBlock,
params=[],
sequences=[
seq_dot_block.output, qtrue_labels,
qmask_list if with_mask else [None] * len(qx)
])
qx.length = sequence_len
for e in qx:
e.fprop()
for e in qtrue_labels:
e.fprop()
qmask.assign_npa(context, mask)
qmask.fprop()
qlr_W.fprop()
qlr_b.fprop()
qh_0.fprop()
qc_0.fprop()
qW.fprop()
qR.fprop()
lstm.fprop()
seq_dot_block.fprop()
seq_sce_block.fprop()
seq_sce_block.bprop()
seq_dot_block.bprop()
lstm.bprop()
quagga_grads = [
qlr_b.backward_matrix.to_host(),
qlr_W.backward_matrix.to_host(),
qW.backward_matrix.to_host(),
qR.backward_matrix.to_host()
]
if learn_inital_states:
quagga_grads.append(qc_0.backward_matrix.to_host())
quagga_grads.append(qh_0.backward_matrix.to_host())
quagga_grads.append(
[e.backward_matrix.to_host() for e in qx])
del qx
del qlr_b
del qlr_W
del qW
del qR
del qmask
del lstm
del seq_dot_block
del seq_sce_block
# theano model
th_x = T.ftensor3()
th_true_labels = T.imatrix()
th_mask = T.fmatrix()
lstm_layer = LstmLayer(W, R, c_0, h_0, reverse=reverse)
th_h = lstm_layer.get_output_expr(
th_x, th_mask if with_mask else None)
seq_softmax_layer = SequentialSoftmaxLayer(
lr_W, lr_b, reverse)
loss = seq_softmax_layer.get_loss(
th_h, th_true_labels,
th_mask if with_mask else None)
wrt = [
seq_softmax_layer.b, seq_softmax_layer.W,
lstm_layer.W, lstm_layer.R
]
if learn_inital_states:
wrt.append(lstm_layer.c0)
wrt.append(lstm_layer.h0)
wrt.append(th_x)
grads = T.grad(loss, wrt)
if with_mask:
get_theano_grads = theano.function(
[th_x, th_true_labels, th_mask], grads)
theano_grads = get_theano_grads(
np.dstack(x[:sequence_len]),
np.hstack(true_labels[:sequence_len]),
mask[:, :sequence_len])
else:
get_theano_grads = theano.function(
[th_x, th_true_labels], grads)
theano_grads = get_theano_grads(
np.dstack(x[:sequence_len]),
np.hstack(true_labels[:sequence_len]))
for quagga_grad, theano_grad in izip(
quagga_grads[:-1], theano_grads[:-1]):
r.append(
np.allclose(quagga_grad,
theano_grad,
atol=1e-6))
for i in xrange(theano_grads[-1].shape[-1]):
if not np.allclose(quagga_grads[-1][i],
theano_grads[-1][..., i],
atol=1e-6):
r.append(False)
break
else:
r.append(True)
self.assertEqual(sum(r), len(r))
def test_theano_fprop(self):
quagga.processor_type = 'gpu'
r = []
for i in xrange(self.N):
max_input_sequence_len = self.rng.random_integers(500)
sequence_len = max_input_sequence_len if i == 0 else self.rng.random_integers(
max_input_sequence_len)
batch_size = self.rng.random_integers(256)
input_dim, hidden_dim = self.rng.random_integers(1500, size=2)
x = [
self.rng.randn(batch_size, input_dim).astype(np.float32)
for _ in xrange(max_input_sequence_len)
]
mask = (self.rng.rand(batch_size, sequence_len) < 0.8).astype(
np.float32)
h_0 = self.rng.randn(batch_size, hidden_dim).astype(np.float32)
c_0 = self.rng.randn(batch_size, hidden_dim).astype(np.float32)
W_z = self.get_orthogonal_matrix(input_dim, hidden_dim)
W_i = self.get_orthogonal_matrix(input_dim, hidden_dim)
W_f = self.get_orthogonal_matrix(input_dim, hidden_dim)
W_o = self.get_orthogonal_matrix(input_dim, hidden_dim)
W = np.hstack((W_z, W_i, W_f, W_o))
R_z = self.get_orthogonal_matrix(hidden_dim, hidden_dim)
R_i = self.get_orthogonal_matrix(hidden_dim, hidden_dim)
R_f = self.get_orthogonal_matrix(hidden_dim, hidden_dim)
R_o = self.get_orthogonal_matrix(hidden_dim, hidden_dim)
R = np.hstack((R_z, R_i, R_f, R_o))
for reverse in [False, True]:
for with_mask in [False, True]:
context = Context()
qx = List([Connector(Matrix.from_npa(e)) for e in x])
qmask = Connector(
Matrix.empty(batch_size, len(qx), 'float'))
qh_0 = Connector(Matrix.from_npa(h_0))
qc_0 = Connector(Matrix.from_npa(c_0))
qW = Connector(Matrix.from_npa(W))
qR = Connector(Matrix.from_npa(R))
lstm = SequencerBlock(block_class=LstmBlock,
params=[qW, qR],
sequences=[qx] +
([qmask] if with_mask else []),
output_names=['h'],
prev_names=['c', 'h'],
paddings=[qc_0, qh_0],
reverse=reverse)
qx.length = sequence_len
for e in qx:
e.fprop()
qmask.assign_npa(context, mask)
qmask.fprop()
qh_0.fprop()
qc_0.fprop()
qW.fprop()
qR.fprop()
lstm.fprop()
q_h = lstm.h.to_host()
th_x = T.ftensor3()
lstm_layer = LstmLayer(W, R, c_0, h_0, reverse)
if with_mask:
th_mask = T.fmatrix()
get_th_h = theano.function([th_x, th_mask],
lstm_layer.get_output_expr(
th_x, th_mask))
th_h = get_th_h(np.dstack(x[:sequence_len]),
mask[:, :sequence_len])
else:
get_th_h = theano.function(
[th_x], lstm_layer.get_output_expr(th_x))
th_h = get_th_h(np.dstack(x[:sequence_len]))
for i in xrange(th_h.shape[0]):
if not np.allclose(q_h[i], th_h[i]):
r.append(False)
break
else:
r.append(True)
self.assertEqual(sum(r), len(r))
