def test_bprop(self):
"""
compare `bprop` results for cpu and gpu backends
"""
r = []
for i in xrange(self.N):
max_input_sequence_len = self.rng.random_integers(500)
sequence_len = max_input_sequence_len if i == 0 else self.rng.random_integers(
max_input_sequence_len)
batch_size = self.rng.random_integers(512)
dim = self.rng.random_integers(1500)
x = [
self.rng.rand(batch_size, dim).astype(dtype=np.float32)
for _ in xrange(max_input_sequence_len)
]
state = self.rng.get_state()
quagga.processor_type = 'gpu'
context = Context()
x_gpu = List(
[Connector(Matrix.from_npa(e), context, context) for e in x])
smean_pooling_block_gpu = SequentialMeanPoolingBlock(x_gpu)
x_gpu.set_length(sequence_len)
_, dL_doutput = smean_pooling_block_gpu.output.register_usage(
context, context)
smean_pooling_block_gpu.fprop()
random_matrix = self.rng.rand(dL_doutput.nrows, dL_doutput.ncols)
Matrix.from_npa(random_matrix,
'float').copy_to(context, dL_doutput)
smean_pooling_block_gpu.bprop()
dL_dmatrices_gpu = [e.backward_matrix.to_host() for e in x_gpu]
self.rng.set_state(state)
quagga.processor_type = 'cpu'
context = Context()
x_cpu = List(
[Connector(Matrix.from_npa(e), context, context) for e in x])
smean_pooling_block_cpu = SequentialMeanPoolingBlock(x_cpu)
x_cpu.set_length(sequence_len)
_, dL_doutput = smean_pooling_block_cpu.output.register_usage(
context, context)
smean_pooling_block_cpu.fprop()
random_matrix = self.rng.rand(dL_doutput.nrows, dL_doutput.ncols)
Matrix.from_npa(random_matrix,
'float').copy_to(context, dL_doutput)
smean_pooling_block_cpu.bprop()
dL_dmatrices_cpu = [e.backward_matrix.to_host() for e in x_cpu]
for dL_dmatrix_gpu, dL_dmatrix_cpu in izip(dL_dmatrices_gpu,
dL_dmatrices_cpu):
if not np.allclose(dL_dmatrix_gpu, dL_dmatrix_cpu):
r.append(False)
break
else:
r.append(True)
self.assertEqual(sum(r), self.N)
def test_bprop(self):
"""
compare `bprop` results for cpu and gpu backends
"""
r = []
for i in xrange(self.N):
max_input_sequence_len = self.rng.random_integers(500)
sequence_len = max_input_sequence_len if i == 0 else self.rng.random_integers(max_input_sequence_len)
batch_size = self.rng.random_integers(512)
dim = self.rng.random_integers(1500)
x = [self.rng.rand(batch_size, dim).astype(dtype=np.float32) for _ in xrange(max_input_sequence_len)]
state = self.rng.get_state()
quagga.processor_type = 'gpu'
context = Context()
x_gpu = List([Connector(Matrix.from_npa(e), context, context) for e in x])
smean_pooling_block_gpu = SequentialMeanPoolingBlock(x_gpu)
x_gpu.set_length(sequence_len)
_, dL_doutput = smean_pooling_block_gpu.output.register_usage(context, context)
smean_pooling_block_gpu.fprop()
random_matrix = self.rng.rand(dL_doutput.nrows, dL_doutput.ncols)
Matrix.from_npa(random_matrix, 'float').copy_to(context, dL_doutput)
smean_pooling_block_gpu.bprop()
dL_dmatrices_gpu = [e.backward_matrix.to_host() for e in x_gpu]
self.rng.set_state(state)
quagga.processor_type = 'cpu'
context = Context()
x_cpu = List([Connector(Matrix.from_npa(e), context, context) for e in x])
smean_pooling_block_cpu = SequentialMeanPoolingBlock(x_cpu)
x_cpu.set_length(sequence_len)
_, dL_doutput = smean_pooling_block_cpu.output.register_usage(context, context)
smean_pooling_block_cpu.fprop()
random_matrix = self.rng.rand(dL_doutput.nrows, dL_doutput.ncols)
Matrix.from_npa(random_matrix, 'float').copy_to(context, dL_doutput)
smean_pooling_block_cpu.bprop()
dL_dmatrices_cpu = [e.backward_matrix.to_host() for e in x_cpu]
for dL_dmatrix_gpu, dL_dmatrix_cpu in izip(dL_dmatrices_gpu, dL_dmatrices_cpu):
if not np.allclose(dL_dmatrix_gpu, dL_dmatrix_cpu):
r.append(False)
break
else:
r.append(True)
self.assertEqual(sum(r), self.N)
def test_theano_grad(self):
class SequentialMeanPoolingLayer(object):
def get_output_expr(self, input_sequence):
return T.mean(input_sequence, axis=2)
class LogisticRegressionLayer(object):
def __init__(self, W_init, b_init):
self.W = theano.shared(value=W_init())
self.b = theano.shared(value=b_init())
def get_output_expr(self, input_expr):
return T.nnet.sigmoid(T.dot(input_expr, self.W) + self.b)
quagga.processor_type = 'gpu'
r = []
for i in xrange(self.N):
max_input_sequence_len = self.rng.random_integers(500)
sequence_len = max_input_sequence_len if i == 0 else self.rng.random_integers(max_input_sequence_len)
batch_size = self.rng.random_integers(512)
dim = self.rng.random_integers(1500)
x = [self.rng.rand(batch_size, dim).astype(dtype=np.float32) for _ in xrange(max_input_sequence_len)]
true_labels = self.rng.randint(1, size=(batch_size, 1)).astype(dtype=np.float32)
W_init = self.get_orthogonal_initializer(dim, 1)
b_init = lambda: self.rng.rand(1, 1).astype(dtype=np.float32)
# Theano model
state = self.rng.get_state()
th_x = T.ftensor3()
th_true_labels = T.fmatrix()
smp_layer = SequentialMeanPoolingLayer()
lr_layer = LogisticRegressionLayer(W_init, lambda: b_init()[0])
probs = lr_layer.get_output_expr(smp_layer.get_output_expr(th_x))
loss = T.mean(T.nnet.binary_crossentropy(probs, th_true_labels))
grad_x = T.grad(loss, wrt=th_x)
get_grad_x = theano.function([th_x, th_true_labels], grad_x)
# quagga model
self.rng.set_state(state)
context = Context()
x = List([Connector(Matrix.from_npa(e), context, context) for e in x])
true_labels = Connector(Matrix.from_npa(true_labels))
smp_block = SequentialMeanPoolingBlock(x)
dot_block = DotBlock(W_init, b_init, smp_block.output)
sce_block = SigmoidCeBlock(dot_block.output, true_labels)
x.set_length(sequence_len)
smp_block.fprop()
dot_block.fprop()
sce_block.fprop()
sce_block.bprop()
dot_block.bprop()
smp_block.bprop()
dL_dx = [e.backward_matrix.to_host() for e in x]
dL_dx_th = get_grad_x(np.dstack([e.to_host() for e in x]), true_labels.to_host())
for i in xrange(dL_dx_th.shape[-1]):
if not np.allclose(dL_dx[i], dL_dx_th[..., i]):
r.append(False)
break
else:
r.append(True)
self.assertEqual(sum(r), self.N)
def test_theano_grad(self):
device_id = 0
class SequentialHorizontalStackLayer(object):
def get_output_expr(self, x_sequence, y_sequence):
return T.concatenate((x_sequence, y_sequence), axis=1)
class SequentialMeanPoolingLayer(object):
def get_output_expr(self, input_sequence):
return T.mean(input_sequence, axis=2)
class LogisticRegressionLayer(object):
def __init__(self, W_init, b_init):
self.W = theano.shared(value=W_init())
self.b = theano.shared(value=b_init())
def get_output_expr(self, input_expr):
return T.nnet.sigmoid(T.dot(input_expr, self.W) + self.b)
quagga.processor_type = 'gpu'
r = []
for i in xrange(self.N):
max_input_sequence_len = self.rng.random_integers(500)
sequence_len = max_input_sequence_len if i == 0 else self.rng.random_integers(max_input_sequence_len)
batch_size = self.rng.random_integers(256)
dim_x, dim_y = self.rng.random_integers(1280, size=2)
x = [self.rng.rand(batch_size, dim_x).astype(dtype=np.float32) for _ in xrange(max_input_sequence_len)]
y = [self.rng.rand(batch_size, dim_y).astype(dtype=np.float32) for _ in xrange(max_input_sequence_len)]
true_labels = self.rng.randint(1, size=(batch_size, 1)).astype(dtype=np.float32)
W_init = self.get_orthogonal_initializer(dim_x + dim_y, 1)
b_init = lambda: self.rng.rand(1, 1).astype(dtype=np.float32)
# Theano model
state = self.rng.get_state()
th_x = T.ftensor3()
th_y = T.ftensor3()
th_true_labels = T.fmatrix()
shs_layer = SequentialHorizontalStackLayer()
smp_layer = SequentialMeanPoolingLayer()
lr_layer = LogisticRegressionLayer(W_init, lambda: b_init()[0])
probs = shs_layer.get_output_expr(th_x, th_y)
probs = lr_layer.get_output_expr(smp_layer.get_output_expr(probs))
loss = T.mean(T.nnet.binary_crossentropy(probs, th_true_labels))
grads = T.grad(loss, wrt=[th_x, th_y])
get_grads = theano.function([th_x, th_y, th_true_labels], grads)
dL_dx_sequence_th, dL_dy_sequence_th = get_grads(np.dstack(x[:sequence_len]), np.dstack(y[:sequence_len]), true_labels)
# quagga model
self.rng.set_state(state)
W = Connector(Matrix.from_npa(W_init(), device_id=device_id), device_id)
b = Connector(Matrix.from_npa(b_init(), device_id=device_id), device_id)
x = List([Connector(Matrix.from_npa(e), device_id) for e in x])
y = List([Connector(Matrix.from_npa(e), device_id) for e in y])
true_labels = Connector(Matrix.from_npa(true_labels))
shs_block = SequentialHorizontalStackBlock(x, y)
smp_block = SequentialMeanPoolingBlock(shs_block.output)
dot_block = DotBlock(W, b, smp_block.output)
sce_block = SigmoidCeBlock(dot_block.output, true_labels)
x.length = sequence_len
y.length = sequence_len
shs_block.fprop()
smp_block.fprop()
dot_block.fprop()
sce_block.fprop()
sce_block.bprop()
dot_block.bprop()
smp_block.bprop()
shs_block.bprop()
dL_dx_sequence = [e.backward_matrix.to_host() for e in x]
dL_dy_sequence = [e.backward_matrix.to_host() for e in y]
for i in xrange(dL_dx_sequence_th.shape[-1]):
if not np.allclose(dL_dx_sequence[i], dL_dx_sequence_th[..., i], atol=1.e-6):
r.append(False)
break
else:
r.append(True)
for i in xrange(dL_dy_sequence_th.shape[-1]):
if not np.allclose(dL_dy_sequence[i], dL_dy_sequence_th[..., i], atol=1.e-6):
r.append(False)
break
else:
r.append(True)
self.assertEqual(sum(r), self.N * 2)
def test_theano_grad(self):
class SequentialMeanPoolingLayer(object):
def get_output_expr(self, input_sequence):
return T.mean(input_sequence, axis=2)
class LogisticRegressionLayer(object):
def __init__(self, W_init, b_init):
self.W = theano.shared(value=W_init())
self.b = theano.shared(value=b_init())
def get_output_expr(self, input_expr):
return T.nnet.sigmoid(T.dot(input_expr, self.W) + self.b)
quagga.processor_type = 'gpu'
r = []
for i in xrange(self.N):
max_input_sequence_len = self.rng.random_integers(500)
sequence_len = max_input_sequence_len if i == 0 else self.rng.random_integers(
max_input_sequence_len)
batch_size = self.rng.random_integers(512)
dim = self.rng.random_integers(1500)
x = [
self.rng.rand(batch_size, dim).astype(dtype=np.float32)
for _ in xrange(max_input_sequence_len)
]
true_labels = self.rng.randint(1,
size=(batch_size,
1)).astype(dtype=np.float32)
W_init = self.get_orthogonal_initializer(dim, 1)
b_init = lambda: self.rng.rand(1, 1).astype(dtype=np.float32)
# Theano model
state = self.rng.get_state()
th_x = T.ftensor3()
th_true_labels = T.fmatrix()
smp_layer = SequentialMeanPoolingLayer()
lr_layer = LogisticRegressionLayer(W_init, lambda: b_init()[0])
probs = lr_layer.get_output_expr(smp_layer.get_output_expr(th_x))
loss = T.mean(T.nnet.binary_crossentropy(probs, th_true_labels))
grad_x = T.grad(loss, wrt=th_x)
get_grad_x = theano.function([th_x, th_true_labels], grad_x)
# quagga model
self.rng.set_state(state)
context = Context()
x = List(
[Connector(Matrix.from_npa(e), context, context) for e in x])
true_labels = Connector(Matrix.from_npa(true_labels))
smp_block = SequentialMeanPoolingBlock(x)
dot_block = DotBlock(W_init, b_init, smp_block.output)
sce_block = SigmoidCeBlock(dot_block.output, true_labels)
x.set_length(sequence_len)
smp_block.fprop()
dot_block.fprop()
sce_block.fprop()
sce_block.bprop()
dot_block.bprop()
smp_block.bprop()
dL_dx = [e.backward_matrix.to_host() for e in x]
dL_dx_th = get_grad_x(np.dstack([e.to_host() for e in x]),
true_labels.to_host())
for i in xrange(dL_dx_th.shape[-1]):
if not np.allclose(dL_dx[i], dL_dx_th[..., i]):
r.append(False)
break
else:
r.append(True)
self.assertEqual(sum(r), self.N)
def test_theano_grad(self):
device_id = 0
class SequentialHorizontalStackLayer(object):
def get_output_expr(self, x_sequence, y_sequence):
return T.concatenate((x_sequence, y_sequence), axis=1)
class SequentialMeanPoolingLayer(object):
def get_output_expr(self, input_sequence):
return T.mean(input_sequence, axis=2)
class LogisticRegressionLayer(object):
def __init__(self, W_init, b_init):
self.W = theano.shared(value=W_init())
self.b = theano.shared(value=b_init())
def get_output_expr(self, input_expr):
return T.nnet.sigmoid(T.dot(input_expr, self.W) + self.b)
quagga.processor_type = 'gpu'
r = []
for i in xrange(self.N):
max_input_sequence_len = self.rng.random_integers(500)
sequence_len = max_input_sequence_len if i == 0 else self.rng.random_integers(
max_input_sequence_len)
batch_size = self.rng.random_integers(256)
dim_x, dim_y = self.rng.random_integers(1280, size=2)
x = [
self.rng.rand(batch_size, dim_x).astype(dtype=np.float32)
for _ in xrange(max_input_sequence_len)
]
y = [
self.rng.rand(batch_size, dim_y).astype(dtype=np.float32)
for _ in xrange(max_input_sequence_len)
]
true_labels = self.rng.randint(1,
size=(batch_size,
1)).astype(dtype=np.float32)
W_init = self.get_orthogonal_initializer(dim_x + dim_y, 1)
b_init = lambda: self.rng.rand(1, 1).astype(dtype=np.float32)
# Theano model
state = self.rng.get_state()
th_x = T.ftensor3()
th_y = T.ftensor3()
th_true_labels = T.fmatrix()
shs_layer = SequentialHorizontalStackLayer()
smp_layer = SequentialMeanPoolingLayer()
lr_layer = LogisticRegressionLayer(W_init, lambda: b_init()[0])
probs = shs_layer.get_output_expr(th_x, th_y)
probs = lr_layer.get_output_expr(smp_layer.get_output_expr(probs))
loss = T.mean(T.nnet.binary_crossentropy(probs, th_true_labels))
grads = T.grad(loss, wrt=[th_x, th_y])
get_grads = theano.function([th_x, th_y, th_true_labels], grads)
dL_dx_sequence_th, dL_dy_sequence_th = get_grads(
np.dstack(x[:sequence_len]), np.dstack(y[:sequence_len]),
true_labels)
# quagga model
self.rng.set_state(state)
W = Connector(Matrix.from_npa(W_init(), device_id=device_id),
device_id)
b = Connector(Matrix.from_npa(b_init(), device_id=device_id),
device_id)
x = List([Connector(Matrix.from_npa(e), device_id) for e in x])
y = List([Connector(Matrix.from_npa(e), device_id) for e in y])
true_labels = Connector(Matrix.from_npa(true_labels))
shs_block = SequentialHorizontalStackBlock(x, y)
smp_block = SequentialMeanPoolingBlock(shs_block.output)
dot_block = DotBlock(W, b, smp_block.output)
sce_block = SigmoidCeBlock(dot_block.output, true_labels)
x.length = sequence_len
y.length = sequence_len
shs_block.fprop()
smp_block.fprop()
dot_block.fprop()
sce_block.fprop()
sce_block.bprop()
dot_block.bprop()
smp_block.bprop()
shs_block.bprop()
dL_dx_sequence = [e.backward_matrix.to_host() for e in x]
dL_dy_sequence = [e.backward_matrix.to_host() for e in y]
for i in xrange(dL_dx_sequence_th.shape[-1]):
if not np.allclose(dL_dx_sequence[i],
dL_dx_sequence_th[..., i],
atol=1.e-6):
r.append(False)
break
else:
r.append(True)
for i in xrange(dL_dy_sequence_th.shape[-1]):
if not np.allclose(dL_dy_sequence[i],
dL_dy_sequence_th[..., i],
atol=1.e-6):
r.append(False)
break
else:
r.append(True)
self.assertEqual(sum(r), self.N * 2)