def _check_data_type_forward(self, in_data):
in_type = type_check.get_types(in_data, 'in_types', False)
try:
self.check_type_forward(in_type)
except type_check.InvalidType as e:
msg = """
Invalid operation is performed in: {0} (Forward)
{1}""".format(self.label, str(e))
raise type_check.InvalidType(e.expect, e.actual, msg=msg)
def _check_data_type_forward(self, in_data):
in_type = type_check.get_types(in_data, 'in_types', False)
try:
self.check_type_forward(in_type)
except type_check.InvalidType as e:
msg = """
Invalid operation is performed in: {0} (Forward)
{1}""".format(self.label, str(e))
raise type_check.InvalidType(e.expect, e.actual, msg=msg)
def _check_data_type_forward(self, in_data):
in_type = type_check.get_light_types(in_data)
try:
with type_check.light_mode:
self.check_type_forward(in_type)
return
except type_check.InvalidType:
# Ignore errors on first run
pass
in_type = type_check.get_types(in_data, 'in_types', False)
with type_check.get_function_check_context(self):
self.check_type_forward(in_type)
def _check_data_type_forward(self, in_data):
in_type = type_check.get_light_types(in_data)
try:
with type_check.light_mode:
self.check_type_forward(in_type)
return
except type_check.InvalidType:
# Ignore errors on first run
pass
in_type = type_check.get_types(in_data, 'in_types', False)
with type_check.get_function_check_context(self):
self.check_type_forward(in_type)
def test_simple(self):
data = (numpy.zeros((1, 2, 3)).astype(numpy.float32),)
ts = T.get_types(data, 'name', False)
self.assertIsInstance(ts, T.TypeInfoTuple)
self.assertEqual(1, len(ts))
self.assertEqual('name', ts.name)
t = ts[0]
self.assertIsInstance(t, T.Expr)
self.assertEqual(1, t.shape[0].eval())
self.assertEqual(2, t.shape[1].eval())
self.assertEqual(3, t.shape[2].eval())
self.assertEqual(3, t.ndim.eval())
self.assertEqual(numpy.float32, t.dtype.eval())
def __call__(self, y, y_batch, train=True):
# BEGIN: Type Check
in_data = tuple([x.data for x in [y, y_batch]])
in_types = type_check.get_types(in_data, 'in_types', False)
self.check_type_forward(in_types)
# END: Type Check
accum_loss = 0 if train else None
if train:
if self.__gpu >= 0:
y_batch = cuda.to_gpu(y_batch.data)
accum_loss = F.softmax_cross_entropy(y, y_batch)
return accum_loss
def __call__(self, x_batch, train=True):
"""
Compute an integer lookup on an embedding matrix.
Args:
x_batch: List of B x S
Returns:
x_emb: List of flatten embeddings B x S x E
"""
# BEGIN: Type Check
in_data = tuple([x.data for x in [x_batch]])
in_types = type_check.get_types(in_data, 'in_types', False)
self.check_type_forward(in_types)
# END: Type Check
emb = self.raw_embeddings.take(x_batch.data, axis=0)
return emb
def _check_data_type_forward(self, in_data):
in_type = type_check.get_types(in_data, 'in_types', False)
self.check_type_forward(in_type)
def _check_data_type_backward(self, in_data, grad_data):
in_type = type_check.get_types(in_data, 'in_types', False)
grad_type = type_check.get_types(grad_data, 'grad_types', True)
self.check_type_backward(in_type, grad_type)
def _check_data_type_forward(self, in_data):
in_type = type_check.get_types(in_data, 'in_types', False)
with type_check.get_function_check_context(self):
self.check_type_forward(in_type)
def test_invalid_arg(self):
with self.assertRaises(AssertionError):
T.get_types(1, 'name', False)
def test_empty(self):
ts = T.get_types((), 'name', False)
self.assertIsInstance(ts, T.TypeInfoTuple)
self.assertEqual(0, len(ts))
self.assertEqual('name', ts.name)
def _check_data_type_forward(self, in_data):
in_type = type_check.get_types(in_data, "in_types", False)
self.check_type_forward(in_type)
def _check_data_type_backward(self, in_data, grad_data):
in_type = type_check.get_types(in_data, 'in_types', False)
grad_type = type_check.get_types(grad_data, 'grad_types', True)
self.check_type_backward(in_type, grad_type)
def _check_data_type_forward(self, in_data):
in_type = type_check.get_types(in_data, 'in_types', False)
with type_check.get_function_check_context(self):
self.check_type_forward(in_type)
def __call__(self, buffers, transitions, train=True, keep_hs=False):
"""
Pass over batches of transitions, modifying their associated
buffers at each iteration.
Args:
buffers: List of B x S* x E
transitions: List of B x S
Returns:
final_state: List of B x E
"""
# BEGIN: Type Check
in_data = tuple([x.data for x in [buffers]])
in_types = type_check.get_types(in_data, 'in_types', False)
self.check_type_forward(in_types)
transition_num = 0
transition_acc = 0.0
# END: Type Check
batch_size, seq_length, hidden_dim = buffers.shape[0], buffers.shape[
1], buffers.shape[2]
transitions = transitions.T
assert len(transitions) == seq_length
buffers = [
F.split_axis(b, seq_length, axis=0, force_tuple=True)
for b in buffers
]
buffers_t = [0 for _ in buffers]
# Initialize stack with at least one item, otherwise gradient might
# not propogate.
stacks = [[] for b in buffers]
def pseudo_reduce(lefts, rights):
for l, r in zip(lefts, rights):
yield l + r
def better_reduce(lefts, rights, h):
lstm_state = self.reduce(lefts, rights, h, train=train)
batch_size = lstm_state.shape[0]
lstm_state = F.split_axis(lstm_state,
batch_size,
axis=0,
force_tuple=True)
for state in lstm_state:
yield state
self.reset_transitions()
for ii, ts in enumerate(transitions):
assert len(ts) == batch_size
assert len(ts) == len(buffers)
assert len(ts) == len(stacks)
# TODO! The tracking inputs for shifts and reduces should be split,
# in order to do consecutive shifts. This would (maybe) allow us
# to get the performance benefits from dynamic batch sizes while still
# predicting actions.
if self.use_tracking_lstm:
if self.use_shift_composition:
tracking_input = self.tracking_input(
stacks, buffers, buffers_t, train)
c = F.concat(self.c, axis=0)
h = F.concat(self.h, axis=0)
c, h, logits = self.tracking_lstm(c, h, tracking_input,
train)
if self.make_logits and np.any(ts != -1):
_rpt = np.repeat(np.expand_dims(ts == -1, 1),
2,
axis=1)
try:
selectedTransitions = F.where(
Variable(_rpt), logits,
Variable(np.zeros_like(logits.data)))
except:
import ipdb
ipdb.set_trace()
transition_loss = self.transition_classifier(
selectedTransitions, Variable(ts))
transition_acc += np.sum(
np.argmax(logits.data[ts != -1], axis=1) == ts[
ts != -1])
transition_num += np.sum(ts != -1)
# print("Accuracy: ", np.sum(ts != -1), np.mean(np.argmax(logits.data[ts != -1], axis=1) == ts[ts != -1]))
transition_loss.backward()
# Assign appropriate states after they've been calculated.
self.c = F.split_axis(c,
c.shape[0],
axis=0,
force_tuple=True)
self.h = F.split_axis(h,
h.shape[0],
axis=0,
force_tuple=True)
else:
tracking_size = len([t for t in ts if t == 1])
if tracking_size > 0:
tracking_ix = [i for (i, t) in enumerate(ts) if t == 1]
tracking_stacks = [
x for (t, x) in zip(ts, stacks) if t == 1
]
tracking_buffers = [
x for (t, x) in zip(ts, buffers) if t == 1
]
tracking_buffers_t = [
x for (t, x) in zip(ts, buffers_t) if t == 1
]
tracking_input = self.tracking_input(
tracking_stacks, tracking_buffers,
tracking_buffers_t, train)
c = F.concat(
[x for (t, x) in zip(ts, self.c) if t == 1],
axis=0)
h = F.concat(
[x for (t, x) in zip(ts, self.h) if t == 1],
axis=0)
c, h, logits = self.tracking_lstm(
c, h, tracking_input, train)
# Assign appropriate states after they've been calculated.
_c = F.split_axis(c,
tracking_size,
axis=0,
force_tuple=True)
for i, ix in enumerate(tracking_ix):
if t == 1:
self.c[ix] = _c[i]
_h = F.split_axis(h,
tracking_size,
axis=0,
force_tuple=True)
for i, ix in enumerate(tracking_ix):
if t == 1:
self.h[ix] = _h[i]
lefts = []
rights = []
for i, (t, buf, stack) in enumerate(zip(ts, buffers, stacks)):
if t == -1: # skip
pass
elif t == 0: # shift
new_stack_item = buf[buffers_t[i]]
stack.append(new_stack_item)
assert buffers_t[i] < seq_length
buffers_t[i] += 1
elif t == 1: # reduce
for lr in [rights, lefts]:
if len(stack) > 0:
lr.append(stack.pop())
else:
lr.append(
Variable(self.__mod.zeros(
(
1,
hidden_dim,
),
dtype=self.__mod.float32),
volatile=not train))
else:
raise Exception("Action not implemented: {}".format(t))
assert len(lefts) == len(rights)
if len(rights) > 0:
if self.use_tracking_lstm:
external = F.concat(
[x for (t, x) in zip(ts, self.h) if t == 1], axis=0)
else:
external = None
reduced = iter(better_reduce(lefts, rights, external))
for i, (t, buf, stack) in enumerate(zip(ts, buffers, stacks)):
if t == -1 or t == 0:
continue
elif t == 1:
composition = next(reduced)
stack.append(composition)
else:
raise Exception("Action not implemented: {}".format(t))
ret = F.concat([s.pop() for s in stacks], axis=0)
self.update_transitions()
assert ret.shape == (batch_size, hidden_dim)
if self.make_logits:
avg_transition_acc = transition_acc / transition_num
else:
avg_transition_acc = 0.0
# print("Avg tr acc:", transition_acc / transition_num)
return ret, avg_transition_acc
