def to_flow(self, fl):
spec = self.spec
# Specify the encoder.
lstm_embeddings = [e.weight.data.numpy() for e in self.lstm_embeddings]
lex = LexicalEncoder(fl, spec, lstm_embeddings, self.lr_lstm, self.rl_lstm)
# Write the specification to the flow.
spec.to_flow(fl)
# Adds a flow variable that will store raw indices for the given feature.
def index_vars(bldr, feature_spec):
return bldr.var(name=feature.name, dtype="int32", shape=[1, feature.num])
# Adds flow variables and ops for the given fixed feature.
# The output variable of the feature is added as an input to 'concat_op'.
def write_fixed_feature(feature, bag, bldr, concat_op):
indices = index_vars(bldr, feature)
s = bag.weight.size()
embedding = bldr.var(name=feature.name + "_embedding", shape=[s[0], s[1]])
embedding.data = bag.weight.data.numpy()
name = feature.name + '/GatherSum'
embedded = bldr.gather_sum(embedding, indices, name=name)
concat_op.add_input(embedded)
# Finishes the concatenation op assuming all inputs have been added.
def finish_concat_op(bldr, op):
op.add_attr("N", len(op.inputs))
axis = bldr.const(1, "int32")
op.add_input(axis)
# Specify the FF trunk = FF feature vector + hidden layer computation.
ff = builder.Builder(fl, "ff_trunk")
ff_input = ff.var(name="input", shape=[1, spec.ff_input_dim])
flow_ff = flownn.FF(
ff, \
input=ff_input, \
layers=[spec.ff_hidden_dim],
hidden=0)
flow_ff.set_layer_data(0, self.ff_layer.weight.data.numpy(), \
self.ff_layer.bias.data.numpy())
ff_concat_op = ff.rawop(optype="ConcatV2", name="concat")
ff_concat_op.add_output(ff_input)
# Add link variable to the given connector.
def link(bldr, name, dim, cnx, prefix=True):
if prefix: name = "link/" + name
l = bldr.var(name, shape=[-1, dim])
l.ref = True
cnx.add(l)
return l
# Add links to the two LSTMs.
ff_lr = link(ff, "lr_lstm", spec.lstm_hidden_dim, lex.lr_lstm.cnx_hidden)
ff_rl = link(ff, "rl_lstm", spec.lstm_hidden_dim, lex.rl_lstm.cnx_hidden)
# Add link and connector for previous FF steps.
ff_cnx = ff.cnx("step", args=[])
ff_steps = link(ff, "steps", spec.ff_hidden_dim, ff_cnx, False)
ff_cnx.add(flow_ff.hidden_out)
# Add FF's input variables.
for feature in spec.ff_fixed_features:
write_fixed_feature(feature, feature.bag, ff, ff_concat_op)
for feature in spec.ff_link_features:
indices = index_vars(ff, feature)
activations = None
n = feature.name
if n == "frame-end-lr" or n == "lr" or n == "mark-lr":
activations = ff_lr
elif n == "frame-end-rl" or n == "rl" or n == "mark-rl":
activations = ff_rl
elif n in ["frame-creation-steps", "frame-focus-steps", \
"history", "mark-step"]:
activations = ff_steps
else:
raise ValueError("Unknown feature %r" % n)
activation_dim = activations.shape[1]
name = feature.name + "/Gather"
oov_var = ff.var(name=name + "/oov", shape=[1, activation_dim])
oov_var.data = feature.transform.oov.data.numpy()
gathered = ff.gather(activations, indices, oov_var, name=name)
projection = feature.transform.projection.weight
sz = projection.size()
transform = ff.var(name=feature.name + "/transform", shape=[sz[0], sz[1]])
transform.data = projection.data.numpy()
name = feature.name + "/MatMul"
output = ff.matmul(gathered, transform, name=name)
ff_concat_op.add_input(output)
finish_concat_op(ff, ff_concat_op)
# Specify one cell per FF head (= delegate).
ff_trunk_width = flow_ff.hidden_out.shape[1]
delegate_cell_prefix = "delegate"
for i, head in enumerate(self.ff_heads):
delegate = spec.cascade.delegates[i]
assert_softmax_delegate(delegate)
d = builder.Builder(fl, delegate_cell_prefix + str(i))
head_input = link(d, "input", ff_trunk_width, ff_cnx, False)
W = d.var("W", shape=[ff_trunk_width, delegate.size()])
W.data = head.softmax.weight.data.numpy()
output = d.matmul(head_input, W)
output.type = W.type
output.shape = [1, delegate.size()]
b = d.var("b", shape=[1, delegate.size()])
b.data = head.softmax.bias.data.numpy()
logits = d.add(output, b)
logits.type = b.type
logits.shape = [1, delegate.size()]
best_op = d.rawop(optype="ArgMax")
best_op.add_input(logits)
best = d.var("output")
best.type = builder.DT_INT
best.shape = [1]
best_op.add_output(best)
best.producer.add_attr("output", 1)
def __init__(self, flow, spec, lstm_feature_embeddings, lr_lstm, rl_lstm):
# Add blobs for the lexical resources.
lexicon = flow.blob("lexicon")
lexicon.type = "dict"
lexicon.add_attr("delimiter", 10)
lexicon.add_attr("oov", spec.words.oov_index)
normalization = ""
if spec.words.normalize_digits: normalization = "d"
lexicon.add_attr("normalization", normalization)
lexicon.data = str(spec.words) + "\n"
self.lexicon_blob = lexicon
def read_file(filename):
fin = open(filename, "rb")
data = fin.read()
fin.close()
return data
f = tempfile.NamedTemporaryFile(delete=False)
fname = f.name
spec.commons.save(fname, binary=True)
f.close()
commons = flow.blob("commons")
commons.type = "frames"
commons.data = read_file(fname)
os.unlink(fname)
self.commons_blob = commons
suffix = flow.blob("suffixes")
suffix.type = "affix"
suffix.data = str(spec.write_suffix_table())
self.suffix_blob = suffix
# Add feature extraction related ops.
bldr = builder.Builder(flow, "features")
self.feature_ids = []
concat_args = []
for f, e in zip(spec.lstm_features, lstm_feature_embeddings):
shape = [f.vocab_size, f.dim]
embedding = bldr.var(name=f.name + "_embeddings", shape=shape)
embedding.data = e
ids_input = bldr.var(name=f.name, dtype="int32", shape=[1, f.num])
self.feature_ids.append(ids_input)
gather_op_type = "Gather"
if f.num > 1: gather_op_type = "GatherSum"
gather_op = bldr.rawop(gather_op_type)
gather_op.dtype = "float32"
gather_op.add_input(embedding)
gather_op.add_input(ids_input)
gather_output = bldr.var(gather_op.name + ":0", "float32",
[1, f.dim])
gather_op.add_output(gather_output)
concat_args.append(gather_output)
self.feature_vector = bldr.concat(concat_args)
bldr.rename(self.feature_vector, "feature_vector")
self.feature_vector.ref = True
self.feature_vector.input = True
self.feature_vector.output = True
# Add BiLSTM.
lr = builder.Builder(flow, "lstm/lr")
lr_input = lr.var(name="input", shape=[1, spec.lstm_input_dim])
lr_input.ref = True
flow_lr_lstm = nn.LSTM(lr, input=lr_input, size=spec.lstm_hidden_dim)
lr_lstm.copy_to_flow_lstm(flow_lr_lstm)
self.lr_lstm = flow_lr_lstm
rl = builder.Builder(flow, "lstm/rl")
rl_input = rl.var(name="input", shape=[1, spec.lstm_input_dim])
rl_input.ref = True
flow_rl_lstm = nn.LSTM(rl, input=rl_input, size=spec.lstm_hidden_dim)
rl_lstm.copy_to_flow_lstm(flow_rl_lstm)
self.rl_lstm = flow_rl_lstm
cnxin = flow.cnx("features")
cnxin.add(self.feature_vector)
cnxin.add(lr_input)
cnxin.add(rl_input)