def testNormalFixedFeaturesAreDifferentiable(self):
component_spec = spec_pb2.ComponentSpec()
text_format.Parse("""
name: "test"
network_unit {
registered_name: "IdentityNetwork"
}
fixed_feature {
name: "fixed" embedding_dim: 32 size: 1
pretrained_embedding_matrix { part {} }
vocab { part {} }
}
component_builder {
registered_name: "bulk_component.BulkFeatureExtractorComponentBuilder"
}
""", component_spec)
with tf.Graph().as_default():
comp = bulk_component.BulkFeatureExtractorComponentBuilder(
self.master, component_spec)
# Get embedding matrix variables.
with tf.variable_scope(comp.name, reuse=True):
fixed_embedding_matrix = tf.get_variable(
network_units.fixed_embeddings_name(0))
# Get output layer.
comp.build_greedy_training(self.master_state, self.network_states)
activations = self.network_states[comp.name].activations
outputs = activations[comp.network.layers[0].name].bulk_tensor
# Compute the gradient of the output layer w.r.t. the embedding matrix.
# This should be well-defined for in the normal case.
gradients = tf.gradients(outputs, fixed_embedding_matrix)
self.assertEqual(len(gradients), 1)
self.assertFalse(gradients[0] is None)
def __init__(self, build_runtime_graph=False):
self.spec = spec_pb2.MasterSpec()
self.hyperparams = spec_pb2.GridPoint()
self.lookup_component = {
'previous': MockComponent(self, spec_pb2.ComponentSpec())
}
self.build_runtime_graph = build_runtime_graph
def _make_biaffine_spec():
"""Returns a ComponentSpec that the BiaffineDigraphNetwork works on."""
component_spec = spec_pb2.ComponentSpec()
text_format.Parse(
"""
name: "test_component"
backend { registered_name: "TestComponent" }
linked_feature {
name: "sources"
fml: "input.focus"
source_translator: "identity"
source_component: "previous"
source_layer: "sources"
size: 1
embedding_dim: -1
}
linked_feature {
name: "targets"
fml: "input.focus"
source_translator: "identity"
source_component: "previous"
source_layer: "targets"
size: 1
embedding_dim: -1
}
network_unit {
registered_name: "biaffine_units.BiaffineDigraphNetwork"
}
""", component_spec)
return component_spec
def testBulkFeatureIdExtractorOkWithMultipleFixedFeatures(self):
component_spec = spec_pb2.ComponentSpec()
text_format.Parse("""
name: "test"
network_unit {
registered_name: "IdentityNetwork"
}
fixed_feature {
name: "fixed1" embedding_dim: -1 size: 1
}
fixed_feature {
name: "fixed2" embedding_dim: -1 size: 1
}
fixed_feature {
name: "fixed3" embedding_dim: -1 size: 1
}
""", component_spec)
with tf.Graph().as_default():
comp = bulk_component.BulkFeatureIdExtractorComponentBuilder(
self.master, component_spec)
# Should not raise errors.
self.network_states[component_spec.name] = component.NetworkState()
comp.build_greedy_training(self.master_state, self.network_states)
self.network_states[component_spec.name] = component.NetworkState()
comp.build_greedy_inference(self.master_state, self.network_states)
def testGraphConstructionWithSigmoidLoss(self):
component_spec = spec_pb2.ComponentSpec()
text_format.Parse(
"""
name: "test"
network_unit {
registered_name: "IdentityNetwork"
}
fixed_feature {
name: "fixed" embedding_dim: 32 size: 1
}
component_builder {
registered_name: "component.DynamicComponentBuilder"
parameters {
key: "loss_function"
value: "sigmoid_cross_entropy"
}
}
""", component_spec)
comp = component.DynamicComponentBuilder(self.master, component_spec)
comp.build_greedy_training(self.master_state, self.network_states)
# Check that the loss op is present.
op_names = [op.name for op in tf.get_default_graph().get_operations()]
self.assertTrue('train_test/compute_loss/'
'sigmoid_cross_entropy_with_logits' in op_names)
def testFailsOnNonIdentityTranslator(self):
component_spec = spec_pb2.ComponentSpec()
text_format.Parse("""
name: "test"
network_unit {
registered_name: "IdentityNetwork"
}
linked_feature {
name: "features" embedding_dim: -1 size: 1
source_translator: "history"
source_component: "mock"
}
""", component_spec)
# For feature extraction:
with tf.Graph().as_default():
comp = bulk_component.BulkFeatureExtractorComponentBuilder(
self.master, component_spec)
# Expect feature extraction to generate a error due to the "history"
# translator.
with self.assertRaises(NotImplementedError):
comp.build_greedy_training(self.master_state, self.network_states)
# As well as annotation:
with tf.Graph().as_default():
comp = bulk_component.BulkAnnotatorComponentBuilder(
self.master, component_spec)
with self.assertRaises(NotImplementedError):
comp.build_greedy_training(self.master_state, self.network_states)
def testConstantFixedFeatureFailsIfNotPretrained(self):
component_spec = spec_pb2.ComponentSpec()
text_format.Parse("""
name: "test"
network_unit {
registered_name: "IdentityNetwork"
}
fixed_feature {
name: "fixed" embedding_dim: 32 size: 1
is_constant: true
}
component_builder {
registered_name: "bulk_component.BulkFeatureExtractorComponentBuilder"
}
""", component_spec)
with tf.Graph().as_default():
comp = bulk_component.BulkFeatureExtractorComponentBuilder(
self.master, component_spec)
with self.assertRaisesRegexp(ValueError,
'Constant embeddings must be pretrained'):
comp.build_greedy_training(self.master_state, self.network_states)
with self.assertRaisesRegexp(ValueError,
'Constant embeddings must be pretrained'):
comp.build_greedy_inference(
self.master_state, self.network_states, during_training=True)
with self.assertRaisesRegexp(ValueError,
'Constant embeddings must be pretrained'):
comp.build_greedy_inference(
self.master_state, self.network_states, during_training=False)
def testFailsOnRecurrentLinkedFeature(self):
component_spec = spec_pb2.ComponentSpec()
text_format.Parse("""
name: "test"
network_unit {
registered_name: "FeedForwardNetwork"
parameters {
key: 'hidden_layer_sizes' value: '64'
}
}
linked_feature {
name: "features" embedding_dim: -1 size: 1
source_translator: "identity"
source_component: "test"
source_layer: "layer_0"
}
""", component_spec)
# For feature extraction:
with tf.Graph().as_default():
comp = bulk_component.BulkFeatureExtractorComponentBuilder(
self.master, component_spec)
# Expect feature extraction to generate a error due to the "history"
# translator.
with self.assertRaises(RuntimeError):
comp.build_greedy_training(self.master_state, self.network_states)
# As well as annotation:
with tf.Graph().as_default():
comp = bulk_component.BulkAnnotatorComponentBuilder(
self.master, component_spec)
with self.assertRaises(RuntimeError):
comp.build_greedy_training(self.master_state, self.network_states)
def __init__(self, name, builder='DynamicComponentBuilder'):
"""Initializes the ComponentSpec with some defaults for SyntaxNet.
Args:
name: The name of this Component in the pipeline.
builder: The component builder type.
"""
self.spec = spec_pb2.ComponentSpec(
name=name,
backend=self.make_module('SyntaxNetComponent'),
component_builder=self.make_module(builder))
def testBulkFeatureIdExtractorFailsOnEmbeddedFixedFeature(self):
component_spec = spec_pb2.ComponentSpec()
text_format.Parse("""
name: "test"
network_unit {
registered_name: "IdentityNetwork"
}
fixed_feature {
name: "fixed" embedding_dim: 2 size: 1
}
""", component_spec)
with self.assertRaises(ValueError):
unused_comp = bulk_component.BulkFeatureIdExtractorComponentBuilder(
self.master, component_spec)
def testConstantFixedFeaturesAreNotDifferentiableButOthersAre(self):
component_spec = spec_pb2.ComponentSpec()
text_format.Parse(
"""
name: "test"
network_unit {
registered_name: "IdentityNetwork"
}
fixed_feature {
name: "constant" embedding_dim: 32 size: 1
is_constant: true
pretrained_embedding_matrix { part {} }
vocab { part {} }
}
fixed_feature {
name: "trainable" embedding_dim: 32 size: 1
pretrained_embedding_matrix { part {} }
vocab { part {} }
}
component_builder {
registered_name: "bulk_component.BulkFeatureExtractorComponentBuilder"
}
""", component_spec)
with tf.Graph().as_default():
comp = bulk_component.BulkFeatureExtractorComponentBuilder(
self.master, component_spec)
# Get embedding matrix variables.
with tf.variable_scope(comp.name, reuse=True):
constant_embedding_matrix = tf.get_variable(
network_units.fixed_embeddings_name(0))
trainable_embedding_matrix = tf.get_variable(
network_units.fixed_embeddings_name(1))
# Get output layer.
comp.build_greedy_training(self.master_state, self.network_states)
activations = self.network_states[comp.name].activations
outputs = activations[comp.network.layers[0].name].bulk_tensor
# The constant embeddings are non-differentiable.
constant_gradients = tf.gradients(outputs,
constant_embedding_matrix)
self.assertEqual(len(constant_gradients), 1)
self.assertTrue(constant_gradients[0] is None)
# The trainable embeddings are differentiable.
trainable_gradients = tf.gradients(outputs,
trainable_embedding_matrix)
self.assertEqual(len(trainable_gradients), 1)
self.assertFalse(trainable_gradients[0] is None)
def testGraphConstruction(self):
component_spec = spec_pb2.ComponentSpec()
text_format.Parse(
"""
name: "test"
network_unit {
registered_name: "IdentityNetwork"
}
fixed_feature {
name: "fixed" embedding_dim: 32 size: 1
}
component_builder {
registered_name: "component.DynamicComponentBuilder"
}
""", component_spec)
comp = component.DynamicComponentBuilder(self.master, component_spec)
comp.build_greedy_training(self.master_state, self.network_states)
def testFailsOnFixedFeature(self):
component_spec = spec_pb2.ComponentSpec()
text_format.Parse("""
name: "annotate"
network_unit {
registered_name: "IdentityNetwork"
}
fixed_feature {
name: "fixed" embedding_dim: 32 size: 1
}
""", component_spec)
with tf.Graph().as_default():
comp = bulk_component.BulkAnnotatorComponentBuilder(
self.master, component_spec)
# Expect feature extraction to generate a runtime error due to the
# fixed feature.
with self.assertRaises(RuntimeError):
comp.build_greedy_training(self.master_state, self.network_states)
def testBulkFeatureIdExtractorFailsOnLinkedFeature(self):
component_spec = spec_pb2.ComponentSpec()
text_format.Parse("""
name: "test"
network_unit {
registered_name: "IdentityNetwork"
}
fixed_feature {
name: "fixed" embedding_dim: -1 size: 1
}
linked_feature {
name: "linked" embedding_dim: -1 size: 1
source_translator: "identity"
source_component: "mock"
}
""", component_spec)
with tf.Graph().as_default():
with self.assertRaises(ValueError):
unused_comp = bulk_component.BulkFeatureIdExtractorComponentBuilder(
self.master, component_spec)
def _get_master_spec():
return spec_pb2.MasterSpec(
component=[spec_pb2.ComponentSpec(name='jalapeño')])
def testPreCreateCalledBeforeCreate(self):
component_spec = spec_pb2.ComponentSpec()
text_format.Parse(
"""
name: "test"
network_unit {
registered_name: "IdentityNetwork"
}
""", component_spec)
class AssertPreCreateBeforeCreateNetwork(
network_units.NetworkUnitInterface):
"""Mock that asserts that .create() is called before .pre_create()."""
def __init__(self, comp, test_fixture):
super(AssertPreCreateBeforeCreateNetwork, self).__init__(comp)
self._test_fixture = test_fixture
self._pre_create_called = False
def get_logits(self, network_tensors):
return tf.zeros([2, 1], dtype=tf.float32)
def pre_create(self, *unused_args):
self._pre_create_called = True
def create(self, *unused_args, **unuesd_kwargs):
self._test_fixture.assertTrue(self._pre_create_called)
return []
builder = bulk_component.BulkFeatureExtractorComponentBuilder(
self.master, component_spec)
builder.network = AssertPreCreateBeforeCreateNetwork(builder, self)
builder.build_greedy_training(component.MasterState(['foo', 'bar'], 2),
self.network_states)
self.setUp()
builder = bulk_component.BulkFeatureExtractorComponentBuilder(
self.master, component_spec)
builder.network = AssertPreCreateBeforeCreateNetwork(builder, self)
builder.build_greedy_inference(
component.MasterState(['foo', 'bar'], 2), self.network_states)
self.setUp()
builder = bulk_component.BulkFeatureIdExtractorComponentBuilder(
self.master, component_spec)
builder.network = AssertPreCreateBeforeCreateNetwork(builder, self)
builder.build_greedy_training(component.MasterState(['foo', 'bar'], 2),
self.network_states)
self.setUp()
builder = bulk_component.BulkFeatureIdExtractorComponentBuilder(
self.master, component_spec)
builder.network = AssertPreCreateBeforeCreateNetwork(builder, self)
builder.build_greedy_inference(
component.MasterState(['foo', 'bar'], 2), self.network_states)
self.setUp()
builder = bulk_component.BulkAnnotatorComponentBuilder(
self.master, component_spec)
builder.network = AssertPreCreateBeforeCreateNetwork(builder, self)
builder.build_greedy_training(component.MasterState(['foo', 'bar'], 2),
self.network_states)
self.setUp()
builder = bulk_component.BulkAnnotatorComponentBuilder(
self.master, component_spec)
builder.network = AssertPreCreateBeforeCreateNetwork(builder, self)
builder.build_greedy_inference(
component.MasterState(['foo', 'bar'], 2), self.network_states)
def __init__(self):
self.spec = spec_pb2.MasterSpec()
self.hyperparams = spec_pb2.GridPoint()
self.lookup_component = {
'previous': MockComponent(self, spec_pb2.ComponentSpec())
}
def assertSpecEqual(self, expected_spec_text, spec):
expected_spec = spec_pb2.ComponentSpec()
text_format.Parse(expected_spec_text, expected_spec)
self.assertProtoEquals(expected_spec, spec)
def __init__(self):
self.name = 'test_component'
self.spec = spec_pb2.ComponentSpec()
with tf.variable_scope(self.name):
self.network = MockNetwork()
