def test_stack(self): """Test invoking Stack.""" input1 = np.random.rand(5, 4).astype(np.float32) input2 = np.random.rand(5, 4).astype(np.float32) result = layers.Stack()([input1, input2]) assert result.shape == (5, 2, 4) assert np.array_equal(input1, result[:, 0, :]) assert np.array_equal(input2, result[:, 1, :])
def _create_decoder(self, n_layers, dropout):
"""Create the decoder as a tf.keras.Model."""
input = Input(shape=(self._embedding_dimension, ))
prev_layer = layers.Stack()(self._max_output_length * [input])
for i in range(n_layers):
if dropout > 0.0:
prev_layer = Dropout(dropout)(prev_layer)
prev_layer = GRU(self._embedding_dimension,
return_sequences=True)(prev_layer)
output = Dense(len(self._output_tokens),
activation=tf.nn.softmax)(prev_layer)
return tf.keras.Model(inputs=input, outputs=output)
def __init__(self,
n_tasks,
K=10,
penalty=0.0,
mode="classification",
**kwargs):
"""Initialize MultitaskIRVClassifier
Parameters
----------
n_tasks: int
Number of tasks
K: int
Number of nearest neighbours used in classification
penalty: float
Amount of penalty (l2 or l1 applied)
"""
self.n_tasks = n_tasks
self.K = K
self.n_features = 2 * self.K * self.n_tasks
self.penalty = penalty
mol_features = Input(shape=(self.n_features, ))
predictions = IRVLayer(self.n_tasks, self.K,
self.penalty)(mol_features)
logits = []
outputs = []
for task in range(self.n_tasks):
task_output = Slice(task, 1)(predictions)
sigmoid = Activation(tf.sigmoid)(task_output)
logits.append(task_output)
outputs.append(sigmoid)
outputs = layers.Stack(axis=1)(outputs)
outputs2 = Lambda(lambda x: 1 - x)(outputs)
outputs = [
Concatenate(axis=2)([outputs2, outputs]),
logits[0] if len(logits) == 1 else Concatenate(axis=1)(logits)
]
model = tf.keras.Model(inputs=[mol_features], outputs=outputs)
super(MultitaskIRVClassifier,
self).__init__(model,
SigmoidCrossEntropy(),
output_types=['prediction', 'loss'],
**kwargs)
def __init__(self, n_generators=1, n_discriminators=1, **kwargs):
"""Construct a GAN.
In addition to the parameters listed below, this class accepts all the
keyword arguments from KerasModel.
Parameters
----------
n_generators: int
the number of generators to include
n_discriminators: int
the number of discriminators to include
"""
self.n_generators = n_generators
self.n_discriminators = n_discriminators
# Create the inputs.
self.noise_input = Input(shape=self.get_noise_input_shape())
self.data_input_layers = []
for shape in self.get_data_input_shapes():
self.data_input_layers.append(Input(shape=shape))
self.data_inputs = [i.ref() for i in self.data_input_layers]
self.conditional_input_layers = []
for shape in self.get_conditional_input_shapes():
self.conditional_input_layers.append(Input(shape=shape))
self.conditional_inputs = [
i.ref() for i in self.conditional_input_layers
]
# Create the generators.
self.generators = []
self.gen_variables = []
generator_outputs = []
for i in range(n_generators):
generator = self.create_generator()
self.generators.append(generator)
generator_outputs.append(
generator(
_list_or_tensor([self.noise_input] +
self.conditional_input_layers)))
self.gen_variables += generator.trainable_variables
# Create the discriminators.
self.discriminators = []
self.discrim_variables = []
discrim_train_outputs = []
discrim_gen_outputs = []
for i in range(n_discriminators):
discriminator = self.create_discriminator()
self.discriminators.append(discriminator)
discrim_train_outputs.append(
self._call_discriminator(discriminator, self.data_input_layers,
True))
for gen_output in generator_outputs:
if isinstance(gen_output, tf.Tensor):
gen_output = [gen_output]
discrim_gen_outputs.append(
self._call_discriminator(discriminator, gen_output, False))
self.discrim_variables += discriminator.trainable_variables
# Compute the loss functions.
gen_losses = [
self.create_generator_loss(d) for d in discrim_gen_outputs
]
discrim_losses = []
for i in range(n_discriminators):
for j in range(n_generators):
discrim_losses.append(
self.create_discriminator_loss(
discrim_train_outputs[i],
discrim_gen_outputs[i * n_generators + j]))
if n_generators == 1 and n_discriminators == 1:
total_gen_loss = gen_losses[0]
total_discrim_loss = discrim_losses[0]
else:
# Create learnable weights for the generators and discriminators.
gen_alpha = layers.Variable(np.ones((1, n_generators)),
dtype=tf.float32)
# We pass an input to the Variable layer to work around a bug in TF 1.14.
gen_weights = Softmax()(gen_alpha([self.noise_input]))
discrim_alpha = layers.Variable(np.ones((1, n_discriminators)),
dtype=tf.float32)
discrim_weights = Softmax()(discrim_alpha([self.noise_input]))
# Compute the weighted errors
weight_products = Reshape(
(n_generators * n_discriminators, ))(Multiply()([
Reshape((n_discriminators, 1))(discrim_weights),
Reshape((1, n_generators))(gen_weights)
]))
stacked_gen_loss = layers.Stack(axis=0)(gen_losses)
stacked_discrim_loss = layers.Stack(axis=0)(discrim_losses)
total_gen_loss = Lambda(lambda x: tf.reduce_sum(x[0] * x[1]))(
[stacked_gen_loss, weight_products])
total_discrim_loss = Lambda(lambda x: tf.reduce_sum(x[0] * x[1]))(
[stacked_discrim_loss, weight_products])
self.gen_variables += gen_alpha.trainable_variables
self.discrim_variables += gen_alpha.trainable_variables
self.discrim_variables += discrim_alpha.trainable_variables
# Add an entropy term to the loss.
entropy = Lambda(lambda x: -(tf.reduce_sum(tf.math.log(x[
0])) / n_generators + tf.reduce_sum(tf.math.log(x[
1])) / n_discriminators))([gen_weights, discrim_weights])
total_discrim_loss = Lambda(lambda x: x[0] + x[1])(
[total_discrim_loss, entropy])
# Create the Keras model.
inputs = [self.noise_input
] + self.data_input_layers + self.conditional_input_layers
outputs = [total_gen_loss, total_discrim_loss]
self.gen_loss_fn = lambda outputs, labels, weights: outputs[0]
self.discrim_loss_fn = lambda outputs, labels, weights: outputs[1]
model = tf.keras.Model(inputs=inputs, outputs=outputs)
super(GAN, self).__init__(model, self.gen_loss_fn, **kwargs)
def __init__(self,
n_tasks,
n_features,
alpha_init_stddevs=0.02,
layer_sizes=[1000],
weight_init_stddevs=0.02,
bias_init_consts=1.0,
weight_decay_penalty=0.0,
weight_decay_penalty_type="l2",
dropouts=0.5,
activation_fns=tf.nn.relu,
n_outputs=1,
**kwargs):
"""Creates a progressive network.
Only listing parameters specific to progressive networks here.
Parameters
----------
n_tasks: int
Number of tasks
n_features: int
Number of input features
alpha_init_stddevs: list
List of standard-deviations for alpha in adapter layers.
layer_sizes: list
the size of each dense layer in the network. The length of this list determines the number of layers.
weight_init_stddevs: list or float
the standard deviation of the distribution to use for weight initialization of each layer. The length
of this list should equal len(layer_sizes)+1. The final element corresponds to the output layer.
Alternatively this may be a single value instead of a list, in which case the same value is used for every layer.
bias_init_consts: list or float
the value to initialize the biases in each layer to. The length of this list should equal len(layer_sizes)+1.
The final element corresponds to the output layer. Alternatively this may be a single value instead of a list,
in which case the same value is used for every layer.
weight_decay_penalty: float
the magnitude of the weight decay penalty to use
weight_decay_penalty_type: str
the type of penalty to use for weight decay, either 'l1' or 'l2'
dropouts: list or float
the dropout probablity to use for each layer. The length of this list should equal len(layer_sizes).
Alternatively this may be a single value instead of a list, in which case the same value is used for every layer.
activation_fns: list or object
the Tensorflow activation function to apply to each layer. The length of this list should equal
len(layer_sizes). Alternatively this may be a single value instead of a list, in which case the
same value is used for every layer.
"""
if weight_decay_penalty != 0.0:
raise ValueError('Weight decay is not currently supported')
self.n_tasks = n_tasks
self.n_features = n_features
self.layer_sizes = layer_sizes
self.alpha_init_stddevs = alpha_init_stddevs
self.weight_init_stddevs = weight_init_stddevs
self.bias_init_consts = bias_init_consts
self.dropouts = dropouts
self.activation_fns = activation_fns
self.n_outputs = n_outputs
n_layers = len(layer_sizes)
if not isinstance(weight_init_stddevs, SequenceCollection):
self.weight_init_stddevs = [weight_init_stddevs] * n_layers
if not isinstance(alpha_init_stddevs, SequenceCollection):
self.alpha_init_stddevs = [alpha_init_stddevs] * n_layers
if not isinstance(bias_init_consts, SequenceCollection):
self.bias_init_consts = [bias_init_consts] * n_layers
if not isinstance(dropouts, SequenceCollection):
self.dropouts = [dropouts] * n_layers
if not isinstance(activation_fns, SequenceCollection):
self.activation_fns = [activation_fns] * n_layers
# Add the input features.
mol_features = Input(shape=(n_features, ))
all_layers = {}
outputs = []
self._task_layers = []
for task in range(self.n_tasks):
task_layers = []
for i in range(n_layers):
if i == 0:
prev_layer = mol_features
else:
prev_layer = all_layers[(i - 1, task)]
if task > 0:
lateral_contrib, trainables = self.add_adapter(
all_layers, task, i)
task_layers.extend(trainables)
dense = Dense(
layer_sizes[i],
kernel_initializer=tf.keras.initializers.TruncatedNormal(
stddev=self.weight_init_stddevs[i]),
bias_initializer=tf.constant_initializer(
value=self.bias_init_consts[i]))
layer = dense(prev_layer)
task_layers.append(dense)
if i > 0 and task > 0:
layer = Add()([layer, lateral_contrib])
assert self.activation_fns[
i] is tf.nn.relu, "Only ReLU is supported"
layer = ReLU()(layer)
if self.dropouts[i] > 0.0:
layer = Dropout(self.dropouts[i])(layer)
all_layers[(i, task)] = layer
prev_layer = all_layers[(n_layers - 1, task)]
dense = Dense(
n_outputs,
kernel_initializer=tf.keras.initializers.TruncatedNormal(
stddev=self.weight_init_stddevs[-1]),
bias_initializer=tf.constant_initializer(
value=self.bias_init_consts[-1]))
layer = dense(prev_layer)
task_layers.append(dense)
if task > 0:
lateral_contrib, trainables = self.add_adapter(
all_layers, task, n_layers)
task_layers.extend(trainables)
layer = Add()([layer, lateral_contrib])
output_layer = self.create_output(layer)
outputs.append(output_layer)
self._task_layers.append(task_layers)
outputs = layers.Stack(axis=1)(outputs)
model = tf.keras.Model(inputs=mol_features, outputs=outputs)
super(ProgressiveMultitaskRegressor,
self).__init__(model, self.create_loss(), **kwargs)
