def test_stack(self): """Test invoking Stack.""" input1 = np.random.rand(5, 4).astype(np.float32) input2 = np.random.rand(5, 4).astype(np.float32) result = layers.Stack()([input1, input2]) assert result.shape == (5, 2, 4) assert np.array_equal(input1, result[:, 0, :]) assert np.array_equal(input2, result[:, 1, :])
def _create_decoder(self, n_layers, dropout): """Create the decoder as a tf.keras.Model.""" input = Input(shape=(self._embedding_dimension, )) prev_layer = layers.Stack()(self._max_output_length * [input]) for i in range(n_layers): if dropout > 0.0: prev_layer = Dropout(dropout)(prev_layer) prev_layer = GRU(self._embedding_dimension, return_sequences=True)(prev_layer) output = Dense(len(self._output_tokens), activation=tf.nn.softmax)(prev_layer) return tf.keras.Model(inputs=input, outputs=output)
def __init__(self, n_tasks, K=10, penalty=0.0, mode="classification", **kwargs): """Initialize MultitaskIRVClassifier Parameters ---------- n_tasks: int Number of tasks K: int Number of nearest neighbours used in classification penalty: float Amount of penalty (l2 or l1 applied) """ self.n_tasks = n_tasks self.K = K self.n_features = 2 * self.K * self.n_tasks self.penalty = penalty mol_features = Input(shape=(self.n_features, )) predictions = IRVLayer(self.n_tasks, self.K, self.penalty)(mol_features) logits = [] outputs = [] for task in range(self.n_tasks): task_output = Slice(task, 1)(predictions) sigmoid = Activation(tf.sigmoid)(task_output) logits.append(task_output) outputs.append(sigmoid) outputs = layers.Stack(axis=1)(outputs) outputs2 = Lambda(lambda x: 1 - x)(outputs) outputs = [ Concatenate(axis=2)([outputs2, outputs]), logits[0] if len(logits) == 1 else Concatenate(axis=1)(logits) ] model = tf.keras.Model(inputs=[mol_features], outputs=outputs) super(MultitaskIRVClassifier, self).__init__(model, SigmoidCrossEntropy(), output_types=['prediction', 'loss'], **kwargs)
def __init__(self, n_generators=1, n_discriminators=1, **kwargs): """Construct a GAN. In addition to the parameters listed below, this class accepts all the keyword arguments from KerasModel. Parameters ---------- n_generators: int the number of generators to include n_discriminators: int the number of discriminators to include """ self.n_generators = n_generators self.n_discriminators = n_discriminators # Create the inputs. self.noise_input = Input(shape=self.get_noise_input_shape()) self.data_input_layers = [] for shape in self.get_data_input_shapes(): self.data_input_layers.append(Input(shape=shape)) self.data_inputs = [i.ref() for i in self.data_input_layers] self.conditional_input_layers = [] for shape in self.get_conditional_input_shapes(): self.conditional_input_layers.append(Input(shape=shape)) self.conditional_inputs = [ i.ref() for i in self.conditional_input_layers ] # Create the generators. self.generators = [] self.gen_variables = [] generator_outputs = [] for i in range(n_generators): generator = self.create_generator() self.generators.append(generator) generator_outputs.append( generator( _list_or_tensor([self.noise_input] + self.conditional_input_layers))) self.gen_variables += generator.trainable_variables # Create the discriminators. self.discriminators = [] self.discrim_variables = [] discrim_train_outputs = [] discrim_gen_outputs = [] for i in range(n_discriminators): discriminator = self.create_discriminator() self.discriminators.append(discriminator) discrim_train_outputs.append( self._call_discriminator(discriminator, self.data_input_layers, True)) for gen_output in generator_outputs: if isinstance(gen_output, tf.Tensor): gen_output = [gen_output] discrim_gen_outputs.append( self._call_discriminator(discriminator, gen_output, False)) self.discrim_variables += discriminator.trainable_variables # Compute the loss functions. gen_losses = [ self.create_generator_loss(d) for d in discrim_gen_outputs ] discrim_losses = [] for i in range(n_discriminators): for j in range(n_generators): discrim_losses.append( self.create_discriminator_loss( discrim_train_outputs[i], discrim_gen_outputs[i * n_generators + j])) if n_generators == 1 and n_discriminators == 1: total_gen_loss = gen_losses[0] total_discrim_loss = discrim_losses[0] else: # Create learnable weights for the generators and discriminators. gen_alpha = layers.Variable(np.ones((1, n_generators)), dtype=tf.float32) # We pass an input to the Variable layer to work around a bug in TF 1.14. gen_weights = Softmax()(gen_alpha([self.noise_input])) discrim_alpha = layers.Variable(np.ones((1, n_discriminators)), dtype=tf.float32) discrim_weights = Softmax()(discrim_alpha([self.noise_input])) # Compute the weighted errors weight_products = Reshape( (n_generators * n_discriminators, ))(Multiply()([ Reshape((n_discriminators, 1))(discrim_weights), Reshape((1, n_generators))(gen_weights) ])) stacked_gen_loss = layers.Stack(axis=0)(gen_losses) stacked_discrim_loss = layers.Stack(axis=0)(discrim_losses) total_gen_loss = Lambda(lambda x: tf.reduce_sum(x[0] * x[1]))( [stacked_gen_loss, weight_products]) total_discrim_loss = Lambda(lambda x: tf.reduce_sum(x[0] * x[1]))( [stacked_discrim_loss, weight_products]) self.gen_variables += gen_alpha.trainable_variables self.discrim_variables += gen_alpha.trainable_variables self.discrim_variables += discrim_alpha.trainable_variables # Add an entropy term to the loss. entropy = Lambda(lambda x: -(tf.reduce_sum(tf.math.log(x[ 0])) / n_generators + tf.reduce_sum(tf.math.log(x[ 1])) / n_discriminators))([gen_weights, discrim_weights]) total_discrim_loss = Lambda(lambda x: x[0] + x[1])( [total_discrim_loss, entropy]) # Create the Keras model. inputs = [self.noise_input ] + self.data_input_layers + self.conditional_input_layers outputs = [total_gen_loss, total_discrim_loss] self.gen_loss_fn = lambda outputs, labels, weights: outputs[0] self.discrim_loss_fn = lambda outputs, labels, weights: outputs[1] model = tf.keras.Model(inputs=inputs, outputs=outputs) super(GAN, self).__init__(model, self.gen_loss_fn, **kwargs)
def __init__(self, n_tasks, n_features, alpha_init_stddevs=0.02, layer_sizes=[1000], weight_init_stddevs=0.02, bias_init_consts=1.0, weight_decay_penalty=0.0, weight_decay_penalty_type="l2", dropouts=0.5, activation_fns=tf.nn.relu, n_outputs=1, **kwargs): """Creates a progressive network. Only listing parameters specific to progressive networks here. Parameters ---------- n_tasks: int Number of tasks n_features: int Number of input features alpha_init_stddevs: list List of standard-deviations for alpha in adapter layers. layer_sizes: list the size of each dense layer in the network. The length of this list determines the number of layers. weight_init_stddevs: list or float the standard deviation of the distribution to use for weight initialization of each layer. The length of this list should equal len(layer_sizes)+1. The final element corresponds to the output layer. Alternatively this may be a single value instead of a list, in which case the same value is used for every layer. bias_init_consts: list or float the value to initialize the biases in each layer to. The length of this list should equal len(layer_sizes)+1. The final element corresponds to the output layer. Alternatively this may be a single value instead of a list, in which case the same value is used for every layer. weight_decay_penalty: float the magnitude of the weight decay penalty to use weight_decay_penalty_type: str the type of penalty to use for weight decay, either 'l1' or 'l2' dropouts: list or float the dropout probablity to use for each layer. The length of this list should equal len(layer_sizes). Alternatively this may be a single value instead of a list, in which case the same value is used for every layer. activation_fns: list or object the Tensorflow activation function to apply to each layer. The length of this list should equal len(layer_sizes). Alternatively this may be a single value instead of a list, in which case the same value is used for every layer. """ if weight_decay_penalty != 0.0: raise ValueError('Weight decay is not currently supported') self.n_tasks = n_tasks self.n_features = n_features self.layer_sizes = layer_sizes self.alpha_init_stddevs = alpha_init_stddevs self.weight_init_stddevs = weight_init_stddevs self.bias_init_consts = bias_init_consts self.dropouts = dropouts self.activation_fns = activation_fns self.n_outputs = n_outputs n_layers = len(layer_sizes) if not isinstance(weight_init_stddevs, SequenceCollection): self.weight_init_stddevs = [weight_init_stddevs] * n_layers if not isinstance(alpha_init_stddevs, SequenceCollection): self.alpha_init_stddevs = [alpha_init_stddevs] * n_layers if not isinstance(bias_init_consts, SequenceCollection): self.bias_init_consts = [bias_init_consts] * n_layers if not isinstance(dropouts, SequenceCollection): self.dropouts = [dropouts] * n_layers if not isinstance(activation_fns, SequenceCollection): self.activation_fns = [activation_fns] * n_layers # Add the input features. mol_features = Input(shape=(n_features, )) all_layers = {} outputs = [] self._task_layers = [] for task in range(self.n_tasks): task_layers = [] for i in range(n_layers): if i == 0: prev_layer = mol_features else: prev_layer = all_layers[(i - 1, task)] if task > 0: lateral_contrib, trainables = self.add_adapter( all_layers, task, i) task_layers.extend(trainables) dense = Dense( layer_sizes[i], kernel_initializer=tf.keras.initializers.TruncatedNormal( stddev=self.weight_init_stddevs[i]), bias_initializer=tf.constant_initializer( value=self.bias_init_consts[i])) layer = dense(prev_layer) task_layers.append(dense) if i > 0 and task > 0: layer = Add()([layer, lateral_contrib]) assert self.activation_fns[ i] is tf.nn.relu, "Only ReLU is supported" layer = ReLU()(layer) if self.dropouts[i] > 0.0: layer = Dropout(self.dropouts[i])(layer) all_layers[(i, task)] = layer prev_layer = all_layers[(n_layers - 1, task)] dense = Dense( n_outputs, kernel_initializer=tf.keras.initializers.TruncatedNormal( stddev=self.weight_init_stddevs[-1]), bias_initializer=tf.constant_initializer( value=self.bias_init_consts[-1])) layer = dense(prev_layer) task_layers.append(dense) if task > 0: lateral_contrib, trainables = self.add_adapter( all_layers, task, n_layers) task_layers.extend(trainables) layer = Add()([layer, lateral_contrib]) output_layer = self.create_output(layer) outputs.append(output_layer) self._task_layers.append(task_layers) outputs = layers.Stack(axis=1)(outputs) model = tf.keras.Model(inputs=mol_features, outputs=outputs) super(ProgressiveMultitaskRegressor, self).__init__(model, self.create_loss(), **kwargs)