def identity_block(self, input, kernel_size, filters):
filters1, filters2, filters3 = filters
output = Conv2D(num_outputs=filters1,
kernel_size=1,
activation='linear',
padding='same',
in_layers=[input])
output = BatchNorm(in_layers=[output])
output = ReLU(output)
output = Conv2D(num_outputs=filters2,
kernel_size=kernel_size,
activation='linear',
padding='same',
in_layers=[input])
output = BatchNorm(in_layers=[output])
output = ReLU(output)
output = Conv2D(num_outputs=filters3,
kernel_size=1,
activation='linear',
padding='same',
in_layers=[input])
output = BatchNorm(in_layers=[output])
output = Add(in_layers=[output, input])
output = ReLU(output)
return output
def test_relu(self):
"""Test that Sigmoid can be invoked."""
batch_size = 10
n_features = 5
in_tensor = np.random.rand(batch_size, n_features)
with self.session() as sess:
in_tensor = tf.convert_to_tensor(in_tensor, dtype=tf.float32)
out_tensor = ReLU()(in_tensor)
out_tensor = out_tensor.eval()
assert out_tensor.shape == (batch_size, n_features)
def test_relu(self):
"""Test that Sigmoid can be invoked."""
batch_size = 10
n_features = 5
in_tensor = np.random.rand(batch_size, n_features)
with self.session() as sess:
in_tensor = tf.convert_to_tensor(in_tensor, dtype=tf.float32)
out_tensor = ReLU()(in_tensor)
out_tensor = out_tensor.eval()
assert out_tensor.shape == (batch_size, n_features)
def test_ReLU_pickle():
tg = TensorGraph()
feature = Feature(shape=(tg.batch_size, 1))
layer = ReLU(in_layers=feature)
tg.add_output(layer)
tg.set_loss(layer)
tg.build()
tg.save()
def _build(self):
self.A_tilda_k = list()
for k in range(1, self.k_max + 1):
self.A_tilda_k.append(
Feature(name="graph_adjacency_{}".format(k),
dtype=tf.float32,
shape=[None, self.max_nodes, self.max_nodes]))
self.X = Feature(name='atom_features',
dtype=tf.float32,
shape=[None, self.max_nodes, self.num_node_features])
graph_layers = list()
adaptive_filters = list()
for index, k in enumerate(range(1, self.k_max + 1)):
in_layers = [self.A_tilda_k[index], self.X]
adaptive_filters.append(
AdaptiveFilter(batch_size=self.batch_size,
in_layers=in_layers,
num_nodes=self.max_nodes,
num_node_features=self.num_node_features,
combine_method=self.combine_method))
graph_layers.append(
KOrderGraphConv(batch_size=self.batch_size,
in_layers=in_layers +
[adaptive_filters[index]],
num_nodes=self.max_nodes,
num_node_features=self.num_node_features,
init='glorot_uniform'))
graph_features = Concat(in_layers=graph_layers, axis=2)
graph_features = ReLU(in_layers=[graph_features])
flattened = Flatten(in_layers=[graph_features])
dense1 = Dense(in_layers=[flattened],
out_channels=64,
activation_fn=tf.nn.relu)
dense2 = Dense(in_layers=[dense1],
out_channels=16,
activation_fn=tf.nn.relu)
dense3 = Dense(in_layers=[dense2],
out_channels=1 * self.n_tasks,
activation_fn=None)
output = Reshape(in_layers=[dense3], shape=(-1, self.n_tasks, 1))
self.add_output(output)
label = Label(shape=(None, self.n_tasks, 1))
weights = Weights(shape=(None, self.n_tasks))
loss = ReduceSum(L2Loss(in_layers=[label, output]))
weighted_loss = WeightedError(in_layers=[loss, weights])
self.set_loss(weighted_loss)
def __init__(self,
img_rows=224,
img_cols=224,
weights="imagenet",
classes=1000,
**kwargs):
super(ResNet50, self).__init__(use_queue=False, **kwargs)
self.img_cols = img_cols
self.img_rows = img_rows
self.weights = weights
self.classes = classes
input = Feature(shape=(None, self.img_rows, self.img_cols, 3))
labels = Label(shape=(None, self.classes))
conv1 = Conv2D(num_outputs=64,
kernel_size=7,
stride=2,
activation='linear',
padding='same',
in_layers=[input])
bn1 = BatchNorm(in_layers=[conv1])
ac1 = ReLU(bn1)
pool1 = MaxPool2D(ksize=[1, 3, 3, 1], in_layers=[bn1])
cb1 = self.conv_block(pool1, 3, [64, 64, 256], 1)
id1 = self.identity_block(cb1, 3, [64, 64, 256])
id1 = self.identity_block(id1, 3, [64, 64, 256])
cb2 = self.conv_block(id1, 3, [128, 128, 512])
id2 = self.identity_block(cb2, 3, [128, 128, 512])
id2 = self.identity_block(id2, 3, [128, 128, 512])
id2 = self.identity_block(id2, 3, [128, 128, 512])
cb3 = self.conv_block(id2, 3, [256, 256, 1024])
id3 = self.identity_block(cb3, 3, [256, 256, 1024])
id3 = self.identity_block(id3, 3, [256, 256, 1024])
id3 = self.identity_block(id3, 3, [256, 256, 1024])
id3 = self.identity_block(cb3, 3, [256, 256, 1024])
id3 = self.identity_block(id3, 3, [256, 256, 1024])
cb4 = self.conv_block(id3, 3, [512, 512, 2048])
id4 = self.identity_block(cb4, 3, [512, 512, 2048])
id4 = self.identity_block(id4, 3, [512, 512, 2048])
pool2 = AvgPool2D(ksize=[1, 7, 7, 1], in_layers=[id4])
flatten = Flatten(in_layers=[pool2])
dense = Dense(classes, in_layers=[flatten])
loss = SoftMaxCrossEntropy(in_layers=[labels, dense])
loss = ReduceMean(in_layers=[loss])
self.set_loss(loss)
self.add_output(dense)
def conv_block(self, input, kernel_size, filters, strides=2):
filters1, filters2, filters3 = filters
output = Conv2D(num_outputs=filters1,
kernel_size=1,
stride=strides,
activation='linear',
padding='same',
in_layers=[input])
output = BatchNorm(in_layers=[output])
output = ReLU(output)
output = Conv2D(num_outputs=filters2,
kernel_size=kernel_size,
activation='linear',
padding='same',
in_layers=[output])
output = BatchNorm(in_layers=[output])
output = ReLU(output)
output = Conv2D(num_outputs=filters3,
kernel_size=1,
activation='linear',
padding='same',
in_layers=[output])
output = BatchNorm(in_layers=[output])
shortcut = Conv2D(num_outputs=filters3,
kernel_size=1,
stride=strides,
activation='linear',
padding='same',
in_layers=[input])
shortcut = BatchNorm(in_layers=[shortcut])
output = Add(in_layers=[shortcut, output])
output = ReLU(output)
return output
def build_graph(self):
# inputs placeholder
self.inputs = Feature(shape=(None, self.image_size, self.image_size,
3),
dtype=tf.float32)
# data preprocessing and augmentation
in_layer = DRAugment(self.augment,
self.batch_size,
size=(self.image_size, self.image_size),
in_layers=[self.inputs])
# first conv layer
in_layer = Conv2D(int(self.n_init_kernel),
kernel_size=7,
activation_fn=None,
in_layers=[in_layer])
in_layer = BatchNorm(in_layers=[in_layer])
in_layer = ReLU(in_layers=[in_layer])
# downsample by max pooling
res_in = MaxPool2D(ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
in_layers=[in_layer])
for ct_module in range(self.n_downsample - 1):
# each module is a residual convolutional block
# followed by a convolutional downsample layer
in_layer = Conv2D(int(self.n_init_kernel * 2**(ct_module - 1)),
kernel_size=1,
activation_fn=None,
in_layers=[res_in])
in_layer = BatchNorm(in_layers=[in_layer])
in_layer = ReLU(in_layers=[in_layer])
in_layer = Conv2D(int(self.n_init_kernel * 2**(ct_module - 1)),
kernel_size=3,
activation_fn=None,
in_layers=[in_layer])
in_layer = BatchNorm(in_layers=[in_layer])
in_layer = ReLU(in_layers=[in_layer])
in_layer = Conv2D(int(self.n_init_kernel * 2**ct_module),
kernel_size=1,
activation_fn=None,
in_layers=[in_layer])
res_a = BatchNorm(in_layers=[in_layer])
res_out = res_in + res_a
res_in = Conv2D(int(self.n_init_kernel * 2**(ct_module + 1)),
kernel_size=3,
stride=2,
in_layers=[res_out])
res_in = BatchNorm(in_layers=[res_in])
# max pooling over the final outcome
in_layer = ReduceMax(axis=(1, 2), in_layers=[res_in])
for layer_size in self.n_fully_connected:
# fully connected layers
in_layer = Dense(layer_size,
activation_fn=tf.nn.relu,
in_layers=[in_layer])
# dropout for dense layers
#in_layer = Dropout(0.25, in_layers=[in_layer])
logit_pred = Dense(self.n_tasks * self.n_classes,
activation_fn=None,
in_layers=[in_layer])
logit_pred = Reshape(shape=(None, self.n_tasks, self.n_classes),
in_layers=[logit_pred])
weights = Weights(shape=(None, self.n_tasks))
labels = Label(shape=(None, self.n_tasks), dtype=tf.int32)
output = SoftMax(logit_pred)
self.add_output(output)
loss = SparseSoftMaxCrossEntropy(in_layers=[labels, logit_pred])
weighted_loss = WeightedError(in_layers=[loss, weights])
# weight decay regularizer
# weighted_loss = WeightDecay(0.1, 'l2', in_layers=[weighted_loss])
self.set_loss(weighted_loss)
def __init__(self,
n_tasks,
n_features,
alpha_init_stddevs=0.02,
layer_sizes=[1000],
weight_init_stddevs=0.02,
bias_init_consts=1.0,
weight_decay_penalty=0.0,
weight_decay_penalty_type="l2",
dropouts=0.5,
activation_fns=tf.nn.relu,
**kwargs):
"""Creates a progressive network.
Only listing parameters specific to progressive networks here.
Parameters
----------
n_tasks: int
Number of tasks
n_features: int
Number of input features
alpha_init_stddevs: list
List of standard-deviations for alpha in adapter layers.
layer_sizes: list
the size of each dense layer in the network. The length of this list determines the number of layers.
weight_init_stddevs: list or float
the standard deviation of the distribution to use for weight initialization of each layer. The length
of this list should equal len(layer_sizes)+1. The final element corresponds to the output layer.
Alternatively this may be a single value instead of a list, in which case the same value is used for every layer.
bias_init_consts: list or float
the value to initialize the biases in each layer to. The length of this list should equal len(layer_sizes)+1.
The final element corresponds to the output layer. Alternatively this may be a single value instead of a list,
in which case the same value is used for every layer.
weight_decay_penalty: float
the magnitude of the weight decay penalty to use
weight_decay_penalty_type: str
the type of penalty to use for weight decay, either 'l1' or 'l2'
dropouts: list or float
the dropout probablity to use for each layer. The length of this list should equal len(layer_sizes).
Alternatively this may be a single value instead of a list, in which case the same value is used for every layer.
activation_fns: list or object
the Tensorflow activation function to apply to each layer. The length of this list should equal
len(layer_sizes). Alternatively this may be a single value instead of a list, in which case the
same value is used for every layer.
"""
super(ProgressiveMultitaskRegressor, self).__init__(**kwargs)
self.n_tasks = n_tasks
self.n_features = n_features
self.layer_sizes = layer_sizes
self.alpha_init_stddevs = alpha_init_stddevs
self.weight_init_stddevs = weight_init_stddevs
self.bias_init_consts = bias_init_consts
self.dropouts = dropouts
self.activation_fns = activation_fns
n_layers = len(layer_sizes)
if not isinstance(weight_init_stddevs, collections.Sequence):
self.weight_init_stddevs = [weight_init_stddevs] * n_layers
if not isinstance(alpha_init_stddevs, collections.Sequence):
self.alpha_init_stddevs = [alpha_init_stddevs] * n_layers
if not isinstance(bias_init_consts, collections.Sequence):
self.bias_init_consts = [bias_init_consts] * n_layers
if not isinstance(dropouts, collections.Sequence):
self.dropouts = [dropouts] * n_layers
if not isinstance(activation_fns, collections.Sequence):
self.activation_fns = [activation_fns] * n_layers
# Add the input features.
self.mol_features = Feature(shape=(None, n_features))
all_layers = {}
outputs = []
for task in range(self.n_tasks):
task_layers = []
for i in range(n_layers):
if i == 0:
prev_layer = self.mol_features
else:
prev_layer = all_layers[(i - 1, task)]
if task > 0:
lateral_contrib, trainables = self.add_adapter(
all_layers, task, i)
task_layers.extend(trainables)
layer = Dense(in_layers=[prev_layer],
out_channels=layer_sizes[i],
activation_fn=None,
weights_initializer=TFWrapper(
tf.truncated_normal_initializer,
stddev=self.weight_init_stddevs[i]),
biases_initializer=TFWrapper(
tf.constant_initializer,
value=self.bias_init_consts[i]))
task_layers.append(layer)
if i > 0 and task > 0:
layer = layer + lateral_contrib
assert self.activation_fns[
i] is tf.nn.relu, "Only ReLU is supported"
layer = ReLU(in_layers=[layer])
if self.dropouts[i] > 0.0:
layer = Dropout(self.dropouts[i], in_layers=[layer])
all_layers[(i, task)] = layer
prev_layer = all_layers[(n_layers - 1, task)]
layer = Dense(in_layers=[prev_layer],
out_channels=1,
weights_initializer=TFWrapper(
tf.truncated_normal_initializer,
stddev=self.weight_init_stddevs[-1]),
biases_initializer=TFWrapper(
tf.constant_initializer,
value=self.bias_init_consts[-1]))
task_layers.append(layer)
if task > 0:
lateral_contrib, trainables = self.add_adapter(
all_layers, task, n_layers)
task_layers.extend(trainables)
layer = layer + lateral_contrib
outputs.append(layer)
self.add_output(layer)
task_label = Label(shape=(None, 1))
task_weight = Weights(shape=(None, 1))
weighted_loss = ReduceSum(
L2Loss(in_layers=[task_label, layer, task_weight]))
self.create_submodel(layers=task_layers,
loss=weighted_loss,
optimizer=None)
# Weight decay not activated
"""
