def __init__(self, channels=32, name='residual_conv_block'):
super(ResidualConvBlock, self).__init__(name=name)
with self._enter_variable_scope():
self._block = snt.Sequential([
snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu,
snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu,
snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu
])
def __init__(self, channels=32, name='big_conv_stem'):
super(BigConvStem, self).__init__(name=name)
with self._enter_variable_scope():
self._block = snt.Sequential([
snt.Conv2D(channels, 3), tf.nn.elu,
snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu,
snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu,
snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu,
snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu,
snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu,
snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu,
snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu,
snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu,
snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu,
snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu
])
def testShape(self):
inputs = tf.placeholder(tf.float32, shape=[self.batch_size, self.in_size])
linear = snt.Linear(self.in_size)
residual_wrapper = snt.Residual(linear, name="residual")
output = residual_wrapper(inputs)
shape = np.ndarray((self.batch_size, self.in_size))
self.assertShapeEqual(shape, output)
def __init__(self, conf, name="encoder_layer"):
""" Inits the module.
Args:
name: The module name.
"""
super(EncoderLayer, self).__init__(name=name)
self.conf = conf
self.training = True
with self._enter_variable_scope():
batch_initializer = {
'gamma': utils.initializer(conf.embedding_dim),
'moving_mean': utils.initializer(conf.embedding_dim),
'moving_variance': utils.initializer(conf.embedding_dim),
'beta': utils.initializer(conf.embedding_dim)
}
self._mha = MultiHeadAttentionResidual(conf)
self._batch_norm0 = snt.BatchNormV2(scale=True,
initializers=batch_initializer,
name="batch_norm0")
self._batch_norm1 = snt.BatchNormV2(scale=True,
initializers=batch_initializer,
name="batch_norm1")
self._lin_to_hidden = snt.Linear(
output_size=conf.ff_hidden_size,
initializers={
'w': utils.initializer(conf.embedding_dim),
'b': utils.initializer(conf.embedding_dim)
},
name="lin_to_hidden")
self._hidden_to_ouput = snt.Linear(
output_size=conf.embedding_dim,
initializers={
'w': utils.initializer(conf.ff_hidden_size),
'b': utils.initializer(conf.ff_hidden_size)
},
name="hidden_to_ouput")
self._feed_forward = snt.Sequential(
[self._lin_to_hidden, tf.nn.relu, self._hidden_to_ouput],
name="feed_forward")
self._feed_forward_residual = snt.Residual(
self._feed_forward, name="feed_forward_residual")
def __init__(self,
repr_module_name,
classification_module_name,
num_channels,
num_classes,
name='residual_boosting_module'):
"""
Args:
repr_module_name (str): name of a non-residual block to calculate
next residual representation
hypothesis_module_name (str): name of a classification block
"""
super(ResidualBoostingModule, self).__init__(name=name)
with self._enter_variable_scope():
repr_module = get_module(repr_module_name)(channels=num_channels)
self.residual_module = snt.Residual(repr_module)
self.hypothesis_module = get_module(classification_module_name)(
classes=num_classes)
self.alpha = tf.get_variable("alpha", shape=())
def testComputation(self):
inputs = tf.placeholder(tf.float32, shape=[self.batch_size, self.in_size])
linear = snt.Linear(name="rnn", output_size=self.in_size)
residual = snt.Residual(linear, name="residual")
output = residual(inputs)
w = linear.w
b = linear.b
with self.test_session() as sess:
# With random data, check the TF calculation matches the Numpy version.
input_data = np.random.randn(self.batch_size, self.in_size)
tf.global_variables_initializer().run()
fetches = [output, w, b]
output = sess.run(fetches, {inputs: input_data})
output_v, w_v, b_v = output
output = np.dot(input_data, w_v) + b_v
residual_output = output + input_data
self.assertAllClose(residual_output, output_v)
def __init__(self, initial_trans, modules):
self._initial_trans = initial_trans
self._res_modules = [snt.Residual(m) for m in modules]
labeled_train_data = train_data[:args.num_labeled_data]
labeled_train_labels = train_labels[:args.num_labeled_data]
# shuffle data and create generators
labeled_train_gen = data.parallel_data_generator(
[labeled_train_data, labeled_train_labels], args.labeled_batch_size)
unlabeled_train_gen = data.parallel_data_generator([train_data, train_labels],
args.unlabeled_batch_size)
test_gen = data.parallel_data_generator([test_data, test_labels],
args.labeled_batch_size)
# build the model
if args.dataset == 'cifar10':
encoder_module = snt.Sequential([
snt.Conv2D(16, 3),
snt.Residual(snt.Conv2D(16, 3)),
snt.Residual(snt.Conv2D(16, 3)), scaling.squeeze2d,
snt.Conv2D(64, 3),
snt.Residual(snt.Conv2D(64, 3)),
snt.Residual(snt.Conv2D(64, 3)), scaling.squeeze2d,
snt.Conv2D(64, 3),
snt.Residual(snt.Conv2D(64, 3)),
snt.Residual(snt.Conv2D(64, 3)), scaling.squeeze2d,
snt.Conv2D(128, 3),
snt.Residual(snt.Conv2D(128, 3)),
snt.Residual(snt.Conv2D(128, 3)), scaling.squeeze2d,
snt.Conv2D(256, 3),
snt.Residual(snt.Conv2D(256, 3)),
snt.Residual(snt.Conv2D(256, 3)), scaling.squeeze2d,
tf.keras.layers.Flatten(),
snt.Linear(100)
# load the data
if args.dataset == 'cifar10':
train_data, _, _, _ = data.load_cifar10('./data/')
elif args.dataset == 'mnist':
train_data, _, _, _ = data.load_mnist('./data/')
train_data = np.reshape(train_data, [-1, 28, 28, 1])
data_shape = (args.batch_size, ) + train_data.shape[1:]
batches_per_epoch = train_data.shape[0] // args.batch_size
train_gen = data.data_generator(train_data, args.batch_size)
# build the model
if args.dataset == 'cifar10':
encoder_module = snt.Sequential([
snt.Conv2D(16, 3),
snt.Residual(snt.Conv2D(16, 3)),
snt.Residual(snt.Conv2D(16, 3)), scaling.squeeze2d,
snt.Conv2D(64, 3),
snt.Residual(snt.Conv2D(64, 3)),
snt.Residual(snt.Conv2D(64, 3)), scaling.squeeze2d,
snt.Conv2D(64, 3),
snt.Residual(snt.Conv2D(64, 3)),
snt.Residual(snt.Conv2D(64, 3)), scaling.squeeze2d,
snt.Conv2D(128, 3),
snt.Residual(snt.Conv2D(128, 3)),
snt.Residual(snt.Conv2D(128, 3)), scaling.squeeze2d,
snt.Conv2D(256, 3),
snt.Residual(snt.Conv2D(256, 3)),
snt.Residual(snt.Conv2D(256, 3)), scaling.squeeze2d,
tf.keras.layers.Flatten(),
snt.Linear(100)
