def __init__(self, channels=32, name='residual_conv_block'): super(ResidualConvBlock, self).__init__(name=name) with self._enter_variable_scope(): self._block = snt.Sequential([ snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu, snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu, snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu ])
def __init__(self, channels=32, name='big_conv_stem'): super(BigConvStem, self).__init__(name=name) with self._enter_variable_scope(): self._block = snt.Sequential([ snt.Conv2D(channels, 3), tf.nn.elu, snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu, snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu, snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu, snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu, snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu, snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu, snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu, snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu, snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu, snt.Residual(snt.Conv2D(channels, 3)), tf.nn.elu ])
def testShape(self): inputs = tf.placeholder(tf.float32, shape=[self.batch_size, self.in_size]) linear = snt.Linear(self.in_size) residual_wrapper = snt.Residual(linear, name="residual") output = residual_wrapper(inputs) shape = np.ndarray((self.batch_size, self.in_size)) self.assertShapeEqual(shape, output)
def __init__(self, conf, name="encoder_layer"): """ Inits the module. Args: name: The module name. """ super(EncoderLayer, self).__init__(name=name) self.conf = conf self.training = True with self._enter_variable_scope(): batch_initializer = { 'gamma': utils.initializer(conf.embedding_dim), 'moving_mean': utils.initializer(conf.embedding_dim), 'moving_variance': utils.initializer(conf.embedding_dim), 'beta': utils.initializer(conf.embedding_dim) } self._mha = MultiHeadAttentionResidual(conf) self._batch_norm0 = snt.BatchNormV2(scale=True, initializers=batch_initializer, name="batch_norm0") self._batch_norm1 = snt.BatchNormV2(scale=True, initializers=batch_initializer, name="batch_norm1") self._lin_to_hidden = snt.Linear( output_size=conf.ff_hidden_size, initializers={ 'w': utils.initializer(conf.embedding_dim), 'b': utils.initializer(conf.embedding_dim) }, name="lin_to_hidden") self._hidden_to_ouput = snt.Linear( output_size=conf.embedding_dim, initializers={ 'w': utils.initializer(conf.ff_hidden_size), 'b': utils.initializer(conf.ff_hidden_size) }, name="hidden_to_ouput") self._feed_forward = snt.Sequential( [self._lin_to_hidden, tf.nn.relu, self._hidden_to_ouput], name="feed_forward") self._feed_forward_residual = snt.Residual( self._feed_forward, name="feed_forward_residual")
def __init__(self, repr_module_name, classification_module_name, num_channels, num_classes, name='residual_boosting_module'): """ Args: repr_module_name (str): name of a non-residual block to calculate next residual representation hypothesis_module_name (str): name of a classification block """ super(ResidualBoostingModule, self).__init__(name=name) with self._enter_variable_scope(): repr_module = get_module(repr_module_name)(channels=num_channels) self.residual_module = snt.Residual(repr_module) self.hypothesis_module = get_module(classification_module_name)( classes=num_classes) self.alpha = tf.get_variable("alpha", shape=())
def testComputation(self): inputs = tf.placeholder(tf.float32, shape=[self.batch_size, self.in_size]) linear = snt.Linear(name="rnn", output_size=self.in_size) residual = snt.Residual(linear, name="residual") output = residual(inputs) w = linear.w b = linear.b with self.test_session() as sess: # With random data, check the TF calculation matches the Numpy version. input_data = np.random.randn(self.batch_size, self.in_size) tf.global_variables_initializer().run() fetches = [output, w, b] output = sess.run(fetches, {inputs: input_data}) output_v, w_v, b_v = output output = np.dot(input_data, w_v) + b_v residual_output = output + input_data self.assertAllClose(residual_output, output_v)
def __init__(self, initial_trans, modules): self._initial_trans = initial_trans self._res_modules = [snt.Residual(m) for m in modules]
labeled_train_data = train_data[:args.num_labeled_data] labeled_train_labels = train_labels[:args.num_labeled_data] # shuffle data and create generators labeled_train_gen = data.parallel_data_generator( [labeled_train_data, labeled_train_labels], args.labeled_batch_size) unlabeled_train_gen = data.parallel_data_generator([train_data, train_labels], args.unlabeled_batch_size) test_gen = data.parallel_data_generator([test_data, test_labels], args.labeled_batch_size) # build the model if args.dataset == 'cifar10': encoder_module = snt.Sequential([ snt.Conv2D(16, 3), snt.Residual(snt.Conv2D(16, 3)), snt.Residual(snt.Conv2D(16, 3)), scaling.squeeze2d, snt.Conv2D(64, 3), snt.Residual(snt.Conv2D(64, 3)), snt.Residual(snt.Conv2D(64, 3)), scaling.squeeze2d, snt.Conv2D(64, 3), snt.Residual(snt.Conv2D(64, 3)), snt.Residual(snt.Conv2D(64, 3)), scaling.squeeze2d, snt.Conv2D(128, 3), snt.Residual(snt.Conv2D(128, 3)), snt.Residual(snt.Conv2D(128, 3)), scaling.squeeze2d, snt.Conv2D(256, 3), snt.Residual(snt.Conv2D(256, 3)), snt.Residual(snt.Conv2D(256, 3)), scaling.squeeze2d, tf.keras.layers.Flatten(), snt.Linear(100)
# load the data if args.dataset == 'cifar10': train_data, _, _, _ = data.load_cifar10('./data/') elif args.dataset == 'mnist': train_data, _, _, _ = data.load_mnist('./data/') train_data = np.reshape(train_data, [-1, 28, 28, 1]) data_shape = (args.batch_size, ) + train_data.shape[1:] batches_per_epoch = train_data.shape[0] // args.batch_size train_gen = data.data_generator(train_data, args.batch_size) # build the model if args.dataset == 'cifar10': encoder_module = snt.Sequential([ snt.Conv2D(16, 3), snt.Residual(snt.Conv2D(16, 3)), snt.Residual(snt.Conv2D(16, 3)), scaling.squeeze2d, snt.Conv2D(64, 3), snt.Residual(snt.Conv2D(64, 3)), snt.Residual(snt.Conv2D(64, 3)), scaling.squeeze2d, snt.Conv2D(64, 3), snt.Residual(snt.Conv2D(64, 3)), snt.Residual(snt.Conv2D(64, 3)), scaling.squeeze2d, snt.Conv2D(128, 3), snt.Residual(snt.Conv2D(128, 3)), snt.Residual(snt.Conv2D(128, 3)), scaling.squeeze2d, snt.Conv2D(256, 3), snt.Residual(snt.Conv2D(256, 3)), snt.Residual(snt.Conv2D(256, 3)), scaling.squeeze2d, tf.keras.layers.Flatten(), snt.Linear(100)