def __init__(self,
z_dim=None,
sample_shape=None,
output_shape=None,
mark=None,
classes=0):
# Call parent's constructor
Model.__init__(self, mark)
self._targets = None
self._conditional = classes > 0
self._classes = classes
if self._conditional:
with self._graph.as_default():
self._targets = tf.placeholder(dtype=tf.float32,
shape=[None, classes],
name='one_hot_labels')
# Define generator and discriminator
self.Generator = Net(pedia.Generator)
self.Discriminator = Net(pedia.Discriminator)
# Alias
self.G = self.Generator
self.D = self.Discriminator
# If z_dim/sample_shape is provided, define the input for
# generator/discriminator accordingly
if z_dim is not None:
self.G.add(Input(sample_shape=[None, z_dim], name='z'))
if sample_shape is not None:
if (not isinstance(sample_shape, list)
and not isinstance(sample_shape, tuple)):
raise TypeError('sample shape must be a list or a tuple')
self.D.add(
Input(sample_shape=[None] + list(sample_shape),
name='samples'))
self._z_dim = z_dim
self._sample_shape = sample_shape
self._output_shape = output_shape
self._sample_num = None
# Private tensors and ops
self._G, self._outputs = None, None
self._Dr, self._Df = None, None
self._logits_Dr, self._logits_Df = None, None
self._loss_G, self._loss_D = None, None
self._loss_Dr, self._loss_Df = None, None
self._train_step_G, self._train_step_D = None, None
self._merged_summary_G, self._merged_summary_D = None, None
def mlp_00(learning_rate=0.001, memory_depth=80):
"""
Performance on WH:
[0] depth = 80
"""
# Configuration
hidden_dims = [2 * memory_depth] * 4
strength = 0
activation = 'lrelu'
mark = 'mlp_D{}_{}_{}'.format(memory_depth, hidden_dims, activation)
# Initiate a predictor
model = NeuralNet(memory_depth, mark=mark)
nn = model.nn
assert isinstance(nn, Predictor)
# Add layers
nn.add(Input([memory_depth]))
lc._add_fc_relu_layers(nn, hidden_dims, activation, strength=strength)
nn.add(Linear(output_dim=1, weight_regularizer='l2', strength=strength))
# Build model
# optimizer = tf.train.GradientDescentOptimizer(learning_rate)
optimizer = tf.train.AdamOptimizer(learning_rate)
nn.build(loss='euclid',
metric='rms_ratio',
metric_name='RMS(err)%',
optimizer=optimizer)
# Return model
return model
def svn_01(memory_depth, mark, hidden_dim, order1, learning_rate=0.001):
strength = 0
# Initiate a predictor
model = NeuralNet(memory_depth, mark=mark)
nn = model.nn
assert isinstance(nn, Predictor)
# Add layers
nn.add(Input([memory_depth]))
nn.add(Linear(output_dim=hidden_dim))
nn.add(Polynomial(order=order1))
nn.add(
Linear(output_dim=1,
weight_regularizer='l2',
strength=strength,
use_bias=False))
# Build model
nn.build(loss='euclid',
metric='rms_ratio',
metric_name='RMS(err)%',
optimizer=tf.train.AdamOptimizer(learning_rate))
# Return model
return model
def mlp_00(memory_depth, mark):
D = memory_depth
hidden_dims = [10, 10, 10]
activation = lambda: Activation('relu')
learning_rate = 0.001
reg = 0.00
# Initiate model
model = NeuralNet(memory_depth, mark)
model.nn.add(Input([D]))
for dim in hidden_dims:
model.nn.add(
Linear(output_dim=dim, weight_regularizer='l2', strength=reg))
model.nn.add(activation())
model.nn.add(Linear(output_dim=1, weight_regularizer='l2', strength=reg))
# Build model
model.nn.build(loss='euclid',
metric='ratio',
metric_name='Err%',
optimizer=tf.train.AdamOptimizer(learning_rate))
return model
def mlp_00(mark,
memory_depth,
layer_dim,
layer_num,
learning_rate,
activation='relu'):
# Configurations
pass
# Initiate a predictor
model = NeuralNet(memory_depth, mark=mark)
nn = model.nn
assert isinstance(nn, Predictor)
# Add layers
nn.add(Input([memory_depth]))
for i in range(layer_num):
nn.add(Linear(output_dim=layer_dim))
nn.add(Activation(activation))
nn.add(Linear(output_dim=1))
# Build model
model.default_build(learning_rate)
# Return model
return model
def test_00(memory, learning_rate=0.001):
# Configurations
mark = 'test'
D = memory
# Initiate model
model = NeuralNet(memory, mark=mark)
nn = model.nn
assert isinstance(nn, Predictor)
# Add layers
nn.add(Input([memory]))
nn.add(Linear(output_dim=2 * D))
nn.add(Activation('relu'))
nn.add(Linear(output_dim=2 * D))
nn.add(Activation('relu'))
nn.add(Linear(output_dim=2 * D))
nn.add(Polynomial(order=3))
nn.add(Linear(output_dim=1))
# Build model
model.default_build(learning_rate)
# Return model
return model
def __init__(self,
degree,
depth,
mark=None,
max_volterra_order=3,
**kwargs):
# Check parameters
if degree < 1: raise ValueError('!! Degree must be a positive integer')
if depth < 0: raise ValueError('!! Depth must be a positive integer')
# Call parent's constructor
Model.__init__(self, mark)
# Initialize fields
self.degree = degree
self.depth = depth
self._max_volterra_order = min(max_volterra_order, degree)
self.T = {}
self._input = Input([depth], name='input')
self._output = None
self._target = None
self._alpha = 1.1
self._outputs = {}
# Initialize operators in each degree
orders = kwargs.get('orders', None)
if orders is None: orders = list(range(1, self.degree + 1))
self.orders = orders
self._init_T()
def res_00(th, activation='relu'):
assert isinstance(th, NlsHub)
model = NeuralNet(th.memory_depth, mark=th.mark, nn_class=Predictor)
nn = model.nn
assert isinstance(nn, Predictor)
# Add blocks
nn.add(Input([th.memory_depth]))
nn.add(
Linear(output_dim=th.hidden_dim,
weight_regularizer=th.regularizer,
strength=th.reg_strength))
nn.add(Activation(activation))
def add_res_block():
net = nn.add(ResidualNet())
net.add(
Linear(output_dim=th.hidden_dim,
weight_regularizer=th.regularizer,
strength=th.reg_strength))
net.add(Activation(activation))
net.add_shortcut()
for _ in range(th.num_blocks):
add_res_block()
nn.add(Linear(output_dim=1))
# Build model
model.default_build(th.learning_rate)
# Return model
return model
def bres_net_res0(th, activation='relu'):
assert isinstance(th, NlsHub)
# Initiate a neural net model
th.mark = '{}-{}'.format(th.mark, 'res')
model = NeuralNet(th.memory_depth, mark=th.mark, nn_class=BResNet)
nn = model.nn
assert isinstance(nn, BResNet)
nn.strict_residual = False
# Add layers
nn.add(Input([th.memory_depth]))
nn.add(Linear(output_dim=th.hidden_dim))
nn.add(Activation(activation))
branch = nn.add_branch()
branch.add(Linear(output_dim=1))
def add_res_block():
net = nn.add(ResidualNet())
net.add(Linear(output_dim=th.hidden_dim))
net.add(Activation(activation))
net.add_shortcut()
branch = nn.add_branch()
branch.add(Linear(output_dim=1))
for _ in range(th.num_blocks - 1):
add_res_block()
nn.add(Linear(output_dim=1))
# Build
model.default_build(th.learning_rate)
return model
def fc_lstm(th):
assert isinstance(th, Config)
th.mark = 'fc_lstm_' + th.mark
# Initiate a model
model = Classifier(mark=th.mark, net_type=Recurrent)
# Add input layer
model.add(Input(sample_shape=th.input_shape))
# Add fc layers
for dim in th.fc_dims:
checker.check_positive_integer(dim)
model.add(Linear(output_dim=dim))
# model.add(BatchNorm())
model.add(Activation('relu'))
# Add lstm cells
for dim in th.rc_dims:
model.add(BasicLSTMCell(state_size=dim))
# Add output layer
# model.add(Linear(output_dim=th.output_dim))
# Build model
optimizer = tf.train.AdamOptimizer(th.learning_rate)
model.build(optimizer)
return model
def multinput(th):
assert isinstance(th, Config)
model = Classifier(mark=th.mark)
# Add hidden layers
assert isinstance(th.fc_dims, list)
subnet = model.add(inter_type=model.CONCAT)
for dims in th.fc_dims:
subsubnet = subnet.add()
# Add input layer
subsubnet.add(Input(sample_shape=th.input_shape))
subsubnet.add(Flatten())
assert isinstance(dims, list)
for dim in dims:
subsubnet.add(Linear(output_dim=dim))
if core.use_bn: subsubnet.add(BatchNormalization())
subsubnet.add(Activation(th.actype1))
# Add output layer
model.add(Linear(output_dim=th.num_classes))
# Build model
optimizer = tf.train.AdamOptimizer(learning_rate=th.learning_rate)
model.build(optimizer=optimizer)
return model
def get_container(th, flatten=False):
assert isinstance(th, Config)
model = Classifier(mark=th.mark)
model.add(Input(sample_shape=th.input_shape))
if th.centralize_data: model.add(Normalize(mu=th.data_mean, sigma=255.))
if flatten: model.add(Flatten())
return model
def multinput(th):
assert isinstance(th, Config)
model = Classifier(mark=th.mark)
# Add hidden layers
assert isinstance(th.fc_dims, list)
subnet = model.add(inter_type=model.CONCAT)
for dims in th.fc_dims:
subsubnet = subnet.add()
# Add input layer
subsubnet.add(Input(sample_shape=th.input_shape))
subsubnet.add(Flatten())
assert isinstance(dims, list)
for dim in dims:
subsubnet.add(Linear(output_dim=dim))
# if cf10_core.use_bn: subsubnet.add(BatchNormalization())
subsubnet.add(Activation(th.actype1))
# Add output layer
model.add(Linear(output_dim=th.num_classes))
# Build model
model.build(metric=['accuracy', 'loss'],
batch_metric='accuracy',
eval_metric='accuracy')
return model
def __init__(self,
z_dim=None,
sample_shape=None,
output_shape=None,
mark=None,
classes=0):
# Call parent's constructor
Model.__init__(self, mark)
# Fields
self._output_shape = output_shape
# Define encoder and decoder
self.Encoder = Net(pedia.Encoder)
self.Decoder = Net(pedia.Decoder)
self.Q = self.Encoder
self.P = self.Decoder
# If z_dim/sample_shape is provided, define the input for
# decoder/encoder accordingly
if z_dim is not None:
self.P.add(Input(sample_shape=[None, z_dim], name='z'))
if sample_shape is not None:
if (not isinstance(sample_shape, list)
and not isinstance(sample_shape, tuple)):
raise TypeError('sample shape must be a list or a tuple')
self.Q.add(
Input(sample_shape=[None] + list(sample_shape),
name='samples'))
# Placeholders
self._sample_num = None
self._classes = classes
self._conditional = classes > 0
if self._conditional:
self._targets = tf.placeholder(dtype=tf.float32,
shape=[None, classes],
name='one_hot_labels')
self._P, self._outputs = None, None
# ...
pass
def conv_2d_test(th):
assert isinstance(th, Config)
# Initiate model
th.mark = 'cnn_2d' + th.mark
def data_dim(sample_rate=44100, duration=2, n_mfcc=40):
audio_length = sample_rate * duration
dim = (n_mfcc, 1 + int(np.floor(audio_length / 512)), 1)
return dim
dim = data_dim()
model = Classifier(mark=th.mark)
# Add input layer
model.add(Input(sample_shape=[dim[0], dim[1], 1]))
# Add hidden layers
model.add(Conv2D(32, (4, 10), padding='same'))
model.add(BatchNorm())
model.add(Activation('relu'))
model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2)))
# model.add(Dropout(0.7))
model.add(Conv2D(32, (4, 10), padding='same'))
model.add(BatchNorm())
model.add(Activation('relu'))
model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2)))
# model.add(Dropout(0.7))
model.add(Conv2D(32, (4, 10), padding='same'))
model.add(BatchNorm())
model.add(Activation('relu'))
model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2)))
# model.add(Dropout(0.7))
model.add(Conv2D(32, (4, 10), padding='same'))
model.add(BatchNorm())
model.add(Activation('relu'))
model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2)))
# model.add(Dropout(0.7))
model.add(Flatten())
model.add(Linear(output_dim=64))
model.add(BatchNorm())
model.add(Activation('relu'))
# Add output layer
model.add(Linear(output_dim=41))
model.add(Activation('softmax'))
# Build model
optimizer = tf.train.AdamOptimizer(learning_rate=th.learning_rate)
model.build(optimizer=optimizer)
return model
def typical(th, cell):
assert isinstance(th, Config)
# Initiate a model
model = Classifier(mark=th.mark, net_type=Recurrent)
# Add layers
model.add(Input(sample_shape=th.input_shape))
# Add hidden layers
model.add(cell)
# Build model and return
output_and_build(model, th)
return model
def get_container(th, flatten=False):
assert isinstance(th, Config)
model = Classifier(mark=th.mark)
model.add(Input(sample_shape=th.input_shape))
model.add(Normalize(sigma=255.))
if th.centralize_data: model.add(Normalize(mu=th.data_mean))
if flatten:
model.add(Flatten())
# Register extractor and researcher
model.register_extractor(mn_du.MNIST.connection_heat_map_extractor)
monitor.register_grad_researcher(mn_du.MNIST.flatten_researcher)
return model
def typical(th, cells): assert isinstance(th, Config) # Initiate a model model = Predictor(mark=th.mark, net_type=Recurrent) # Add layers model.add(Input(sample_shape=th.input_shape)) # Add hidden layers if not isinstance(cells, (list, tuple)): cells = [cells] for cell in cells: model.add(cell) # Build model and return output_and_build(model, th) return model
def conv_test(th):
assert isinstance(th, Config)
# Initiate model
th.mark = 'cnn' + th.mark
model = Classifier(mark=th.mark)
# Add input layer
model.add(Input(sample_shape=[32000, 1]))
# Add hidden layers
model.add(Conv1D(filters=16, kernel_size=9, padding='valid'))
model.add(Activation('relu'))
model.add(Conv1D(filters=16, kernel_size=9, padding='valid'))
model.add(Activation('relu'))
model.add(MaxPool1D(pool_size=16, strides=16))
# model.add(Dropout(0.9))
model.add(Conv1D(filters=32, kernel_size=3, padding='valid'))
model.add(Activation('relu'))
model.add(Conv1D(filters=32, kernel_size=3, padding='valid'))
model.add(Activation('relu'))
model.add(MaxPool1D(pool_size=4, strides=4))
# model.add(Dropout(0.9))
model.add(Conv1D(filters=32, kernel_size=3, padding='valid'))
model.add(Activation('relu'))
model.add(Conv1D(filters=32, kernel_size=3, padding='valid'))
model.add(Activation('relu'))
model.add(MaxPool1D(pool_size=4, strides=4))
# model.add(Dropout(0.9))
#
model.add(Conv1D(filters=256, kernel_size=3, padding='valid'))
model.add(Activation('relu'))
model.add(Conv1D(filters=256, kernel_size=3, padding='valid'))
model.add(Activation('relu'))
model.add(GlobalMaxPooling1D())
# model.add(Dropout(0.8))
#
model.add(Linear(output_dim=64))
model.add(Activation('relu'))
model.add(Linear(output_dim=1028))
model.add(Activation('relu'))
# Add output layer
model.add(Linear(output_dim=41))
model.add(Activation('softmax'))
# Build model
optimizer = tf.train.AdamOptimizer(learning_rate=th.learning_rate)
model.build(optimizer=optimizer)
return model
def vanilla_RNN(mark):
batch_size = 3
num_steps = 8
model = Predictor(mark, net_type=Recurrent)
# Add functions
model.add(Input(sample_shape=[1]))
model.add(BasicRNNCell(state_size=10, inner_struct='concat'))
model.add(Linear(output_dim=1))
# Build model
model._build(loss='euclid', metric='ratio', metric_name='Err %')
# Return model
return model
def res_00(memory,
blocks,
order1,
order2,
activation='relu',
learning_rate=0.001):
# Configurations
mark = 'res'
D = memory
# Initiate model
model = NeuralNet(memory, mark=mark)
nn = model.nn
assert isinstance(nn, Predictor)
# Add layers
nn.add(Input([D]))
def add_res_block():
net = nn.add(ResidualNet())
net.add(Linear(output_dim=D))
net.add(Activation(activation))
net.add(Linear(output_dim=D))
net.add_shortcut()
net.add(Activation(activation))
def add_res_block_poly():
net = nn.add(ResidualNet())
net.add(Linear(output_dim=D))
net.add(Polynomial(order=order1))
net.add(Linear(output_dim=D))
net.add_shortcut()
net.add(Polynomial(order=order2))
if activation == 'poly':
for _ in range(blocks):
add_res_block_poly()
else:
for _ in range(blocks):
add_res_block()
nn.add(Linear(output_dim=1))
# Build model
model.default_build(learning_rate)
# Return model
return model
def mlp02(mark,
memory_depth,
branch_num,
hidden_dim,
learning_rate,
activation,
identity_init=False):
# Initiate a neural net
if identity_init:
model = NeuralNet(memory_depth,
mark=mark,
bamboo_braod=True,
identity_initial=True)
else:
model = NeuralNet(memory_depth,
mark=mark,
bamboo_broad=True,
identity_initial=False)
nn = model.nn
assert isinstance(nn, Bamboo_Broad)
# Add layers
nn.add(Input([memory_depth]))
branch = nn.add_branch()
branch.add(Linear(output_dim=hidden_dim))
branch.add(Activation(activation))
branch.add(Linear(output_dim=1))
for _ in range(branch_num - 1):
branch = nn.add_branch()
branch.add(
Linear(output_dim=hidden_dim,
weight_initializer=tf.zeros_initializer(),
bias_initializer=tf.zeros_initializer()))
branch.add(Activation(activation))
branch.add(
Linear(output_dim=1,
weight_initializer=tf.zeros_initializer(),
bias_initializer=tf.zeros_initializer()))
# Build model
model.default_build(learning_rate)
# Return model
return model
def deep_conv(mark):
# Initiate predictor
model = Classifier(mark=mark)
model.add(Input(sample_shape=config.sample_shape))
def ConvBNReLU(filters, strength=1.0, bn=True):
model.add(
Conv2D(filters=filters,
kernel_size=5,
padding='same',
kernel_regularizer=regularizers.L2(strength=strength)))
if bn:
model.add(BatchNorm())
model.add(Activation('relu'))
# Conv layers
reg = 1e-5
ConvBNReLU(32, reg)
model.add(Dropout(0.5))
ConvBNReLU(32, reg)
model.add(MaxPool2D(2, 2, 'same'))
ConvBNReLU(64, reg)
model.add(Dropout(0.5))
ConvBNReLU(64, reg)
model.add(MaxPool2D(2, 2, 'same'))
ConvBNReLU(128, reg)
# FC layers
model.add(Flatten())
model.add(Linear(256))
# model.add(BatchNorm())
model.add(Activation('relu'))
model.add(Linear(256))
# model.add(BatchNorm())
model.add(Activation('relu'))
model.add(Linear(config.y_dim))
# Build model
model.build(optimizer=tf.train.AdamOptimizer(learning_rate=1e-4))
return model
def mlp_res00(mark,
memory_depth,
branch_num,
hidden_dim,
learning_rate,
activation,
identity_init=True):
# Initiate a neural net
if identity_init:
model = NeuralNet(memory_depth,
mark=mark,
bamboo=True,
identity_initial=True)
else:
model = NeuralNet(memory_depth,
mark=mark,
bamboo=True,
identity_initial=False)
nn = model.nn
assert isinstance(nn, Bamboo)
# Add layers
nn.add(Input([memory_depth]))
for _ in range(branch_num):
nn.add(Linear(output_dim=hidden_dim))
nn.add(Activation(activation))
branch = nn.add_branch()
branch.add(Linear(output_dim=1))
resnet = nn.add(ResidualNet())
resnet.add(Linear(output_dim=hidden_dim))
resnet.add(Activation(activation))
resnet.add(Linear(output_dim=hidden_dim))
resnet.add_shortcut()
resnet.add(Activation(activation))
nn.add(Linear(output_dim=hidden_dim))
nn.add(Activation(activation))
nn.add(Linear(output_dim=1))
# Build model
model.default_build(learning_rate)
# Return model
return model
def pet(memory, hidden_dim, order, learning_rate, mark='pet'):
# Initiate a predictor
model = NeuralNet(memory_depth=memory, mark=mark)
nn = model.nn
assert isinstance(nn, Predictor)
# Add layers
nn.add(Input([memory]))
nn.add(Linear(output_dim=hidden_dim, use_bias=False))
nn.add(inter_type=pedia.sum)
for i in range(1, order + 1):
nn.add_to_last_net(Homogeneous(order=i))
# Build model
model.default_build(learning_rate=learning_rate)
return model
def rnn0(th):
assert isinstance(th, NlsHub)
# Initiate a neural net model
nn_class = lambda mark: Predictor(mark=mark, net_type=Recurrent)
model = NeuralNet(th.memory_depth, mark=th.mark, nn_class=nn_class)
nn = model.nn
assert isinstance(nn, Predictor)
# Add layers
nn.add(Input(sample_shape=[th.memory_depth]))
for _ in range(th.num_blocks):
nn.add(BasicRNNCell(state_size=th.hidden_dim))
nn.add(Linear(output_dim=1))
# Build
model.default_build(th.learning_rate)
return model
def lstm(th):
assert isinstance(th, Config)
# Initiate model
th.mark = 'lstm_' + th.mark
model = Predictor(mark=th.mark, net_type=Recurrent)
# Add input layer
model.add(Input(sample_shape=[th.memory_depth]))
# Add hidden layers
for _ in range(th.num_blocks):
model.add(BasicLSTMCell(th.hidden_dim, with_peepholes=False))
# Add output layer
model.add(Linear(output_dim=1))
# Build model
optimizer = tf.train.AdamOptimizer(learning_rate=th.learning_rate)
model.build_as_regressor(optimizer)
return model
def tlp(memory_depth, hidden_dim, mark='tlp'):
# Hyper-parameters
learning_rate = 0.001
# Initiate a predictor
model = NeuralNet(memory_depth, mark=mark)
nn = model.nn
assert isinstance(nn, Predictor)
# Add layers
nn.add(Input([memory_depth]))
nn.add(Linear(output_dim=hidden_dim))
nn.add(Activation('sigmoid'))
nn.add(Linear(output_dim=1, use_bias=False))
# Build model
model.default_build(learning_rate=learning_rate)
return model
def mlp00(mark):
# Define model
model = TDPlayer(mark=mark)
model.add(Input(sample_shape=[15, 15]))
model.add(Flatten())
model.add(Linear(225))
model.add(Activation.ReLU())
model.add(Linear(225))
model.add(Activation.ReLU())
model.add(Linear(1))
model.add(Activation('sigmoid'))
# Build model
model.build()
return model
def vanilla(mark):
model = Classifier(mark=mark)
model.add(Input(sample_shape=[784]))
def fc_bn_relu(bn=True):
model.add(Linear(100))
model.add(Activation('relu'))
if bn:
model.add(BatchNorm())
fc_bn_relu()
fc_bn_relu()
model.add(Linear(10))
# Build model
model.build(loss='cross_entropy',
optimizer=tf.train.GradientDescentOptimizer(0.01))
return model