def define_inner_modules(self, name, activations, filter_shapes,
bias_shapes, ksizes, pool_strides, keep_prob):
self.layers = {}
# first convolutional layer
self.layers["conv0"] = mod.TimeConvolutionalLayerModule(
"conv0", activations[0], filter_shapes[0], [1, 1, 1, 1],
bias_shapes[0])
lateral_filter_shape = filter_shapes[0]
tmp = lateral_filter_shape[2]
lateral_filter_shape[2] = lateral_filter_shape[3]
lateral_filter_shape[3] = tmp
self.layers["lateral0"] = mod.Conv2DModule("lateral0",
lateral_filter_shape,
[1, 1, 1, 1])
# first max-pooling layer
self.layers["pool0"] = mod.MaxPoolingModule("pool0", ksizes[0],
pool_strides[0])
# second convolutional layer
self.layers["conv1"] = mod.TimeConvolutionalLayerModule(
name + "conv1", activations[1], filter_shapes[1], [1, 1, 1, 1],
bias_shapes[1])
lateral_filter_shape = filter_shapes[1]
tmp = lateral_filter_shape[2]
lateral_filter_shape[2] = lateral_filter_shape[3]
lateral_filter_shape[3] = tmp
self.layers["lateral1"] = mod.Conv2DModule("lateral1",
lateral_filter_shape,
[1, 1, 1, 1])
# second max-pooling layer
self.layers["pool1"] = mod.MaxPoolingModule("pool1", ksizes[0],
pool_strides[0])
self.layers["flat_pool1"] = mod.FlattenModule("flat_pool1")
# first fully-connected layer
self.layers["fc0"] = mod.FullyConnectedLayerModule(
"fc0", activations[2],
int(np.prod(np.array(bias_shapes[1]) / np.array(pool_strides[1]))),
np.prod(bias_shapes[2]))
# dropout
self.layers["dropout0"] = mod.DropoutModule("dropout0", keep_prob)
# second fully-connected layer
self.layers["fc1"] = mod.FullyConnectedLayerModule(
"fc1", activations[3], np.prod(bias_shapes[2]),
np.prod(bias_shapes[3]))
# connections
self.layers["lateral0"].add_input(self.layers["conv0"].preactivation)
self.layers["conv0"].add_input(self.layers["lateral0"], -1)
self.layers["pool0"].add_input(self.layers["conv0"])
self.layers["conv1"].add_input(self.layers["pool0"], 0)
self.layers["lateral1"].add_input(self.layers["conv1"].preactivation)
self.layers["conv1"].add_input(self.layers["lateral1"], -1)
self.layers["pool1"].add_input(self.layers["conv1"])
self.layers["flat_pool1"].add_input(self.layers["pool1"])
self.layers["fc0"].add_input(self.layers["flat_pool1"])
self.layers["dropout0"].add_input(self.layers["fc0"])
self.layers["fc1"].add_input(self.layers["dropout0"])
# set input and output
self.input_module = self.layers["conv0"]
self.output_module = self.layers["fc1"]
def define_inner_modules(self, name, in_size, cell_size):
"""Typical LSTM cell with three gates. Detailed tutorial see
http://colah.github.io/posts/2015-08-Understanding-LSTMs/
"""
self.in_size = in_size
self.cell_size = cell_size
self.input_module = mod.ConcatModule("concat", 1, in_size + cell_size)
# Three gates for input, output, cell state
f_t = mod.FullyConnectedLayerModule("f_t", tf.sigmoid,
in_size + cell_size, cell_size)
i_t = mod.FullyConnectedLayerModule("i_t", tf.sigmoid,
in_size + cell_size, cell_size)
o_t = mod.FullyConnectedLayerModule("o_t", tf.sigmoid,
in_size + cell_size, cell_size)
# transformed input and last time hidden-state
CHat_t = mod.FullyConnectedLayerModule("CHat_t", tf.tanh,
in_size + cell_size, cell_size)
# cell states related
self.C_t = mod.AddModule("C_t")
tanh_C_t = mod.ActivationModule("tanh_C_t", tf.tanh)
# residual of last time cell state
state_residual = mod.EleMultiModule("state_residual")
state_update = mod.EleMultiModule(
"state_update") # i_t(eleMulti)CHat_t
# hidden states
self.h_t = mod.EleMultiModule("h_t")
# making connections
self.input_module.add_input(self.h_t, -1)
f_t.add_input(self.input_module)
state_residual.add_input(self.C_t, -1)
state_residual.add_input(f_t, 0)
i_t.add_input(self.input_module)
CHat_t.add_input(self.input_module)
state_update.add_input(i_t, 0)
state_update.add_input(CHat_t, 0)
self.C_t.add_input(state_update) # C_t = state_redidual + state_update
self.C_t.add_input(state_residual)
o_t.add_input(self.input_module)
tanh_C_t.add_input(self.C_t)
self.h_t.add_input(o_t, 0)
self.h_t.add_input(tanh_C_t, 0)
# set input and output
self.output_module = self.h_t
def define_inner_modules(self, name, is_training, activations, filter_shapes, bias_shapes, ksizes, pool_strides, keep_prob):
self.layers = {}
# first convolutional layer
self.layers["conv0"] = mod.TimeConvolutionalLayerWithBatchNormalizationModule("conv0",
bias_shapes[0][-1], is_training, 0.0, 1.0, 0.5, activations[0], filter_shapes[0], [1,1,1,1], bias_shapes[0])
lateral_filter_shape = filter_shapes[0]
tmp = lateral_filter_shape[2]
lateral_filter_shape[2] = lateral_filter_shape[3]
lateral_filter_shape[3] = tmp
self.layers["lateral0"] = mod.Conv2DModule("lateral0", lateral_filter_shape, [1,1,1,1])
self.layers["lateral0_batchnorm"] = mod.BatchNormalizationModule("lateral0_batchnorm", lateral_filter_shape[-1], is_training, beta_init=0.0, gamma_init=0.1, ema_decay_rate=0.5, moment_axes=[0,1,2], variance_epsilon=1e-3)
# first max-pooling layer
self.layers["pool0"] = mod.MaxPoolingModule("pool0", ksizes[0], pool_strides[0])
# second convolutional layer
self.layers["conv1"] = mod.TimeConvolutionalLayerWithBatchNormalizationModule(name + "conv1",
bias_shapes[1][-1], is_training, 0.0, 1.0, 0.5, activations[1], filter_shapes[1], [1,1,1,1], bias_shapes[1])
lateral_filter_shape = filter_shapes[1]
tmp = lateral_filter_shape[2]
lateral_filter_shape[2] = lateral_filter_shape[3]
lateral_filter_shape[3] = tmp
self.layers["lateral1"] = mod.Conv2DModule("lateral1", lateral_filter_shape, [1,1,1,1])
self.layers["lateral1_batchnorm"] = mod.BatchNormalizationModule("lateral1_batchnorm", lateral_filter_shape[-1], is_training, beta_init=0.0, gamma_init=0.1, ema_decay_rate=0.5, moment_axes=[0,1,2], variance_epsilon=1e-3)
# second max-pooling layer
self.layers["pool1"] = mod.MaxPoolingModule("pool1", ksizes[0], pool_strides[0])
self.layers["flat_pool1"] = mod.FlattenModule("flat_pool1")
# first fully-connected layer
self.layers["fc0"] = mod.FullyConnectedLayerModule("fc0", activations[2], int(np.prod(np.array(bias_shapes[1]) / np.array(pool_strides[1]))), np.prod(bias_shapes[2]))
# dropout
self.layers["dropout0"] = mod.DropoutModule("dropout0", keep_prob)
# second fully-connected layer
self.layers["fc1"] = mod.FullyConnectedLayerModule("fc1", activations[3], np.prod(bias_shapes[2]), np.prod(bias_shapes[3]))
# connections
self.layers["lateral0"].add_input(self.layers["conv0"].preactivation)
self.layers["lateral0_batchnorm"].add_input(self.layers["lateral0"])
self.layers["conv0"].add_input(self.layers["lateral0_batchnorm"], -1)
self.layers["pool0"].add_input(self.layers["conv0"])
self.layers["conv1"].add_input(self.layers["pool0"], 0)
self.layers["lateral1"].add_input(self.layers["conv1"].preactivation)
self.layers["lateral1_batchnorm"].add_input(self.layers["lateral1"])
self.layers["conv1"].add_input(self.layers["lateral1_batchnorm"], -1)
self.layers["pool1"].add_input(self.layers["conv1"])
self.layers["flat_pool1"].add_input(self.layers["pool1"])
self.layers["fc0"].add_input(self.layers["flat_pool1"])
self.layers["dropout0"].add_input(self.layers["fc0"])
self.layers["fc1"].add_input(self.layers["dropout0"])
# set input and output
self.input_module = self.layers["conv0"]
self.output_module = self.layers["fc1"]
def define_inner_modules(self, name, activations, filter_shapes, strides,
bias_shapes, ksizes, pool_strides):
self.layers = {}
# first convolutional layer
self.layers["conv0"] = mod.ConvolutionalLayerModule(
"conv0", activations[0], filter_shapes[0], strides[0],
bias_shapes[0])
# first max-pooling layer
self.layers["pool0"] = mod.MaxPoolingModule("pool0", ksizes[0],
pool_strides[0])
# second convolutional layer
self.layers["conv1"] = mod.ConvolutionalLayerModule(
name + "conv1", activations[1], filter_shapes[1], strides[1],
bias_shapes[1])
# second max-pooling layer
self.layers["pool1"] = mod.MaxPoolingModule("pool1", ksizes[0],
pool_strides[0])
self.layers["flat_pool1"] = mod.FlattenModule("flat_pool1")
# first fully-connected layer
self.layers["fc0"] = mod.FullyConnectedLayerModule(
"fc0", activations[2],
int(np.prod(np.array(bias_shapes[1]) / np.array(pool_strides[1]))),
np.prod(bias_shapes[2]))
# second fully-connected layer
self.layers["fc1"] = mod.FullyConnectedLayerModule(
"fc1", activations[3], np.prod(bias_shapes[2]),
np.prod(bias_shapes[3]))
# connections
self.layers["pool0"].add_input(self.layers["conv0"])
self.layers["conv1"].add_input(self.layers["pool0"])
self.layers["pool1"].add_input(self.layers["conv1"])
self.layers["flat_pool1"].add_input(self.layers["pool1"])
self.layers["fc0"].add_input(self.layers["flat_pool1"])
self.layers["fc1"].add_input(self.layers["fc0"])
# set input and output
self.input_module = self.layers["conv0"]
self.output_module = self.layers["fc1"]
def define_inner_modules(self, name, is_training, activations, conv_filter_shapes, bias_shapes, ksizes, pool_strides, topdown_filter_shapes, topdown_output_shapes, keep_prob, FLAGS):
# create all modules of the network
# -----
self.layers = {}
with tf.name_scope('input_normalization'):
self.layers["inp_norm"] = m.NormalizationModule("inp_norm")
with tf.name_scope('convolutional_layer_0'):
if FLAGS.batchnorm:
self.layers["conv0"] = m.TimeConvolutionalLayerWithBatchNormalizationModule("conv0", bias_shapes[0][-1], is_training, 0.0, 1.0, 0.5, activations[0], conv_filter_shapes[0], [1,1,1,1], bias_shapes[0])
else:
self.layers["conv0"] = m.TimeConvolutionalLayerModule("conv0", activations[0], conv_filter_shapes[0], [1,1,1,1], bias_shapes[0])
with tf.name_scope('lateral_layer_0'):
lateral_filter_shape = conv_filter_shapes[0]
tmp = lateral_filter_shape[2]
lateral_filter_shape[2] = lateral_filter_shape[3]
lateral_filter_shape[3] = tmp
self.layers["lateral0"] = m.Conv2DModule("lateral0", lateral_filter_shape, [1,1,1,1])
self.layers["lateral0_batchnorm"] = m.BatchNormalizationModule("lateral0_batchnorm", lateral_filter_shape[-1], is_training, beta_init=0.0, gamma_init=0.1, ema_decay_rate=0.5, moment_axes=[0,1,2], variance_epsilon=1e-3)
with tf.name_scope('pooling_layer_0'):
self.layers["pool0"] = m.MaxPoolingWithArgmaxModule("pool0", ksizes[0], pool_strides[0])
with tf.name_scope('dropout_layer_0'):
self.layers['dropoutc0'] = m.DropoutModule('dropoutc0', keep_prob=keep_prob)
with tf.name_scope('convolutional_layer_1'):
if FLAGS.batchnorm:
self.layers["conv1"] = m.TimeConvolutionalLayerWithBatchNormalizationModule("conv1", bias_shapes[1][-1], is_training, 0.0, 1.0, 0.5, activations[1], conv_filter_shapes[1], [1,1,1,1], bias_shapes[1])
else:
self.layers["conv1"] = m.TimeConvolutionalLayerModule("conv1", activations[1], conv_filter_shapes[1], [1,1,1,1], bias_shapes[1])
with tf.name_scope('topdown_layer_0'):
self.layers["topdown0"] = m.Conv2DTransposeModule("topdown0", topdown_filter_shapes[0], [1,2,2,1], topdown_output_shapes[0])
self.layers["topdown0_batchnorm"] = m.BatchNormalizationModule("topdown0_batchnorm",topdown_output_shapes[0][-1], is_training, beta_init=0.0, gamma_init=0.1, ema_decay_rate=0.5, moment_axes=[0,1,2], variance_epsilon=1e-3)
with tf.name_scope('lateral_layer_1'):
lateral_filter_shape = conv_filter_shapes[1]
tmp = lateral_filter_shape[2]
lateral_filter_shape[2] = lateral_filter_shape[3]
lateral_filter_shape[3] = tmp
self.layers["lateral1"] = m.Conv2DModule("lateral1", lateral_filter_shape, [1,1,1,1])
self.layers["lateral1_batchnorm"] = m.BatchNormalizationModule("lateral1_batchnorm", lateral_filter_shape[-1], is_training, beta_init=0.0, gamma_init=0.1, ema_decay_rate=0.5, moment_axes=[0,1,2], variance_epsilon=1e-3)
with tf.name_scope('pooling_layer_1'):
self.layers["pool1"] = m.MaxPoolingWithArgmaxModule("pool1", ksizes[0], pool_strides[1])
self.layers["flatpool1"] = m.FlattenModule("flatpool1")
with tf.name_scope('dropout_layer_1'):
self.layers['dropoutc1'] = m.DropoutModule('dropoutc1', keep_prob=keep_prob)
with tf.name_scope('fully_connected_layer_0'):
if FLAGS.batchnorm:
self.layers["fc0"] = m.FullyConnectedLayerWithBatchNormalizationModule("fc0", bias_shapes[-1][-1], is_training, 0.0, 1.0, 0.5, activations[2], int(np.prod(np.array(bias_shapes[1]) / np.array(pool_strides[1]))), np.prod(bias_shapes[2]))
else:
self.layers["fc0"] = m.FullyConnectedLayerModule("fc0", activations[2], int(np.prod(np.array(bias_shapes[1]) / np.array(pool_strides[1]))), np.prod(bias_shapes[2]))
with tf.name_scope('unpooling_layer_1'):
self.layers["unpool1"] = m.UnpoolingModule("unpool1", ksizes[0], pool_strides[0])
with tf.name_scope('unconvolution_layer_1'):
self.layers["unconv1"] = m.UnConvolutionModule("unconv1", conv_filter_shapes[1],[1,1,1,1], topdown_output_shapes[0][:1]+ bias_shapes[1][1:])
with tf.name_scope('unpooling_layer_0'):
self.layers["unpool0"] = m.UnpoolingModule("unpool0", ksizes[0], pool_strides[0])
with tf.name_scope('unconvolution_layer_0'):
self.layers["unconv0"] = m.UnConvolutionModule("unconv0", conv_filter_shapes[0],[1,1,1,1],topdown_output_shapes[0])
# connect all modules of the network in a meaningful way
# -----
with tf.name_scope('wiring_of_modules'):
self.layers["conv0"].add_input(self.layers["inp_norm"], 0)
self.layers["pool0"].add_input(self.layers["conv0"])
self.layers["dropoutc0"].add_input(self.layers["pool0"])
self.layers["conv1"].add_input(self.layers["dropoutc0"], 0)
self.layers["pool1"].add_input(self.layers["conv1"])
self.layers["dropoutc1"].add_input(self.layers["pool1"])
self.layers["flatpool1"].add_input(self.layers["dropoutc1"])
self.layers["fc0"].add_input(self.layers["flatpool1"])
#try out unpooling
self.layers["unpool1"].add_input(self.layers["pool1"])
self.layers["unconv1"].add_input(self.layers["unpool1"])
self.layers["unconv1"].add_input(self.layers["conv1"])
self.layers["unpool0"].add_input(self.layers["unconv1"])
self.layers["unpool0"].add_input(self.layers["pool0"])
self.layers["unconv0"].add_input(self.layers["unpool0"])
self.layers["unconv0"].add_input(self.layers["conv0"])
if "L" in FLAGS.architecture:
if FLAGS.batchnorm:
self.layers["lateral0"].add_input(self.layers["conv0"].preactivation)
self.layers["lateral0_batchnorm"].add_input(self.layers["lateral0"])
self.layers["conv0"].add_input(self.layers["lateral0_batchnorm"], -1)
self.layers["lateral1"].add_input(self.layers["conv1"].preactivation)
self.layers["lateral1_batchnorm"].add_input(self.layers["lateral1"])
self.layers["conv1"].add_input(self.layers["lateral1_batchnorm"], -1)
else:
self.layers["lateral0"].add_input(self.layers["conv0"].preactivation)
self.layers["conv0"].add_input(self.layers["lateral0"], -1)
self.layers["lateral1"].add_input(self.layers["conv1"].preactivation)
self.layers["conv1"].add_input(self.layers["lateral1"], -1)
if "T" in FLAGS.architecture:
if FLAGS.batchnorm:
self.layers["topdown0_batchnorm"].add_input(self.layers["topdown0"])
self.layers["conv0"].add_input(self.layers["topdown0_batchnorm"], -1)
self.layers["topdown0"].add_input(self.layers["conv1"].preactivation)
else:
self.layers["conv0"].add_input(self.layers["topdown0"], -1)
self.layers["topdown0"].add_input(self.layers["conv1"].preactivation)
with tf.name_scope('input_output'):
self.input_module = self.layers["inp_norm"]
self.output_module = self.layers["fc0"]
trainY = reformdata_train[WINDOW+delay:, 0:s_train.shape[1]]
testX = reformdata_test[:-WINDOW-delay]
testY = reformdata_test[WINDOW+delay:, 0:s_test.shape[1]]
CELL_SIZE = 300
TIME_DEPTH = 5
BATCH_SIZE = 1
NFFT = 128
in_size = (NFFT + 1) * WINDOW
out_size = NFFT + 1
inp = mod.ConstantPlaceholderModule("input", shape=(BATCH_SIZE, in_size))
target = mod.ConstantPlaceholderModule("target", shape=(BATCH_SIZE, out_size))
cell = lstm.LSTM_Cell("lstm_cell", in_size, CELL_SIZE)
out_prediction = mod.FullyConnectedLayerModule("out_prediction", tf.identity, CELL_SIZE, out_size)
err = mod.ErrorModule("mse", mean_squared_error)
opt = mod.OptimizerModule("adam", tf.train.AdamOptimizer())
# Connect input
cell.add_input(inp)
out_prediction.add_input(cell)
err.add_input(target)
err.add_input(out_prediction)
opt.add_input(err)
opt.create_output(TIME_DEPTH)
out_prediction.create_output(1)
myplot = Plot()
train_length = trainX.shape[0] #2000#