def __init__(self):
super(Controller, self).__init__()
self.tanh = nn.Tanh
self.sigmoid = nn.Sigmoid
# I dont need to exactly clone this, I just want the general idea
# If I can use stuff like QRNN's it might be more efficient anyways
self.qrnn = QRNN(2*FLAGS.LSTM_SIZE, 4*FLAGS.LSTM_SIZE, dropout=.2)
self.g_emb = QRNN(1, FLAGS.LSTM_SIZE, dropout=.2)
# going to go kind of half way and make a bigger LSTM that is reused
for _ in range(FLAGS.NUM_BRANCHES):
self.add_module("start", QRNN(
FLAGS.OUT_FILTERS, FLAGS.LSTM_SIZE, dropout=.2))
self.add_module("count", QRNN(
FLAGS.OUT_FILTERS, FLAGS.LSTM_SIZE, dropout=.2))
self.start = Lookup(self, "start")
self.count = Lookup(self, "count")
self.start_critic = QRNN(FLAGS.LSTM_SIZE, 1, dropout=.2)
self.count_critic = QRNN(FLAGS.LSTM_SIZE, 1, dropout=.2)
self.w_attn_1 = QRNN(FLAGS.LSTM_SIZE, FLAGS.LSTM_SIZE, dropout=.2)
self.w_attn_2 = QRNN(FLAGS.LSTM_SIZE, FLAGS.LSTM_SIZE, dropout=.2)
self.v_attn = QRNN(FLAGS.LSTM_SIZE, 1, dropout=.2)
def qrnn_forward(self, X, size, n_class, batch_size, conv_size):
in_size = int(X.get_shape()[2])
qrnn = QRNN(in_size=in_size, size=size, conv_size=conv_size)
hidden = qrnn.forward(X)
with tf.name_scope("QRNN-Classifier"):
W = tf.Variable(tf.random_normal([size, n_class]), name="W")
b = tf.Variable(tf.random_normal([n_class]), name="b")
output = tf.add(tf.matmul(hidden, W), b)
return output
def qrnn_forward(self, X, size, n_class, batch_size, conv_size):
in_size = int(X.get_shape()[2])
qrnn = QRNN(in_size=in_size, size=size, conv_size=conv_size)
hidden = qrnn.forward(X)
with tf.name_scope("QRNN-Classifier"):
W = tf.Variable(tf.random_normal([size, n_class]), name="W")
b = tf.Variable(tf.random_normal([n_class]), name="b")
output = tf.add(tf.matmul(hidden, W), b)
return output
def _build(self,
layers=[4, 4],
hidden=128,
activation='sigmoid',
optimizer='adam',
dropout_rate=0.2):
days = self._config['term']
dimension = len(self._data.columns)
window = 2
input_raw = Input(shape=(days, dimension))
x = input_raw
for k in range(layers[0]):
if k != layers[0] - 1:
s = True
else:
s = False
x = SpatialDropout1D(dropout_rate)(x)
x = QRNN(
units=hidden,
window_size=window,
return_sequences=s,
stride=1,
)(x)
input_wav = Input(shape=(dimension, days))
y = input_wav
for k in range(layers[1]):
if k != layers[1] - 1:
s = True
else:
s = False
y = SpatialDropout1D(dropout_rate)(y)
y = QRNN(
units=hidden,
window_size=window,
return_sequences=s,
stride=1,
)(y)
merged = Concatenate()([x, y])
label = Dense(units=dimension)(merged)
output = Activation(activation)(label)
model = Model(inputs=[input_raw, input_wav], outputs=output)
base = model
if self._gpus > 1:
model = multi_gpu_model(model, gpus=self._gpus)
model.compile(loss='mse', optimizer=optimizer, metrics=['accuracy'])
return model, base
def create_qrnn_model(l_seq):
input_layer = Input(shape=(l_seq, 1))
qrnn_output_layer = QRNN(64, window_size=60, dropout=0)(input_layer)
prediction_result = Dense(1)(qrnn_output_layer)
model = Model(input=input_layer, output=prediction_result)
model.compile(loss="mean_squared_error", optimizer="adam")
return model
def test_qrnn_convolution(self):
batch_size = 100
sentence_length = 5
word_size = 10
size = 5
data = self.create_test_data(batch_size, sentence_length, word_size)
with tf.Graph().as_default() as q_conv:
qrnn = QRNN(in_size=word_size, size=size, conv_size=3)
X = tf.placeholder(tf.float32, [batch_size, sentence_length, word_size])
forward_graph = qrnn.forward(X)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
hidden = sess.run(forward_graph, feed_dict={X: data})
self.assertEqual((batch_size, size), hidden.shape)
def test_qrnn_convolution(self):
batch_size = 100
sentence_length = 5
word_size = 10
size = 5
data = self.create_test_data(batch_size, sentence_length, word_size)
with tf.Graph().as_default() as q_conv:
qrnn = QRNN(in_size=word_size, size=size, conv_size=3)
X = tf.placeholder(tf.float32,
[batch_size, sentence_length, word_size])
forward_graph = qrnn.forward(X)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
hidden = sess.run(forward_graph, feed_dict={X: data})
self.assertEqual((batch_size, size), hidden.shape)
def __init__(self, vocab_size, ndim_embedding, num_layers, ndim_h, kernel_size=4, pooling="fo", zoneout=0, dropout=0, wgain=1, densely_connected=True):
super(Model, self).__init__(
embed=EmbedID(vocab_size, ndim_embedding, ignore_label=BLANK),
dense=ConvolutionND(1, ndim_h, vocab_size, ksize=1, stride=1, pad=0)
)
assert num_layers > 0
self.vocab_size = vocab_size
self.ndim_embedding = ndim_embedding
self.num_layers = num_layers
self.ndim_h = ndim_h
self.kernel_size = kernel_size
self.using_dropout = True if dropout > 0 else False
self.dropout = dropout
self.densely_connected = densely_connected
self.add_link("qrnn0", QRNN(ndim_embedding, ndim_h, kernel_size=kernel_size, pooling=pooling, zoneout=zoneout, wgain=wgain))
for i in xrange(1, num_layers):
self.add_link("qrnn{}".format(i), QRNN(ndim_h, ndim_h, kernel_size=kernel_size, pooling=pooling, zoneout=zoneout, wgain=wgain))
parser.add_argument("--sequence-length", type=int, default=512)
parser.add_argument("--batch-size", type=int, default=8)
parser.add_argument("--loops", type=int, default=100)
args = parser.parse_args()
hidden_size = args.hidden_size
sequence_length = args.sequence_length
batch_size = args.batch_size
loops = args.loops
print("Benchmark inference speed")
print(f" Hidden Size: {hidden_size}")
print(f" Sequence Length: {sequence_length}")
print(f" Batch Size: {batch_size}")
qrnn = QRNN(hidden_size, kernel_size=2)
lstm = tf.keras.layers.LSTM(hidden_size)
@tf.function
def qrnn_call(input_tensor):
return qrnn(input_tensor)
@tf.function
def lstm_call(input_tensor):
return lstm(input_tensor)
# draw graph
input = tf.keras.Input((sequence_length, hidden_size))
maxlen = (
80
) # cut texts after this number of words (among top max_features most common words)
batch_size = 32
np.random.seed(1337) # for reproducibility
print("Build model...")
model = Sequential()
model.add(Embedding(max_features, 128))
model.add(SpatialDropout1D(0.2))
model.add(
QRNN(
128,
window_size=3,
dropout=0.2,
kernel_regularizer=l2(1e-4),
bias_regularizer=l2(1e-4),
kernel_constraint=maxnorm(10),
bias_constraint=maxnorm(10),
))
model.add(Dense(1))
model.add(Activation("sigmoid"))
# try using different optimizers and different optimizer configs
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
print("Loading data...")
(X_train, y_train), (X_test, y_test) = imdb.load_data(num_words=max_features)
print(len(X_train), "train sequences")
X_ = X[:len(t_lin) - 500 - 150] #学習用
Y_ = Y[:len(t_lin) - 500 - 150] #学習用
N_train = int(len(data) * 0.7)
N_validation = len(data) - N_train
X_train, X_validation, Y_train, Y_validation = \
train_test_split(X_, Y_, test_size=N_validation, shuffle=False)
#予測対象の表示
plt.plot(x_lin[len(x_lin) - 500:len(x_lin)])
plt.show()
#学習モデルの作成
ws = 128 #window_size
input_layer = Input(shape=(maxlen, 1))
qrnn_output_layer = QRNN(64, window_size=ws, dropout=0)(input_layer)
prediction_result = Dense(1)(qrnn_output_layer)
model = Model(inputs=input_layer, outputs=prediction_result)
model.compile(loss="mean_squared_error", optimizer="adam")
pat = 1
early_stopping = EarlyStopping(monitor='val_loss', patience=pat, verbose=1)
csv = CSVLogger('./TrainingWS_' + str(ws) + 'patience_' + str(pat) + '.csv')
fit = model.fit(X_train,
Y_train,
batch_size=512,
epochs=10000,
validation_data=(X_validation, Y_validation),
callbacks=[early_stopping, csv])
#予測
def compile(self,
input_dim=161,
recur_layers=5,
nodes=1000,
conv_context=5,
conv_border_mode='valid',
conv_stride=2,
activation='relu',
lirelu_alpha=.3,
dropout=False,
initialization='glorot_uniform',
batch_norm=True,
stateful=False,
mb_size=None):
logger.info("Building gru model")
assert self.model is None
leaky_relu = False
if activation == 'lirelu':
activation = 'linear'
leaky_relu = True
if stateful:
if mb_size is None:
raise ValueError(
"Stateful QRNN layer needs to know batch size")
acoustic_input = Input(batch_shape=(mb_size, None, input_dim),
name='acoustic_input')
else:
acoustic_input = Input(shape=(None, input_dim),
name='acoustic_input')
# Setup the network
conv_1d = Conv1D(nodes,
conv_context,
name='conv_1d',
padding=conv_border_mode,
strides=conv_stride,
kernel_initializer=initialization,
activation=activation)(acoustic_input)
if batch_norm:
output = BatchNormalization(name='bn_conv_1d', mode=2)(conv_1d)
else:
output = conv_1d
if leaky_relu:
output = LeakyReLU(alpha=lirelu_alpha)(output)
if dropout:
output = Dropout(dropout)(output)
for r in range(recur_layers):
output = QRNN(nodes,
name='rnn_{}'.format(r + 1),
kernel_initializer=initialization,
return_sequences=True,
activation=activation)(output)
if batch_norm:
bn_layer = BatchNormalization(name='bn_rnn_{}'.format(r + 1),
mode=2)
output = bn_layer(output)
if leaky_relu:
output = LeakyReLU(alpha=lirelu_alpha)(output)
output_branch = TimeDistributed(Dense(self.output_dim,
name='text_dense',
init=initialization,
activation=for_tf_or_th(
'softmax', 'linear')),
name=self.outputs)
network_output = output_branch(output)
self.model = Model(input=acoustic_input, output=[network_output])
self.branch_outputs = [output_branch]
self.branch_vars[output_branch.name] = self.model.trainable_weights
self.acoustic_input = self.model.inputs[0]
return self.model
print(len(X_test), 'test sequences')
print('Pad sequences (samples x time)')
X_train = sequence.pad_sequences(X_train, maxlen=maxlen)
X_test = sequence.pad_sequences(X_test, maxlen=maxlen)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
print('Build model...')
model = Sequential()
model.add(Embedding(max_features, 128, dropout=0.2))
model.add(
QRNN(128,
window_size=3,
dropout=0.2,
W_regularizer=l2(1e-4),
b_regularizer=l2(1e-4),
W_constraint=maxnorm(10),
b_constraint=maxnorm(10)))
model.add(Dense(1))
model.add(Activation('sigmoid'))
# try using different optimizers and different optimizer configs
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print('Train...')
model.fit(X_train,
y_train,
batch_size=batch_size,
dr = 0.1
digits = datasets.load_digits()
x = digits.data
y = digits.target
y = np_utils.to_categorical(y)
xr, xt, onehot_yr, onehot_yt = train_test_split(x, y, random_state=0)
xr = xr.reshape(-1, 8, 8)
xt = xt.reshape(-1, 8, 8)
model = Sequential()
model.add(
QRNN(hidden_neurons,
batch_input_shape=(None, length_of_sequences, in_neurons),
return_sequences=True))
model.add(Dense(hidden_neurons))
model.add(Dropout(dr))
model.add(QRNN(hidden_neurons, return_sequences=True))
model.add(Dense(hidden_neurons))
model.add(Dropout(dr))
model.add(QRNN(hidden_neurons, return_sequences=False))
model.add(Dense(10, activation='softmax')) #
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
print(model.summary())
model.fit(xr,
onehot_yr,
y_train = np.array([x[1] for x in d[:split]])
y_test = np.array([x[1] for x in d[split:limit]])
X_train = sequence.pad_sequences(X_train, maxlen=MAX_SEQ_LEN)
X_test = sequence.pad_sequences(X_test, maxlen=MAX_SEQ_LEN)
model = Sequential()
model.add(
Embedding(VOCAB_SIZE,
WORD_DIM,
input_length=MAX_SEQ_LEN,
batch_input_shape=(BATCH_SIZE, MAX_SEQ_LEN)))
model.add(
QRNN(128,
window=3,
input_dim=WORD_DIM,
input_length=MAX_SEQ_LEN,
batch_input_shape=(BATCH_SIZE, MAX_SEQ_LEN, WORD_DIM),
ret_sequence=True))
model.add(
QRNN(128,
window=3,
input_dim=WORD_DIM,
input_length=MAX_SEQ_LEN,
batch_input_shape=(BATCH_SIZE, MAX_SEQ_LEN, WORD_DIM),
ret_sequence=True))
model.add(
QRNN(128,
window=2,
input_dim=WORD_DIM,
input_length=MAX_SEQ_LEN,
batch_input_shape=(BATCH_SIZE, MAX_SEQ_LEN, WORD_DIM)))