def two_layer_lstm(prefix, x, layer_size, hidden_size, axis, is_forward):
tparams = collections.OrderedDict()
tparams['%s_Wx1' % prefix] = theano.shared(0.01*np.random.randn(hidden_size*4, layer_size).astype(np.float32))
tparams['%s_Wh1' % prefix] = theano.shared(0.01*np.random.randn(hidden_size*4, hidden_size).astype(np.float32))
tparams['%s_b1' % prefix] = theano.shared(np.zeros(hidden_size*4).astype(np.float32))
tparams['%s_Wx2' % prefix] = theano.shared(0.01*np.random.randn(hidden_size*4, hidden_size).astype(np.float32))
tparams['%s_Wh2' % prefix] = theano.shared(0.01*np.random.randn(hidden_size*4, hidden_size).astype(np.float32))
tparams['%s_b2' % prefix] = theano.shared(np.zeros(hidden_size*4).astype(np.float32))
tparams['%s_Wx3' % prefix] = theano.shared(0.01*np.random.randn(layer_size, hidden_size).astype(np.float32))
tparams['%s_b3' % prefix] = theano.shared(np.zeros(layer_size).astype(np.float32))
l = x
l = layers.recurrent_layer(
l,
tparams['%s_Wx1' % prefix],
tparams['%s_Wh1' % prefix],
tparams['%s_b1' % prefix],
axis=axis,
is_forward=is_forward,
)
l = layers.recurrent_layer(
l,
tparams['%s_Wx2' % prefix],
tparams['%s_Wh2' % prefix],
tparams['%s_b2' % prefix],
axis=axis,
is_forward=is_forward,
)
l = layers.linear_layer(
l,
tparams['%s_Wx3' % prefix],
tparams['%s_b3' % prefix],
)
return l, tparams
def get_nn_model(self):
params = {
'batch_size': 20,
'learning_rate': 0.01,
'epochs': 30,
'num_classes': 10,
'dropout_rate': 0.1
}
Model = model.model()
Model.add(layers.flatten_layer())
Model.add(layers.linear_layer(20*20, 200))
Model.add(layers.dropout_layer(r = params['dropout_rate']))
Model.add(layers.linear_layer(200, 10))
Model.set_loss(layers.softmax_cross_entropy())
Model.set_hyper_params(params)
return Model
def __init__(self,
channels_time,
window_sizes,
kernel_sizes_time,
num_classes,
attention,
mode="flat",
num_heads=2,
dropout=0.8,
lr=0.001,
betas=(0.9, 0.999),
eps=1e-8):
super(MultiChannelMultiTime, self).__init__()
self.num_classes = num_classes
self.window_sizes = window_sizes
self.num_times_scales = len(window_sizes)
self.attention = attention
self.mode = mode
self.lr = lr
self.betas = betas
self.eps = eps
self.train_acc = pl.metrics.Accuracy()
self.valid_acc = pl.metrics.Accuracy()
self.test_acc = pl.metrics.Accuracy()
self.conv = []
for channels, kernels in zip(channels_time, kernel_sizes_time):
conv = cnn1d(channels, kernels)
if torch.cuda.is_available():
self.conv.append(conv.cuda())
else:
self.conv.append(conv)
if self.attention:
final_sequence_length = get_final_length(window_sizes[0],
kernel_sizes_time[0])
self.attention_layers = nn.MultiheadAttention(
final_sequence_length, num_heads)
if not attention or mode == "flat":
self.num_final_layers = sum([
channels[-1] * get_final_length(window_size, kernels)
for channels, window_size, kernels in zip(
channels_time, window_sizes, kernel_sizes_time)
])
else:
self.num_final_layers = get_final_length(window_sizes[0],
kernel_sizes_time[0])
self.classifier = nn.Sequential(*linear_layer(
self.num_final_layers, num_classes, drop_out=dropout))
self.num_paramaters = sum(p.numel() for p in self.parameters())
def two_layer_lstm(prefix, x, layer_size, hidden_size, axis, is_forward):
tparams = collections.OrderedDict()
tparams['%s_Wx1' % prefix] = theano.shared(
0.01 * np.random.randn(hidden_size * 4, layer_size).astype(np.float32))
tparams['%s_Wh1' % prefix] = theano.shared(
0.01 *
np.random.randn(hidden_size * 4, hidden_size).astype(np.float32))
tparams['%s_b1' % prefix] = theano.shared(
np.zeros(hidden_size * 4).astype(np.float32))
tparams['%s_Wx2' % prefix] = theano.shared(
0.01 *
np.random.randn(hidden_size * 4, hidden_size).astype(np.float32))
tparams['%s_Wh2' % prefix] = theano.shared(
0.01 *
np.random.randn(hidden_size * 4, hidden_size).astype(np.float32))
tparams['%s_b2' % prefix] = theano.shared(
np.zeros(hidden_size * 4).astype(np.float32))
tparams['%s_Wx3' % prefix] = theano.shared(
0.01 * np.random.randn(layer_size, hidden_size).astype(np.float32))
tparams['%s_b3' % prefix] = theano.shared(
np.zeros(layer_size).astype(np.float32))
l = x
l = layers.recurrent_layer(
l,
tparams['%s_Wx1' % prefix],
tparams['%s_Wh1' % prefix],
tparams['%s_b1' % prefix],
axis=axis,
is_forward=is_forward,
)
l = layers.recurrent_layer(
l,
tparams['%s_Wx2' % prefix],
tparams['%s_Wh2' % prefix],
tparams['%s_b2' % prefix],
axis=axis,
is_forward=is_forward,
)
l = layers.linear_layer(
l,
tparams['%s_Wx3' % prefix],
tparams['%s_b3' % prefix],
)
return l, tparams
def get_model(self):
params = {
'batch_size': 20,
'learning_rate': 0.01,
'epochs': 20,
'num_classes': 10,
'dropout_rate': 0.1
}
Model = model.model()
Model.add(layers.convolution_layer(field=8, padding=0, stride=1, depth=1, filters=5))
Model.add(layers.relu_layer())
Model.add(layers.flatten_layer())
Model.add(layers.dropout_layer(r = params['dropout_rate']))
Model.add(layers.linear_layer(13*13*5, 10))
Model.set_loss(layers.softmax_cross_entropy())
Model.set_hyper_params(params)
return Model
def __init__(self,
channels,
kernel_sizes,
sequence_length,
num_classes,
attention=False,
mode="flat",
num_heads=2,
dropout=0.8,
lr=0.001,
betas=(0.9, 0.999),
eps=1e-8):
super(MultiChannelBase, self).__init__()
self.num_classes = num_classes
self.attention = attention
self.mode = mode
self.lr = lr
self.betas = betas
self.eps = eps
self.train_acc = pl.metrics.Accuracy()
self.valid_acc = pl.metrics.Accuracy()
self.test_acc = pl.metrics.Accuracy()
self.conv = cnn1d(channels, kernel_sizes)
if self.attention:
final_sequence_length = get_final_length(sequence_length,
kernel_sizes)
self.attention_layers = nn.MultiheadAttention(
final_sequence_length, num_heads)
# else:
self.num_final_layers = get_final_length(sequence_length, kernel_sizes)
if not attention or mode == "flat":
self.num_final_layers *= channels[-1]
self.classifier = nn.Sequential(*linear_layer(
self.num_final_layers, num_classes, drop_out=dropout))
self.num_paramaters = sum(p.numel() for p in self.parameters())
def gen_decoder(name, z, n_hidden, n_output, keep_prob, reuse=False,\
output_zero_one=True,output_scalar=False,output_softmax=False):
with tf.variable_scope("%s_bernoulli_decoder" % name, reuse=reuse):
if type(n_hidden) is int:
n_hidden = [z.get_shape()[1], n_hidden]
elif type(n_hidden) is list:
n_hidden = [z.get_shape()[1]] + n_hidden
else:
raise ("type of n_hidden needs to be int or list")
num_layers = len(n_hidden)
#h = layers.tanh_layer(x, x.get_shape()[0], n_hidden[0],
# "enc_l0", reuse, True, keep_prob)
h = z
for i in range(num_layers - 1):
h = layers.tanh_layer(h, n_hidden[i], n_hidden[i + 1],
"dec_l%i" % i, reuse, True, keep_prob)
if output_zero_one:
h = layers.sigmoid_layer(h, n_hidden[-1], n_output,
"dec_l%i" % (num_layers - 1), reuse, True,
keep_prob)
elif output_scalar:
h = layers.linear_layer(h, n_hidden[-1], n_output,
"dec_l%i" % (num_layers - 1), reuse, True,
keep_prob)
elif output_softmax:
h = layers.softmax_layer(h, n_hidden[-1], n_output,
"dec_l%i" % (num_layers - 1), reuse, True,
keep_prob)
else:
print("Not reachable")
raise ("wtf")
return h