def __init__(self, n_head, d_model, d_k=None, d_v=None, dropout_rate=0.1):
super(MultiHeadAttention, self).__init__()
self.n_head = n_head
self.d_model = d_model
self.d_k = d_k if d_k else int(d_model / n_head)
self.d_v = d_v if d_v else int(d_model / n_head)
d_k_w_init = normal(mean=0.,
stddev=np.sqrt(2.0 / (d_model + self.d_k)))
d_v_w_init = normal(mean=0.,
stddev=np.sqrt(2.0 / (d_model + self.d_v)))
self.w_qs = Dense(self.d_k * n_head,
use_bias=False,
kernel_initializer=d_k_w_init)
self.w_ks = Dense(self.d_k * n_head,
use_bias=False,
kernel_initializer=d_k_w_init)
self.w_vs = Dense(self.d_v * n_head,
use_bias=False,
kernel_initializer=d_v_w_init)
self.attention = ScaledDotProductAttention(
temperature=np.power(self.d_k, 0.5))
self.layer_norm = LayerNorm(d_model)
self.fc = Dense(self.d_model,
use_bias=False,
kernel_initializer='glorot_normal')
self.dropout = Dropout(dropout_rate)
self.shape_q, self.shape_k, self.shape_v = None, None, None
def model_function(input_shape, action_size, data_format, learning_rate):
model = Sequential()
model.add(
Convolution2D(16, (3, 3),
strides=(3, 3),
name='first',
data_format=data_format,
padding='same',
activation='relu',
kernel_initializer=normal(stddev=0.01),
input_shape=input_shape))
model.add(
Convolution2D(32, (3, 3),
strides=(3, 3),
name='second',
data_format=data_format,
padding='same',
activation='relu',
kernel_initializer=normal(stddev=0.01)))
model.add(Flatten(name='flat'))
model.add(
Dense(64,
name='dense',
activation='relu',
kernel_initializer=normal(stddev=0.01)))
model.add(
Dense(action_size,
name='out',
activation='linear',
kernel_initializer=normal(stddev=0.01)))
model.compile(optimizer=optimizers.Adam(learning_rate), loss=cl.huber_loss)
return model
def generator(d=128, image_shape=[64, 64, 3]):
conv_options = {
'kernel_initializer': initializers.normal(mean=0.0, stddev=0.02),
}
batchnor_options = {
'gamma_initializer': initializers.normal(mean=0.1, stddev=0.02),
'beta_initializer': initializers.constant(0),
'momentum': 0.9
}
inputs = layers.Input([
100,
])
s_h, s_w = image_shape[0], image_shape[1]
s_h2, s_w2 = conv_out_size_same(s_h, 2), conv_out_size_same(s_w, 2)
s_h4, s_w4 = conv_out_size_same(s_h2, 2), conv_out_size_same(s_w2, 2)
s_h8, s_w8 = conv_out_size_same(s_h4, 2), conv_out_size_same(s_w4, 2)
s_h16, s_w16 = conv_out_size_same(s_h8, 2), conv_out_size_same(s_w8, 2)
x = layers.Dense(s_h16 * s_w16 * d * 8, **conv_options)(inputs)
x = layers.Reshape([s_h16, s_w16, d * 8])(x)
x = layers.BatchNormalization(**batchnor_options)(x)
x = layers.Activation("relu")(x)
x = layers.Conv2DTranspose(filters=d * 4,
kernel_size=4,
strides=2,
padding="same",
**conv_options)(x)
x = layers.BatchNormalization(**batchnor_options)(x)
x = layers.Activation("relu")(x)
x = layers.Conv2DTranspose(filters=d * 2,
kernel_size=4,
strides=2,
padding="same",
**conv_options)(x)
x = layers.BatchNormalization(**batchnor_options)(x)
x = layers.Activation("relu")(x)
x = layers.Conv2DTranspose(filters=d,
kernel_size=4,
strides=2,
padding="same",
**conv_options)(x)
x = layers.BatchNormalization(**batchnor_options)(x)
x = layers.Activation("relu")(x)
x = layers.Conv2DTranspose(filters=3,
kernel_size=4,
strides=2,
padding="same",
**conv_options)(x)
x = layers.Activation("tanh")(x)
model = Model(inputs, x)
return model
def create_actor_network(self, state_size, action_dim):
print("Now we build the model")
S = Input(shape=[state_size])
h0 = Dense(HIDDEN1_UNITS, activation='relu')(S)
h1 = Dense(HIDDEN2_UNITS, activation='relu')(h0)
Steering = Dense(1, activation='tanh', kernel_initializer=normal(stddev=1e-4))(h1)
Acceleration = Dense(1, activation='sigmoid', kernel_initializer=normal(stddev=1e-4))(h1)
Brake = Dense(1, activation='sigmoid', kernel_initializer=normal(stddev=1e-4))(h1)
V = Concatenate()([Steering, Acceleration, Brake])
model = Model(inputs=S, outputs=V)
return model, model.trainable_weights, S
def discriminator(d=128, image_shape=[64, 64, 3]):
conv_options = {
'kernel_initializer': initializers.normal(mean=0., stddev=0.02),
}
batchnor_options = {
'gamma_initializer': initializers.normal(mean=0.1, stddev=0.02),
'beta_initializer': initializers.constant(0),
'momentum': 0.9
}
inputs = layers.Input(image_shape)
x = layers.Conv2D(filters=d,
kernel_size=4,
strides=2,
padding="same",
**conv_options)(inputs)
x = layers.LeakyReLU(0.2)(x)
x = layers.Conv2D(filters=2 * d,
kernel_size=4,
strides=2,
padding="same",
**conv_options)(x)
x = layers.BatchNormalization(**batchnor_options)(x)
x = layers.LeakyReLU(0.2)(x)
x = layers.Conv2D(filters=4 * d,
kernel_size=4,
strides=2,
padding="same",
**conv_options)(x)
x = layers.BatchNormalization(**batchnor_options)(x)
x = layers.LeakyReLU(0.2)(x)
x = layers.Conv2D(filters=8 * d,
kernel_size=4,
strides=2,
padding="same",
**conv_options)(x)
x = layers.BatchNormalization(**batchnor_options)(x)
x = layers.LeakyReLU(0.2)(x)
x = layers.Flatten()(x)
x = layers.Dropout(0.3)(x)
x = layers.Dense(1, **conv_options)(x)
x = layers.Activation("sigmoid")(x)
model = Model(inputs, x)
return model
def __init__(self,
input_dim,
output_dim,
embeddings_initializer=initializers.normal(stddev=1.0),
embeddings_regularizer=None,
activity_regularizer=None,
embeddings_constraint=None,
mask_zero=False,
input_length=None,
**kwargs):
if 'input_shape' not in kwargs:
if input_length:
kwargs['input_shape'] = (input_length, )
else:
kwargs['input_shape'] = (None, )
super(EmbeddingNorm, self).__init__(**kwargs)
self.input_dim = input_dim
self.output_dim = output_dim
self.embeddings_initializer = initializers.get(embeddings_initializer)
self.embeddings_regularizer = regularizers.get(embeddings_regularizer)
self.activity_regularizer = regularizers.get(activity_regularizer)
self.embeddings_constraint = constraints.get(embeddings_constraint)
self.mask_zero = mask_zero
self.supports_masking = mask_zero
self.input_length = input_length
def Layer_deNovo(filters, kernel_size, strides=1, padding='valid', activation='sigmoid', lambda_pos =3e-3,
lambda_l1=3e-3, lambda_filter = 1e-8, name='denovo'):
return Conv1D( filters, kernel_size, strides=strides, padding=padding, activation=activation,
kernel_initializer=normal(0,0.5), bias_initializer='zeros',
kernel_regularizer=lambda w: total_reg(w, lambda_filter, lambda_l1, lambda_pos, kernel_size),
kernel_constraint=InfoConstraint(), bias_constraint=NegativeConstraint(),
name=name)
def loadModel(target, source, sourceIndex, predLabel, path):
mmdNetLayerSizes = [25, 25]
l2_penalty = 1e-2
init = lambda shape, name: initializers.normal(
shape, scale=.1e-4, name=name)
space_dim = target.X.shape[1]
calibInput = Input(shape=(space_dim, ))
block1_bn1 = BatchNormalization()(calibInput)
block1_a1 = Activation('relu')(block1_bn1)
block1_w1 = Dense(mmdNetLayerSizes[0],
activation='linear',
W_regularizer=l2(l2_penalty),
init=init)(block1_a1)
block1_bn2 = BatchNormalization()(block1_w1)
block1_a2 = Activation('relu')(block1_bn2)
block1_w2 = Dense(space_dim,
activation='linear',
W_regularizer=l2(l2_penalty),
init=init)(block1_a2)
block1_output = merge([block1_w2, calibInput], mode='sum')
block2_bn1 = BatchNormalization()(block1_output)
block2_a1 = Activation('relu')(block2_bn1)
block2_w1 = Dense(mmdNetLayerSizes[1],
activation='linear',
W_regularizer=l2(l2_penalty),
init=init)(block2_a1)
block2_bn2 = BatchNormalization()(block2_w1)
block2_a2 = Activation('relu')(block2_bn2)
block2_w2 = Dense(space_dim,
activation='linear',
W_regularizer=l2(l2_penalty),
init=init)(block2_a2)
block2_output = merge([block2_w2, block1_output], mode='sum')
block3_bn1 = BatchNormalization()(block2_output)
block3_a1 = Activation('relu')(block3_bn1)
block3_w1 = Dense(mmdNetLayerSizes[1],
activation='linear',
W_regularizer=l2(l2_penalty),
init=init)(block3_a1)
block3_bn2 = BatchNormalization()(block3_w1)
block3_a2 = Activation('relu')(block3_bn2)
block3_w2 = Dense(space_dim,
activation='linear',
W_regularizer=l2(l2_penalty),
init=init)(block3_a2)
block3_output = merge([block3_w2, block2_output], mode='sum')
calibMMDNet = Model(input=calibInput, output=block3_output)
calibMMDNet.load_weights(
os.path.join(
io.DeepLearningRoot(),
'savemodels/' + path + '/ResNet' + str(sourceIndex) + '.h5'))
return calibMMDNet
def create_actor_network(self, state_size, action_dim):
print("Now we build the model")
S = Input(shape=[state_size])
h0 = Dense(HIDDEN1_UNITS, activation='tanh')(S)
h1 = Dense(HIDDEN2_UNITS, activation='tanh')(h0)
V = Dense(1, activation='tanh', init=lambda shape, name: normal(shape, scale=1e-4, name=name))(h1)
model = Model(input=S, output=V)
adam = Adam(lr=self.LEARNING_RATE)
model.compile(loss='mse', optimizer=adam)
return model, model.trainable_weights, S
def build(self, input_shape):
self._trainable_weights.append(prior_params)
self.kernel_mu = self.add_weight(name='kernel_mu',
shape=(input_shape[1], self.output_dim),
initializer=initializers.normal(stddev=prior_sigma),
trainable=True)
self.bias_mu = self.add_weight(name='bias_mu',
shape=(self.output_dim,),
initializer=initializers.normal(stddev=prior_sigma),
trainable=True)
self.kernel_rho = self.add_weight(name='kernel_rho',
shape=(input_shape[1], self.output_dim),
initializer=initializers.constant(0.0),
trainable=True)
self.bias_rho = self.add_weight(name='bias_rho',
shape=(self.output_dim,),
initializer=initializers.constant(0.0),
trainable=True)
super().build(input_shape)
def create_actor_network(self, state_size, action_dim):
print("Now we build the model")
S = Input(shape=[state_size])
h0 = Dense(HIDDEN1_UNITS, activation='relu')(S)
h1 = Dense(HIDDEN2_UNITS, activation='relu')(h0)
Steering = Dense(
1,
activation='tanh',
init=lambda shape, name: normal(shape, scale=1e-4, name=name))(h1)
Acceleration = Dense(
1,
activation='sigmoid',
init=lambda shape, name: normal(shape, scale=1e-4, name=name))(h1)
Brake = Dense(
1,
activation='sigmoid',
init=lambda shape, name: normal(shape, scale=1e-4, name=name))(h1)
V = merge([Steering, Acceleration, Brake], mode='concat')
model = Model(input=S, output=V)
return model, model.trainable_weights, S
def build(self, input_shape):
self.kernel_mu = self.add_weight(
name='kernel_mu',
shape=(input_shape[1], self.units),
initializer=initializers.normal(stddev=self.init_sigma),
trainable=True)
self.bias_mu = self.add_weight(
name='bias_mu',
shape=(self.units, ),
initializer=initializers.normal(stddev=self.init_sigma),
trainable=True)
self.kernel_rho = self.add_weight(
name='kernel_rho',
shape=(input_shape[1], self.units),
initializer=initializers.constant(0.0),
trainable=True)
self.bias_rho = self.add_weight(name='bias_rho',
shape=(self.units, ),
initializer=initializers.constant(0.0),
trainable=True)
super().build(input_shape)
def constructMMD(target):
mmdNetLayerSizes = [25, 25]
l2_penalty = 1e-2
init = lambda shape, name: initializers.normal(
shape, scale=.1e-4, name=name)
space_dim = target.X.shape[1]
calibInput = Input(shape=(space_dim, ))
block1_bn1 = BatchNormalization()(calibInput)
block1_a1 = Activation('relu')(block1_bn1)
block1_w1 = Dense(mmdNetLayerSizes[0],
activation='linear',
W_regularizer=l2(l2_penalty),
init=init)(block1_a1)
block1_bn2 = BatchNormalization()(block1_w1)
block1_a2 = Activation('relu')(block1_bn2)
block1_w2 = Dense(space_dim,
activation='linear',
W_regularizer=l2(l2_penalty),
init=init)(block1_a2)
block1_output = merge([block1_w2, calibInput], mode='sum')
block2_bn1 = BatchNormalization()(block1_output)
block2_a1 = Activation('relu')(block2_bn1)
block2_w1 = Dense(mmdNetLayerSizes[1],
activation='linear',
W_regularizer=l2(l2_penalty),
init=init)(block2_a1)
block2_bn2 = BatchNormalization()(block2_w1)
block2_a2 = Activation('relu')(block2_bn2)
block2_w2 = Dense(space_dim,
activation='linear',
W_regularizer=l2(l2_penalty),
init=init)(block2_a2)
block2_output = merge([block2_w2, block1_output], mode='sum')
block3_bn1 = BatchNormalization()(block2_output)
block3_a1 = Activation('relu')(block3_bn1)
block3_w1 = Dense(mmdNetLayerSizes[1],
activation='linear',
W_regularizer=l2(l2_penalty),
init=init)(block3_a1)
block3_bn2 = BatchNormalization()(block3_w1)
block3_a2 = Activation('relu')(block3_bn2)
block3_w2 = Dense(space_dim,
activation='linear',
W_regularizer=l2(l2_penalty),
init=init)(block3_a2)
block3_output = merge([block3_w2, block2_output], mode='sum')
calibMMDNet = Model(input=calibInput, output=block3_output)
return calibMMDNet, block3_output
def EmgLstmNet(input_shape,
classes,
n_dropout=0.,
n_l2=0.0005,
n_init='glorot_normal',
lstm_units=[256]):
if n_init == 'glorot_normal':
kernel_init = initializers.glorot_normal(seed=0)
elif n_init == 'glorot_uniform':
kernel_init = initializers.glorot_uniform(seed=0)
elif n_init == 'he_normal':
kernel_init = initializers.he_normal(seed=0)
elif n_init == 'he_uniform':
kernel_init = initializers.he_uniform(seed=0)
elif n_init == 'normal':
kernel_init = initializers.normal(seed=0)
elif n_init == 'uniform':
kernel_init = initializers.uniform(seed=0)
kernel_regl = regularizers.l2(n_l2)
x_input = Input(input_shape)
x = Masking(-10.0)(x_input)
for i in range(len(lstm_units) - 1):
x = LSTM(lstm_units[i],
dropout=n_dropout,
recurrent_dropout=n_dropout,
kernel_regularizer=kernel_regl,
kernel_initializer=kernel_init,
recurrent_regularizer=kernel_regl,
recurrent_initializer=kernel_init,
return_sequences=True,
input_shape=input_shape)(x)
x = LSTM(lstm_units[-1],
dropout=n_dropout,
recurrent_dropout=n_dropout,
kernel_regularizer=kernel_regl,
kernel_initializer=kernel_init,
recurrent_regularizer=kernel_regl,
recurrent_initializer=kernel_init,
return_sequences=False)(x)
y = Dense(classes,
activation='softmax',
kernel_regularizer=kernel_regl,
kernel_initializer=kernel_init)(x)
model = Model(x_input, y)
return model
def train_model(model_name, x_train, y_train):
model = Sequential()
model.add(Conv2D(64, kernel_size=4, activation='relu', input_shape=(27, 27, 3)))
model.add(Conv2D(64, kernel_size=3, activation='relu', padding='same'))
if model_name != 'nopool':
model.add(MaxPooling2D())
model.add(Conv2D(128, kernel_size=3, activation='relu', padding='same'))
model.add(Conv2D(128, kernel_size=3, activation='relu', padding='same'))
if model_name != 'nopool':
model.add(MaxPooling2D())
model.add(Flatten())
model.add(Dense(512, activation='relu', kernel_initializer=normal(0, 0.01), kernel_regularizer=l2(0.0005)))
model.add(Dropout(0.5))
model.add(Dense(512, activation='relu', kernel_initializer=normal(0, 0.01), kernel_regularizer=l2(0.0005)))
model.add(Dropout(0.5))
model.add(Dense(2, activation='sigmoid', kernel_initializer=normal(0, 0.01)))
model.compile(SGD(lr=0.001, momentum=0.9), loss='categorical_crossentropy', metrics=['accuracy'])
#model = load_model('nopool_checkpoint.keras')
model.fit(x_train, y_train, batch_size=256, epochs=19, callbacks=[LearningRateScheduler(schedule, verbose=1), ModelCheckpoint('data/'+model_name+'_checkpoint.keras')])
model.save('data/'+model_name+'.keras')
os.remove('data/'+model_name+'_checkpoint.keras')
def training_data(tmpdir_factory):
import h5py
from keras.layers import Dense
from keras.models import Sequential
from keras.optimizers import SGD
from keras.initializers import glorot_normal, normal
from deepreplay.datasets.parabola import load_data
from deepreplay.callbacks import ReplayData
filename = str(tmpdir_factory.mktemp('data').join('training.h5'))
X, y = load_data(xlim=(-1, 1), n_points=1000, shuffle=True, seed=13)
sgd = SGD(lr=0.05)
glorot_initializer = glorot_normal(seed=42)
normal_initializer = normal(seed=42)
replaydata = ReplayData(X,
y,
filename=filename,
group_name='part1_activation_functions')
np.random.seed(13)
model = Sequential()
model.add(
Dense(input_dim=2,
units=2,
kernel_initializer=glorot_initializer,
activation='sigmoid',
name='hidden'))
model.add(
Dense(units=1,
kernel_initializer=normal_initializer,
activation='sigmoid',
name='output'))
model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['acc'])
model.fit(X, y, epochs=20, batch_size=16, callbacks=[replaydata])
training_data = h5py.File(filename, 'r')
return training_data['part1_activation_functions']
def __init__(self):
self.width = 8
self.number_of_pieces = 4 # normal, king for each colour
self.constant_planes = 2 # colour, total_moves
self.planes_per_board = self.number_of_pieces + self.constant_planes
self.history_len = 4
self.n_actions = 32 * 8
self.game_logic = Draughts()
self.n_kernels = 32
self.shared_layers = 3
self.kernel_size = (3, 3)
self.initialiser = normal(mean=0, stddev=0.01)
self.regularisation = 0.01
self.crossentropy_constant = 1e-8
self.model = self.make_uniform()
self.policy_keys = [
self.index_to_xy(index) for index in range(self.n_actions)
]
def training_data(tmpdir_factory):
import h5py
from keras.layers import Dense
from keras.models import Sequential
from keras.optimizers import SGD
from keras.initializers import glorot_normal, normal
from deepreplay.datasets.parabola import load_data
from deepreplay.callbacks import ReplayData
filename = str(tmpdir_factory.mktemp('data').join('training.h5'))
X, y = load_data(xlim=(-1, 1), n_points=1000, shuffle=True, seed=13)
sgd = SGD(lr=0.05)
glorot_initializer = glorot_normal(seed=42)
normal_initializer = normal(seed=42)
replaydata = ReplayData(X, y, filename=filename, group_name='part1_activation_functions')
model = Sequential()
model.add(Dense(input_dim=2,
units=2,
kernel_initializer=glorot_initializer,
activation='sigmoid',
name='hidden'))
model.add(Dense(units=1,
kernel_initializer=normal_initializer,
activation='sigmoid',
name='output'))
model.compile(loss='binary_crossentropy',
optimizer=sgd,
metrics=['acc'])
model.fit(X, y, epochs=20, batch_size=16, callbacks=[replaydata])
training_data = h5py.File(filename, 'r')
return training_data['part1_activation_functions']
def __init__(
self,
input_dim,
output_dim,
kl_loss_weight,
mask_zero=False,
input_length=None,
embeddings_initializer=initializers.normal(stddev=prior_sigma),
**kwargs):
self.input_dim = input_dim
self.output_dim = output_dim
self.kl_loss_weight = kl_loss_weight
self.embeddings_initializer = embeddings_initializer
self.mask_zero = mask_zero
self.supports_masking = mask_zero
self.input_length = input_length
if 'input_shape' not in kwargs:
if input_length:
kwargs['input_shape'] = (input_length, )
else:
kwargs['input_shape'] = (None, )
super(EmbeddingVariation, self).__init__(**kwargs)
def init_normal(shape, name=None):
return initializers.normal(shape)
def init_normal(shape, name=None):
return initializers.normal(shape)
def __init__(self,
input_dim,
n_hidden_units,
n_hidden_layers,
rho=0.6,
nonlinearity='tanh',
init='gau',
bias_sigma=0.0,
weight_sigma=1.25,
input_layer=None,
flip=False,
output_dim=None):
#if input_layer is not None:
# assert input_layer.output_shape[1] == input_dim
self.input_dim = input_dim
self.n_hidden_units = n_hidden_units
self.n_hidden_layers = n_hidden_layers
self.nonlinearity = nonlinearity
self.bias_sigma = bias_sigma
self.weight_sigma = weight_sigma
self.input_layer = input_layer
if output_dim is None:
output_dim = n_hidden_units
self.output_dim = output_dim
model = Sequential()
get_custom_objects().update({'hard_tanh': Activation(hard_tanh)})
get_custom_objects().update({'erf2': Activation(erf2)})
if input_layer is not None:
model.add(input_layer)
# model.add(Dropout(0.1, input_layer))
#model.add(Activation('tanh'))
for i in range(n_hidden_layers):
nunits = n_hidden_units if i < n_hidden_layers - 1 else output_dim
if flip:
if nonlinearity == 'prelu':
model.add(
LeakyReLU(alpha=0.5,
input_shape=(input_dim, ),
name='a%d' % i))
else:
model.add(
Activation(nonlinearity,
input_shape=(input_dim, ),
name='a%d' % i))
# model.add(Activation(nonlinearity, input_dim=1000, name='a%d'%i))
model.add(Dropout(1 - rho)) # dropout = 1 - rho
if init == 'gau':
model.add(
Dense(nunits,
name='d%d' % i,
kernel_initializer=normal(
mean=0.0,
stddev=weight_sigma * 1.0 /
np.sqrt(n_hidden_units)),
bias_initializer=normal(mean=0.0,
stddev=bias_sigma)))
if init == 'orth':
model.add(
Dense(nunits,
name='d%d' % i,
kernel_initializer=orthogonal(gain=weight_sigma),
bias_initializer=normal(mean=0.0,
stddev=bias_sigma)))
# model.add(Dense(nunits, name='d%d'%i))
else:
model.add(
Dense(nunits, input_shape=(input_dim, ), name='%d' % i))
if i < n_hidden_layers - 1 or self.output_dim == self.n_hidden_units:
model.add(Activation(nonlinearity, name='a%d' % i))
else:
# Theano is optimizing out the nonlinearity if it can which is breaking shit
# Give it something that it won't optimize out.
model.add(
Activation(lambda x: T.minimum(x, 999999.999),
name='a%d' % i))
model.build()
self.model = model
# print(self.hs)
self.weights = model.get_weights()
self.dense_layers = filter(lambda x: x.name.startswith('d'),
model.layers)
self.activ_layers = filter(lambda x: x.name.startswith('a'),
model.layers)
self.hs = [h.output for h in self.dense_layers]
self.ac = [b.output for b in self.activ_layers]
self.f_acts = self.f_jac = self.f_jac_hess = self.f_act = None
vec = K.ones_like(self.model.input)
def initNormal():
return initializers.normal(stddev=0.02)
def build(self, input_shape):
# Initialize the fixed dictionary
d = np.linspace(-self.boundary, self.boundary,
self.D).astype(np.float32).reshape(-1, 1)
if self.conv:
self.dict = self.add_weight(name='dict',
shape=(1, 1, 1, 1, self.D),
initializer='uniform',
trainable=False)
K.set_value(self.dict, d.reshape(1, 1, 1, 1, -1))
else:
self.dict = self.add_weight(name='dict',
shape=(1, 1, self.D),
initializer='uniform',
trainable=False)
K.set_value(self.dict, d.reshape(1, 1, -1))
if self.kernel == 'gaussian':
self.kernel_fcn = self.gaussian_kernel
# Rule of thumb for gamma
interval = (d[1] - d[0])
sigma = 2 * interval # empirically chosen
self.gamma = 0.5 / np.square(sigma)
elif self.kernel == 'softplus':
self.kernel_fcn = self.softplus_kernel
else:
self.kernel_fcn = self.relu_kernel
# Mixing coefficients
if self.conv:
self.alpha = self.add_weight(name='alpha',
shape=(1, 1, 1, self.num_parameters,
self.D),
initializer=normal(0, 0.4),
trainable=True)
else:
self.alpha = self.add_weight(name='alpha',
shape=(1, self.num_parameters,
self.D),
initializer=normal(0, 0.4),
trainable=True)
# Optional initialization with kernel ridge regression
if self.init_fcn is not None:
if self.kernel == 'gaussian':
kernel_matrix = np.exp(-self.gamma * (d - d.T)**2)
elif self.kernel == 'softplus':
kernel_matrix = np.log(np.exp(d - d.T) + 1.0)
else:
raise ValueError(
'Cannot perform kernel ridge regression with ReLU kernel (singular matrix)'
)
alpha_init = np.linalg.solve(kernel_matrix + 1e-5 * np.eye(self.D),
self.init_fcn(d)).reshape(-1)
if self.conv:
K.set_value(
self.alpha,
np.repeat(alpha_init.reshape(1, 1, 1, 1, -1),
self.num_parameters,
axis=3))
else:
K.set_value(
self.alpha,
np.repeat(alpha_init.reshape(1, 1, -1),
self.num_parameters,
axis=1))
super(KAF, self).build(input_shape)
def gaussian_init(shape, name=None, dim_ordering=None):
return initializers.normal(shape,
scale=0.001,
name=name,
dim_ordering=dim_ordering)
def normal(shape, name=None):
if keras.__version__.startswith("1"):
return initializations.normal(shape, scale=0.05, name=name)
else:
return initializers.normal(shape, scale=0.05, name=name)
def calibrate(target, source, sourceIndex, predLabel, path):
mmdNetLayerSizes = [25, 25]
l2_penalty = 1e-2
init = lambda shape, name:initializers.normal(shape,
scale=.1e-4, name=name)
space_dim = target.X.shape[1]
calibInput = Input(shape=(space_dim,))
block1_bn1 = BatchNormalization()(calibInput)
block1_a1 = Activation('relu')(block1_bn1)
block1_w1 = Dense(mmdNetLayerSizes[0], activation='linear',
W_regularizer=l2(l2_penalty), init = init)(block1_a1)
block1_bn2 = BatchNormalization()(block1_w1)
block1_a2 = Activation('relu')(block1_bn2)
block1_w2 = Dense(space_dim, activation='linear',
W_regularizer=l2(l2_penalty), init = init)(block1_a2)
block1_output = merge([block1_w2, calibInput], mode = 'sum')
block2_bn1 = BatchNormalization()(block1_output)
block2_a1 = Activation('relu')(block2_bn1)
block2_w1 = Dense(mmdNetLayerSizes[1], activation='linear',
W_regularizer=l2(l2_penalty), init = init)(block2_a1)
block2_bn2 = BatchNormalization()(block2_w1)
block2_a2 = Activation('relu')(block2_bn2)
block2_w2 = Dense(space_dim, activation='linear',
W_regularizer=l2(l2_penalty), init = init)(block2_a2)
block2_output = merge([block2_w2, block1_output], mode = 'sum')
block3_bn1 = BatchNormalization()(block2_output)
block3_a1 = Activation('relu')(block3_bn1)
block3_w1 = Dense(mmdNetLayerSizes[1], activation='linear',
W_regularizer=l2(l2_penalty), init = init)(block3_a1)
block3_bn2 = BatchNormalization()(block3_w1)
block3_a2 = Activation('relu')(block3_bn2)
block3_w2 = Dense(space_dim, activation='linear',
W_regularizer=l2(l2_penalty), init = init)(block3_a2)
block3_output = merge([block3_w2, block2_output], mode = 'sum')
calibMMDNet = Model(input=calibInput, output=block3_output)
n = target.X.shape[0]
p = np.random.permutation(n)
toTake = p[range(int(.2*n))]
targetXMMD = target.X[toTake]
targetYMMD = target.y[toTake]
targetXMMD = targetXMMD[targetYMMD!=0]
targetYMMD = targetYMMD[targetYMMD!=0]
targetYMMD = np.reshape(targetYMMD, (-1, 1))
n = source.X.shape[0]
p = np.random.permutation(n)
toTake = p[range(int(.2*n))]
sourceXMMD = source.X[toTake]
sourceYMMD = predLabel[toTake]
sourceXMMD = sourceXMMD[sourceYMMD!=0]
sourceYMMD = sourceYMMD[sourceYMMD!=0]
sourceYMMD = np.reshape(sourceYMMD, (-1, 1))
lrate = LearningRateScheduler(step_decay)
optimizer = opt.rmsprop(lr=0.0)
calibMMDNet.compile(optimizer = optimizer, loss = lambda y_true,y_pred:
cf.MMD(block3_output, targetXMMD,
MMDTargetValidation_split = 0.1).KerasCost(y_true,y_pred))
sourceLabels = np.zeros(sourceXMMD.shape[0])
calibMMDNet.fit(sourceXMMD,sourceLabels,nb_epoch=500,
batch_size=1000,validation_split=0.1,verbose=0,
callbacks=[lrate,mn.monitorMMD(sourceXMMD, sourceYMMD, targetXMMD,
targetYMMD, calibMMDNet.predict),
cb.EarlyStopping(monitor='val_loss',patience=20,mode='auto')])
plt.close('all')
calibMMDNet.save_weights(os.path.join(io.DeepLearningRoot(),
'savemodels/' + path + '/ResNet'+ str(sourceIndex)+'.h5'))
calibrateSource = Sample(calibMMDNet.predict(source.X),
source.y)
calibMMDNet = None
return calibrateSource
from deepreplay.plot import compose_animations, compose_plots
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.preprocessing import StandardScaler
# Fetch the data file from the Data Folder at https://archive.ics.uci.edu/ml/datasets/spambase
group_name = 'spam'
df = pd.read_csv('spambase.data', header=None)
X, y = df.iloc[:, :57].values, df.iloc[:, 57].values
X = StandardScaler().fit_transform(X)
he_initializer = he_normal(seed=42)
normal_initializer = normal(seed=42)
replaydata = ReplayData(X, y, filename='spambase_dataset.h5', group_name=group_name)
model = Sequential()
model.add(Dense(input_dim=57,
units=10,
kernel_initializer=he_initializer,
activation='tanh'))
model.add(Dense(units=2,
kernel_initializer=normal_initializer,
activation='linear',
name='hidden'))
model.add(Dense(units=1,
kernel_initializer=normal_initializer,
activation='sigmoid',
def AtzoriNet(input_shape,
classes,
n_pool='average',
n_dropout=0.,
n_l2=0.0005,
n_init='glorot_normal',
batch_norm=False):
""" Creates the Deep Neural Network architecture described in the paper of Manfredo Atzori:
Deep Learning with Convolutional Neural Networks Applied to Electromyography Data: A Resource for the Classification of Movements for Prosthetic Hands
https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5013051/
Arguments:
input_shape -- tuple, dimensions of the input in the form (height, width, channels)
classes -- integer, number of classes to be classified, defines the dimension of the softmax unit
n_pool -- string, pool method to be used {'max', 'average'}
n_dropout -- float, rate of dropping units
n_l2 -- float, ampunt of weight decay regularization
n_init -- string, type of kernel initializer {'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform', 'normal', 'uniform'}
batch_norm -- boolean, whether BatchNormalization is applied to the input
Returns:
model -- keras.models.Model (https://keras.io)
"""
if n_init == 'glorot_normal':
kernel_init = initializers.glorot_normal(seed=0)
elif n_init == 'glorot_uniform':
kernel_init = initializers.glorot_uniform(seed=0)
elif n_init == 'he_normal':
kernel_init = initializers.he_normal(seed=0)
elif n_init == 'he_uniform':
kernel_init = initializers.he_uniform(seed=0)
elif n_init == 'normal':
kernel_init = initializers.normal(seed=0)
elif n_init == 'uniform':
kernel_init = initializers.uniform(seed=0)
# kernel_init = n_init
kernel_regl = regularizers.l2(n_l2)
## Block 0 [Input]
X_input = Input(input_shape, name='b0_input')
X = X_input
if batch_norm:
X = BatchNormalization()(X)
## Block 1 [Pad -> Conv -> ReLU -> Dropout]
X = ZeroPadding2D((0, 4))(X)
X = Conv2D(32, (1, 10),
padding='valid',
kernel_regularizer=kernel_regl,
kernel_initializer=kernel_init,
name='b1_conv2d_32_1x10')(X)
X = Activation('relu', name='b1_relu')(X)
X = Dropout(n_dropout, name='b1_dropout')(X)
## Block 2 [Pad -> Conv -> ReLU -> -> Dropout -> Pool]
X = ZeroPadding2D((1, 1))(X)
X = Conv2D(32, (3, 3),
padding='valid',
kernel_regularizer=kernel_regl,
kernel_initializer=kernel_init,
name='b2_conv2d_32_3x3')(X)
X = Activation('relu', name='b2_relu')(X)
X = Dropout(n_dropout, name='b2_dropout')(X)
if n_pool == 'max':
X = MaxPooling2D((3, 3), strides=(3, 3), name='b2_pool')(X)
else:
X = AveragePooling2D((3, 3), strides=(3, 3), name='b2_pool')(X)
## Block 3 [Pad -> Conv -> ReLU -> Dropout -> Pool]
X = ZeroPadding2D((2, 2))(X)
X = Conv2D(64, (5, 5),
padding='valid',
kernel_regularizer=kernel_regl,
kernel_initializer=kernel_init,
name='b3_conv2d_64_5x5')(X)
X = Activation('relu', name='b3_relu')(X)
X = Dropout(n_dropout, name='b3_dropout')(X)
if n_pool == 'max':
X = MaxPooling2D((3, 3), strides=(3, 3), name='b3_pool')(X)
else:
X = AveragePooling2D((3, 3), strides=(3, 3), name='b3_pool')(X)
## Block 4 [Pad -> Conv -> ReLU -> Dropout]
X = ZeroPadding2D((2, 0))(X)
X = Conv2D(64, (5, 1),
padding='valid',
kernel_regularizer=kernel_regl,
kernel_initializer=kernel_init,
name='b4_conv2d_64_5x1')(X)
X = Activation('relu', name='b4_relu')(X)
X = Dropout(n_dropout, name='b4_dropout')(X)
## Block 5 [Pad -> Conv -> Softmax]
X = Conv2D(classes, (1, 1),
padding='same',
kernel_regularizer=kernel_regl,
kernel_initializer=kernel_init,
name='b5_conv2d_{}_1x1'.format(classes))(X)
X = Activation('softmax', name='b5_soft')(X)
X = Reshape((-1, ), name='b5_reshape')(X)
model = Model(inputs=X_input, outputs=X, name='AtzoriNet')
return model
def init_normal(shape, name=None):
return initializers.normal(shape, scale=0.01, name=name)
X_train = X_train.reshape(60000, 784)
# Function for initializing network weights
def initNormal():
return initializers.normal(stddev=0.02)
# Optimizer
adam = Adam(lr=0.0002, beta_1=0.5)
generator = Sequential()
generator.add(
Dense(256,
input_dim=randomDim,
kernel_initializer=initializers.normal(stddev=0.02)))
generator.add(LeakyReLU(0.2))
generator.add(Dense(512))
generator.add(LeakyReLU(0.2))
generator.add(Dense(1024))
generator.add(LeakyReLU(0.2))
generator.add(Dense(784, activation='tanh'))
generator.compile(loss='binary_crossentropy', optimizer=adam)
discriminator = Sequential()
discriminator.add(
Dense(1024,
input_dim=784,
kernel_initializer=initializers.normal(stddev=0.02)))
discriminator.add(LeakyReLU(0.2))
discriminator.add(Dropout(0.3))
def init_normal(shape):
return initializers.normal(shape, initializers.VarianceScaling(scale=0.01))
from deepreplay.callbacks import ReplayData
from deepreplay.replay import Replay
from deepreplay.plot import compose_animations, compose_plots
from sklearn.datasets import make_circles
import matplotlib.pyplot as plt
group_name = 'circles'
X, y = make_circles(n_samples=2000, random_state=27, noise=0.03)
sgd = SGD(lr=0.02)
he_initializer = he_normal(seed=42)
normal_initializer = normal(seed=42)
replaydata = ReplayData(X,
y,
filename='circles_dataset.h5',
group_name=group_name)
model = Sequential()
model.add(Dense(input_dim=2, units=5, kernel_initializer=he_initializer))
model.add(Activation('relu'))
model.add(Dense(units=3, kernel_initializer=he_initializer))
model.add(Activation('relu'))
model.add(
Dense(units=1,
kernel_initializer=normal_initializer,
activation='sigmoid',
