def pretrain(self):
X_train_tmp = train_set
self.trained_encoders = []
self.trained_decoders = []
for i in range(len(dims) - 1):
print('Pre-training the layer: Input {} -> {} -> Output {}'.format(
dims[i], dims[i + 1], dims[i]))
# Create AE and training
print(i)
if i == 0:
x = Input(shape=(dims[0], ), name='input')
x_drop = Dropout(dr_rate)(x)
h = Dense(dims[i + 1],
input_dim=dims[i],
activation='relu',
W_regularizer=Reg.l1_l2(l1=nu1, l2=nu2),
name='encoder_%d' % i)(x_drop)
y = Dense(dims[i],
input_dim=dims[i + 1],
W_regularizer=Reg.l1_l2(l1=nu1, l2=nu2),
name='decoder_%d' % i)(h)
x_diff1 = Subtract()([x, y])
x_diff2 = Subtract()([x, y])
ae = Model(inputs=x, outputs=[x_diff1, x_diff2])
ae.compile(loss=[weighted_mse, weighted_mae], optimizer='adam')
ae.fit_generator(batch_generator(train_set,
batch_size=batch_size,
shuffle=True,
beta=1.0,
gamma=gamma),
steps_per_epoch=steps_per_epoch,
nb_epoch=epochs_pretrain)
ae.summary()
# Store trainined weight
self.trained_encoders.append(ae.layers[2])
self.trained_decoders.append(ae.layers[3])
# Update training data
encoder = Model(ae.input, ae.layers[2].output)
X_train_tmp = encoder.predict(X_train_tmp)
elif i == len(dims) - 2:
ae = Sequential()
ae.add(Dropout(dr_rate))
ae.add(
Dense(dims[i + 1],
input_dim=dims[i],
W_regularizer=Reg.l1_l2(l1=nu1, l2=nu2),
name='encoder_%d' % i))
ae.add(
Dense(dims[i],
input_dim=dims[i + 1],
activation='relu',
W_regularizer=Reg.l1_l2(l1=nu1, l2=nu2),
name='decoder_%d' % i))
ae.compile(loss='mean_squared_error', optimizer='adam')
ae.fit(X_train_tmp,
X_train_tmp,
batch_size=batch_size,
nb_epoch=epochs_pretrain)
ae.summary()
# Store trainined weight
self.trained_encoders.append(ae.layers[1])
self.trained_decoders.append(ae.layers[2])
# Update training data
encoder = Model(ae.input, ae.layers[2].output)
X_train_tmp = encoder.predict(X_train_tmp)
else:
ae = Sequential()
ae.add(Dropout(dr_rate))
ae.add(
Dense(dims[i + 1],
input_dim=dims[i],
activation='relu',
W_regularizer=Reg.l1_l2(l1=nu1, l2=nu2),
name='encoder_%d' % i))
ae.add(
Dense(dims[i],
input_dim=dims[i + 1],
activation='relu',
W_regularizer=Reg.l1_l2(l1=nu1, l2=nu2),
name='decoder_%d' % i))
ae.compile(loss='mean_squared_error', optimizer='adam')
ae.fit(X_train_tmp,
X_train_tmp,
batch_size=batch_size,
nb_epoch=epochs_pretrain)
ae.summary()
# Store trainined weight
self.trained_encoders.append(ae.layers[1])
self.trained_decoders.append(ae.layers[2])
# Update training data
encoder = Model(ae.input, ae.layers[1].output)
X_train_tmp = encoder.predict(X_train_tmp)
def unet2(ldct_img, is_training=True):
"""
Defines the layer configurations and parameters in the contracting,
expanding paths and produces the residual mapping as output
"""
net = ldct_img
c1 = Conv2D(8, (3, 3), padding='same')(net)
#c1 = BatchNormalization()(c1)
c1 = Activation('relu')(c1)
c1 = Conv2D(8, (3, 3), padding='same', use_bias=False)(c1)
c1 = BatchNormalization()(c1)
c1 = Activation('relu')(c1)
p1 = MaxPooling2D((2, 2))(c1)
c2 = Conv2D(16, (3, 3), padding='same', use_bias=False)(p1)
c2 = BatchNormalization()(c2)
c2 = Activation('relu')(c2)
c2 = Conv2D(16, (3, 3), padding='same', use_bias=False)(c2)
c2 = BatchNormalization()(c2)
c2 = Activation('relu')(c2)
p2 = MaxPooling2D((2, 2))(c2)
c3 = Conv2D(32, (3, 3), padding='same', use_bias=False)(p2)
c3 = BatchNormalization()(c3)
c3 = Activation('relu')(c3)
c3 = Conv2D(32, (3, 3), padding='same', use_bias=False)(c3)
c3 = BatchNormalization()(c3)
c3 = Activation('relu')(c3)
p3 = MaxPooling2D((2, 2))(c3)
c4 = Conv2D(64, (3, 3), padding='same', use_bias=False)(p3)
c4 = BatchNormalization()(c4)
c4 = Activation('relu')(c4)
c4 = Conv2D(64, (3, 3), padding='same', use_bias=False)(c4)
c4 = BatchNormalization()(c4)
c4 = Activation('relu')(c4)
p4 = MaxPooling2D(pool_size=(2, 2))(c4)
c5 = Conv2D(128, (3, 3), padding='same', use_bias=False)(p4)
c5 = BatchNormalization()(c5)
c5 = Activation('relu')(c5)
c5 = Conv2D(128, (3, 3), padding='same', use_bias=False)(c5)
c5 = BatchNormalization()(c5)
c5 = Activation('relu')(c5)
u6 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c5)
u6 = concatenate([u6, c4])
c6 = Conv2D(64, (3, 3), padding='same', use_bias=False)(u6)
c6 = BatchNormalization()(c6)
c6 = Activation('relu')(c6)
c6 = Conv2D(64, (3, 3), padding='same', use_bias=False)(c6)
c6 = BatchNormalization()(c6)
c6 = Activation('relu')(c6)
u7 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c6)
u7 = concatenate([u7, c3])
c7 = Conv2D(32, (3, 3), padding='same', use_bias=False)(u7)
c7 = BatchNormalization()(c7)
c7 = Activation('relu')(c7)
c7 = Conv2D(32, (3, 3), padding='same', use_bias=False)(c7)
c7 = BatchNormalization()(c7)
c7 = Activation('relu')(c7)
u8 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c7)
u8 = concatenate([u8, c2])
c8 = Conv2D(16, (3, 3), padding='same', use_bias=False)(u8)
c8 = BatchNormalization()(c8)
c8 = Activation('relu')(c8)
c8 = Conv2D(16, (3, 3), padding='same', use_bias=False)(c8)
c8 = BatchNormalization()(c8)
c8 = Activation('relu')(c8)
u9 = Conv2DTranspose(8, (2, 2), strides=(2, 2), padding='same')(c8)
u9 = concatenate([u9, c1])
c9 = Conv2D(8, (3, 3), padding='same', use_bias=False)(u9)
c9 = BatchNormalization()(c9)
c9 = Activation('relu')(c9)
c9 = Conv2D(8, (3, 3), padding='same', use_bias=False)(c9)
c9 = BatchNormalization()(c9)
c9 = Activation('relu')(c9)
output_img = Conv2D(1, (1, 1), padding='same')(c9)
denoised_image = Subtract()([net, output_img])
return denoised_image
mask = Masking()
gru = GRU(filter_num,
dropout=drop_out_rate,
recurrent_dropout=0.2,
return_sequences=True)
get_last_output = Lambda(lambda x: kb.reshape(x[:, -1, :], [-1, filter_num]))
source_outputs.append(get_last_output(gru(mask(source))))
target_outputs.append(get_last_output(gru(mask(target))))
source_conc = Concatenate()(source_outputs)
target_conc = Concatenate()(target_outputs)
abs = Lambda(lambda x: kb.abs(x))
h_sub = abs(Subtract()([source_conc, target_conc]))
h_mul = Multiply()([source_conc, target_conc])
w1 = Dense(all_filter_num,
activation='tanh',
kernel_regularizer=regularizers.l2(regularizer_rate),
bias_regularizer=regularizers.l2(regularizer_rate))
w2 = Dense(all_filter_num,
activation='tanh',
kernel_regularizer=regularizers.l2(regularizer_rate),
bias_regularizer=regularizers.l2(regularizer_rate))
sdv = Add()([w1(h_sub), w2(h_mul)])
output = Dense(all_filter_num,
activation='tanh',
def learn_embeddings(self, graph=None, edge_f=None):
# TensorFlow wizardry
config = tf.ConfigProto()
# Don't pre-allocate memory; allocate as-needed
config.gpu_options.allow_growth = True
# Only allow a total of half the GPU memory to be allocated
config.gpu_options.per_process_gpu_memory_fraction = 0.1
# Create a session with the above options specified.
KBack.tensorflow_backend.set_session(tf.Session(config=config))
if not graph and not edge_f:
raise Exception('graph/edge_f needed')
if not graph:
graph = graph_util.loadGraphFromEdgeListTxt(edge_f)
S = nx.to_scipy_sparse_matrix(graph)
self._node_num = graph.number_of_nodes()
t1 = time()
# Generate encoder, decoder and autoencoder
self._num_iter = self._n_iter
self._encoder = get_encoder(self._node_num, self._d, self._n_units,
self._nu1, self._nu2, self._actfn)
self._decoder = get_decoder(self._node_num, self._d, self._n_units,
self._nu1, self._nu2, self._actfn)
self._autoencoder = get_autoencoder(self._encoder, self._decoder)
# Initialize self._model
# Input
x_in = Input(shape=(self._node_num, ), name='x_in')
# Process inputs
[x_hat, y] = self._autoencoder(x_in)
# Outputs
x_diff = Subtract()([x_hat, x_in])
# Objectives
def weighted_mse_x(y_true, y_pred):
""" Hack: This fn doesn't accept additional arguments.
We use y_true to pass them.
y_pred: Contains x_hat - x
y_true: Contains b
"""
return KBack.sum(KBack.square(y_pred *
y_true[:, 0:self._node_num]),
axis=-1)
# Model
self._model = Model(input=x_in, output=x_diff)
sgd = SGD(lr=self._xeta, decay=1e-5, momentum=0.99, nesterov=True)
adam = Adam(lr=self._xeta, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
self._model.compile(optimizer=sgd, loss=weighted_mse_x)
history = self._model.fit_generator(
generator=batch_generator_ae(S, self._beta, self._n_batch, True),
nb_epoch=self._num_iter,
samples_per_epoch=S.shape[0] // self._n_batch,
verbose=1,
# callbacks=[tensorboard]
# callbacks=[callbacks.TerminateOnNaN()]
)
loss = history.history['loss']
# Get embedding for all points
if loss[0] == np.inf or np.isnan(loss[0]):
print('Model diverged. Assigning random embeddings')
self._Y = np.random.randn(self._node_num, self._d)
else:
try:
self._Y, self._next_adj = model_batch_predictor_v2(
self._autoencoder, S, self._n_batch)
except:
pdb.set_trace()
t2 = time()
# Save the autoencoder and its weights
if self._weightfile is not None:
saveweights(self._encoder, self._weightfile[0])
saveweights(self._decoder, self._weightfile[1])
if self._modelfile is not None:
savemodel(self._encoder, self._modelfile[0])
savemodel(self._decoder, self._modelfile[1])
if self._savefilesuffix is not None:
saveweights(self._encoder,
'encoder_weights_' + self._savefilesuffix + '.hdf5')
saveweights(self._decoder,
'decoder_weights_' + self._savefilesuffix + '.hdf5')
savemodel(self._encoder,
'encoder_model_' + self._savefilesuffix + '.json')
savemodel(self._decoder,
'decoder_model_' + self._savefilesuffix + '.json')
# Save the embedding
np.savetxt('embedding_' + self._savefilesuffix + '.txt', self._Y)
return self._Y, (t2 - t1)
def __init__(self, env, replay, deep, duel):
# Define your network architecture here. It is also a good idea to define any training operations
# and optimizers here, initialize your variables, or alternately compile your model here.
self.learning_rate = 0.001 #HYPERPARAMETER1
#linear network
if(deep==False and duel==False):
print("Setting up linear network....")
self.model = Sequential()
# self.model.add(Dense(env.action_space.n, input_dim = env.observation_space.shape[0], activation='linear', kernel_initializer='he_uniform', use_bias = True))
self.model.add(Dense(32, input_dim = env.observation_space.shape[0]+1, activation='linear', kernel_initializer='he_uniform',use_bias = True))
self.model.add(Dense(env.action_space.n, input_dim = 32, activation='linear', kernel_initializer='he_uniform',use_bias = True))
self.model.compile(optimizer = Adam(lr=self.learning_rate), loss='mse')
# plot_model(self.model, to_file='graphs/Linear.png', show_shapes = True)
self.model.summary()
#deep network
elif(deep==True):
print("Setting up DDQN network....")
self.model = Sequential()
self.model.add(Dense(32, input_dim = env.observation_space.shape[0]+1, activation='relu', kernel_initializer='he_uniform',use_bias = True))
# self.model.add(BatchNormalization())
self.model.add(Dense(32, input_dim = 32,activation='relu', kernel_initializer='he_uniform',use_bias = True))
# self.model.add(BatchNormalization())
# self.model.add(Dense(64, input_dim = 64, activation='relu', kernel_initializer='he_uniform',use_bias = True))
# self.model.add(Dense(32, input_dim = 64, activation='relu', kernel_initializer='he_uniform',use_bias = True))
self.model.add(Dense(32, input_dim = 32, activation='relu', kernel_initializer='he_uniform',use_bias = True))
# self.model.add(BatchNormalization())
self.model.add(Dense(env.action_space.n, input_dim = 32, activation='linear', kernel_initializer='he_uniform',use_bias = True))
print("Q-Network initialized.... :)\n")
self.model.compile(optimizer = Adam(lr=self.learning_rate), loss='mse')
# plot_model(self.model, to_file='graphs/DDQN.png', show_shapes = True)
self.model.summary()
#dueling network
elif(duel==True):
print("Setting up Dueling DDQN network....")
inp = Input(shape=(env.observation_space.shape[0]+1,))
layer_shared1 = Dense(32,activation='relu',kernel_initializer='he_uniform',use_bias = True)(inp)
# layer_shared1 = BatchNormalization()(layer_shared1)
layer_shared2 = Dense(32,activation='relu',kernel_initializer='he_uniform',use_bias = True)(layer_shared1)
layer_shared2 = Dense(32,activation='relu',kernel_initializer='he_uniform',use_bias = True)(layer_shared2)
# layer_shared2 = BatchNormalization()(layer_shared2)
print("Shared layers initialized....")
layer_v1 = Dense(32,activation='relu',kernel_initializer='he_uniform',use_bias = True)(layer_shared2)
# # layer_v1 = BatchNormalization()(layer_v1)
layer_a1 = Dense(32,activation='relu',kernel_initializer='he_uniform',use_bias = True)(layer_shared2)
# layer_a1 = BatchNormalization()(layer_a1)
layer_v2 = Dense(1,activation='linear',kernel_initializer='he_uniform',use_bias = True)(layer_v1)
layer_a2 = Dense(env.action_space.n,activation='linear',kernel_initializer='he_uniform',use_bias = True)(layer_a1)
print("Value and Advantage Layers initialised....")
layer_mean = Lambda(lambda x: K.mean(x,axis=-1,keepdims=True))(layer_a2)
temp = layer_v2
temp2 = layer_mean
for i in range(env.action_space.n-1):
layer_v2 = keras.layers.concatenate([layer_v2,temp],axis=-1)
layer_mean = keras.layers.concatenate([layer_mean,temp2],axis=-1)
# layer_q = Lambda(lambda x: K.expand_dims(x[0],axis=-1) + x[1] - K.mean(x[1],keepdims=True), output_shape=(env.action_space.n,))([layer_v2, layer_a2])
layer_q = Subtract()([layer_a2,layer_mean])
layer_q = Add()([layer_q,layer_v2])
print("Q-function layer initialized.... :)\n")
self.model = Model(inp, layer_q)
self.model.summary()
self.model.compile(optimizer = Adam(lr=self.learning_rate), loss='mse')
x2 = bilstm1(x2)
e = Dot(axes=2)([x1, x2])
e1 = Softmax(axis=2)(e)
e2 = Softmax(axis=1)(e)
e1 = Lambda(K.expand_dims, arguments={'axis' : 3})(e1)
e2 = Lambda(K.expand_dims, arguments={'axis' : 3})(e2)
_x1 = Lambda(K.expand_dims, arguments={'axis' : 1})(x2)
_x1 = Multiply()([e1, _x1])
_x1 = Lambda(K.sum, arguments={'axis' : 2})(_x1)
_x2 = Lambda(K.expand_dims, arguments={'axis' : 2})(x1)
_x2 = Multiply()([e2, _x2])
_x2 = Lambda(K.sum, arguments={'axis' : 1})(_x2)
m1 = Concatenate()([x1, _x1, Subtract()([x1, _x1]), Multiply()([x1, _x1])])
m2 = Concatenate()([x2, _x2, Subtract()([x2, _x2]), Multiply()([x2, _x2])])
y1 = bilstm2(m1)
y2 = bilstm2(m2)
mx1 = Lambda(K.max, arguments={'axis' : 1})(y1)
av1 = Lambda(K.mean, arguments={'axis' : 1})(y1)
mx2 = Lambda(K.max, arguments={'axis' : 1})(y2)
av2 = Lambda(K.mean, arguments={'axis' : 1})(y2)
y = Concatenate()([av1, mx1, av2, mx2])
y = Dense(1024, activation='tanh')(y)
y = Dropout(0.5)(y)
y = Dense(1024, activation='tanh')(y)
y = Dropout(0.5)(y)
return_sequences=True,
dropout=dropout,
recurrent_dropout=rec_dropout),
merge_mode=merge_m,
weights=None)(embedded_sequences)
elif model_choice == 2:
gru_kata = Bidirectional(GRU(EMBEDDING_DIM,
return_sequences=True,
dropout=dropout,
recurrent_dropout=rec_dropout),
merge_mode=merge_m,
weights=None)(rtwo)
else:
combine = 1 # input('Enter 1 for Add, 2 for Subtract, 3 for Multiply, 4 for Average, 5 for Maximum: ')
if combine == 2:
merge = Subtract()([embedded_sequences, rtwo])
elif combine == 3:
merge = Multiply()([embedded_sequences, rtwo])
elif combine == 4:
merge = Average()([embedded_sequences, rtwo])
elif combine == 5:
merge = Maximum()([embedded_sequences, rtwo])
else:
merge = Add()([embedded_sequences, rtwo])
gru_kata = Bidirectional(GRU(EMBEDDING_DIM,
return_sequences=True,
dropout=dropout,
recurrent_dropout=rec_dropout,
trainable=gtrainable),
merge_mode=merge_m,
weights=None)(merge)
def getModel(self, n_objects, object_dim=27, relation_dim=1):
if n_objects in self.Nets.keys():
return self.Nets[n_objects]
n_relations = n_objects * (n_objects - 1)
#Inputs
objects = Input(shape=(n_objects, object_dim), name='objects')
sender_relations = Input(shape=(n_objects, n_relations),
name='sender_relations')
receiver_relations = Input(shape=(n_objects, n_relations),
name='receiver_relations')
permuted_senders_rel = Permute((2, 1))(sender_relations)
permuted_receiver_rel = Permute((2, 1))(receiver_relations)
relation_info = Input(shape=(n_relations, relation_dim),
name='relation_info')
ls1 = 128
ls2 = 128
ls3 = 128
ls4 = 128
propagation = Input(shape=(n_objects, ls4), name='propagation')
# Getting sender and receiver objects
senders = dot([permuted_senders_rel, objects], axes=(2, 1))
receivers = dot([permuted_receiver_rel, objects], axes=(2, 1))
# Getting specific features of objects for relationNetwork
get_attributes = Lambda(lambda x: x[:, :, 7:13],
output_shape=(n_relations, 6))
get_z = Lambda(lambda x: x[:, :, 2:3], output_shape=(n_relations, 1))
get_pos = Lambda(lambda x: x[:, :, :3], output_shape=(n_relations, 3))
get_orientation = Lambda(lambda x: x[:, :, 3:7],
output_shape=(n_relations, 4))
get_vel = Lambda(lambda x: x[:, :, 13:20],
output_shape=(n_relations, 7))
get_control = Lambda(lambda x: x[:, :, 20:27],
output_shape=(n_relations, 7))
# Getting specific features of objects for objectNetwork
get_attributes2 = Lambda(lambda x: x[:, :, 7:13],
output_shape=(n_objects, 6))
get_pose = Lambda(lambda x: x[:, :, 3:7], output_shape=(n_objects, 4))
get_vel2 = Lambda(lambda x: x[:, :, 13:20],
output_shape=(n_objects, 7))
get_control2 = Lambda(lambda x: x[:, :, 20:27],
output_shape=(n_objects, 7))
get_z2 = Lambda(lambda x: x[:, :, 2:3], output_shape=(n_objects, 1))
if (self.set_weights):
rm = RelationalModel((n_relations, ), 53 + relation_dim,
[ls1, ls1, ls1], self.relnet, True)
om = ObjectModel((n_objects, ), 18, [ls2, ls2], self.objnet, True)
rmp = RelationalModel((n_relations, ), ls4 * 2 + ls1, [ls3, ls3],
self.relnetp, True)
omp = ObjectModel((n_objects, ), ls2 + ls3 + ls4, [ls4, ls4],
self.objnetp, True)
else:
rm = RelationalModel((n_relations, ), 53 + relation_dim,
[ls1, ls1, ls1])
om = ObjectModel((n_objects, ), 18, [ls2, ls2])
rmp = RelationalModel((n_relations, ), ls4 * 2 + ls1, [ls3, ls3])
omp = ObjectModel((n_objects, ), ls2 + ls3 + ls4, [ls4, ls4])
self.set_weights = True
self.relnet = rm.getRelnet()
self.objnet = om.getObjnet()
self.relnetp = rmp.getRelnet()
self.objnetp = omp.getObjnet()
r_att = get_attributes(receivers)
s_att = get_attributes(senders)
r_pos = get_pos(receivers)
s_pos = get_pos(senders)
r_orientation = get_orientation(receivers)
s_orientation = get_orientation(senders)
r_vel = get_vel(receivers)
s_vel = get_vel(senders)
r_control = get_control(receivers)
s_control = get_control(senders)
# r_posvel = Concatenate()([r_pos,r_vel])
# s_posvel = Concatenate()([s_pos,s_vel])
# Getting dynamic state differences.
dif_rs = Subtract()([r_pos, s_pos])
# Creating Input of Relation Network
# 3 + 2*7 +2*6 + 2*7 + 4*2 +2
rel_vector_wo_prop = Concatenate()([
relation_info, dif_rs, r_vel, s_vel, r_att, s_att, r_control,
s_control, r_orientation, s_orientation,
get_z(receivers),
get_z(senders)
])
# 7 + 4 + 6 + 1
obj_vector_wo_er = Concatenate()([
get_vel2(objects),
get_pose(objects),
get_attributes2(objects),
get_z2(objects)
])
rel_encoding = Activation('relu')(rm(rel_vector_wo_prop))
obj_encoding = Activation('relu')(om(obj_vector_wo_er))
# rel_encoding=Dropout(0.1)(rel_encoding)
# obj_encoding=Dropout(0.1)(obj_encoding)
prop = propagation
# prop_layer=Lambda(lambda x: x[:,:,7:], output_shape=(n_objects,256))
for _ in range(1):
senders_prop = dot([permuted_senders_rel, prop], axes=(2, 1))
receivers_prop = dot([permuted_receiver_rel, prop], axes=(2, 1))
rmp_vector = Concatenate()(
[rel_encoding, senders_prop, receivers_prop])
x = rmp(rmp_vector)
effect_receivers = Activation('relu')(dot([receiver_relations, x],
axes=(2, 1)))
omp_vector = Concatenate()([obj_encoding, effect_receivers,
prop]) #
x = omp(omp_vector)
prop = Activation('relu')(Add()([x, prop]))
x = Dense(7,
kernel_regularizer=regularizers.l2(regul),
bias_regularizer=regularizers.l2(regul),
activation='linear')(prop)
velo = Lambda(lambda x: x[:, 1:2, :3],
output_shape=(1, 3),
name='target_vel')(x)
# quat=Lambda(lambda x: x[:,1:2,3:7], output_shape=(1,4),name='target_quat')(x)
quat = Lambda(lambda x: K.l2_normalize(x[:, 1:2, 3:7], axis=2),
output_shape=(1, 4),
name='target_quat')(x)
# target = Concatenate(name='target')([velo,quat])
model = Model(inputs=[
objects, sender_relations, receiver_relations, relation_info,
propagation
],
outputs=[velo, quat])
adam = optimizers.Adam(lr=0.001, epsilon=1e-7, decay=0.0, amsgrad=True)
model.compile(optimizer=adam,
loss=[losses.mse, losses.mae],
loss_weights={
'target_vel': 1,
'target_quat': 1
})
self.Nets[n_objects] = model #quat_loss
return model
def _build_model(self):
# Get the shape of the environment
_, fingerprint_size = self.env.action_space.shape
predict_actions_input = Input(batch_shape=(None, fingerprint_size), name='single_action')
train_action_input = Input(batch_shape=(self.batch_size, fingerprint_size),
name='batch_action')
reward_input = Input(batch_shape=(self.batch_size, 1), name='rewards')
done_input = Input(batch_shape=(self.batch_size, 1), name='done')
new_actions = [Input(batch_shape=(None, fingerprint_size), name=f'next_actions_{i}')
for i in range(self.batch_size)]
# Squeeze the train action and reward input
squeeze = Lambda(K.squeeze, arguments={'axis': 1}, name='squeezer')
reward = squeeze(reward_input)
done = squeeze(done_input)
# Define the Q network. Note that we use a `Network` rather than model because
# this model is not trained
# - Takes a list of actions as input
# - Produces the value of each action as output
# TODO (wardlt): Allow users to specify a different architecture
def make_q_network(input_shape, name=None):
inputs = Input(shape=input_shape)
h1 = Dense(24, activation='relu')(inputs)
h2 = Dense(48, activation='relu')(h1)
h3 = Dense(24, activation='relu')(h2)
output = Dense(1, activation='linear')(h3)
return Network(inputs=inputs, outputs=output, name=name)
q_t = make_q_network((None, fingerprint_size), name='q_t')
q = q_t(predict_actions_input)
self.action_network = Model(inputs=predict_actions_input, outputs=q)
# Make the training network
# Part 1: Computing estimated value of the next state
# Set as the maximum Q for any action from that next state
# Note: This Q network is not updated by the optimizer. Instead, it is
# periodically updated with the weights from `q_t`, which is being updated
q_tp1 = make_q_network((None, fingerprint_size), name='q_tp1')
q_tp1.trainable = False
max_layer = Lambda(K.max, arguments={'axis': 0}, name='max_layer')
q_values = [max_layer(q_tp1(action)) for action in new_actions]
v_tp1 = Concatenate(name='v_tp1')(q_values)
# Part 2: Define the target function, the measured reward of a state
# plus the estimated value of the next state (or zero if this state is terminal)
target = Lambda(_q_target_value, name='target', arguments={'gamma': self.gamma})\
([reward, v_tp1, done])
# Part 3: Define the error single
q_t_train = q_t(train_action_input)
q_t_train = Lambda(K.reshape, arguments={'shape': (self.batch_size,)},
name='squeeze_q')(q_t_train)
error = Subtract(name='error')([q_t_train, target])
error = Lambda(K.reshape, arguments={'shape': (self.batch_size, 1)},
name='wrap_error')(error)
self.train_network = GraphModel(
inputs=[train_action_input, done_input, reward_input] + new_actions,
outputs=error)
# Add the optimizer
self.train_network.compile(optimizer='adam', loss='mean_squared_error')
def learn_embedding(self,
graph=None,
edge_f=None,
is_weighted=False,
no_python=False):
if not graph and not edge_f:
raise Exception('graph/edge_f needed')
if not graph:
graph = graph_util.loadGraphFromEdgeListTxt(edge_f)
S = nx.to_scipy_sparse_matrix(graph)
self._node_num = graph.number_of_nodes()
t1 = time()
# Generate encoder, decoder and autoencoder
self._num_iter = self._n_iter
self._encoder = get_encoder(self._node_num, self._d, self._n_units,
self._nu1, self._nu2, self._actfn)
self._decoder = get_decoder(self._node_num, self._d, self._n_units,
self._nu1, self._nu2, self._actfn)
self._autoencoder = get_autoencoder(self._encoder, self._decoder)
# Initialize self._model
# Input
x_in = Input(shape=(self._node_num, ), name='x_in')
# Process inputs
[x_hat, y] = self._autoencoder(x_in)
# Outputs
x_diff = Subtract()([x_hat, x_in])
# x_diff = merge([x_hat, x_in],
# mode=lambda (a, b): a - b,
# output_shape=lambda L: L[1])
# Objectives
def weighted_mse_x(y_true, y_pred):
''' Hack: This fn doesn't accept additional arguments.
We use y_true to pass them.
y_pred: Contains x_hat - x
y_true: Contains b
'''
return KBack.sum(KBack.square(y_true * y_pred), axis=-1)
# Model
self._model = Model(input=x_in, output=x_diff)
# sgd = SGD(lr=self._xeta, decay=1e-5, momentum=0.99, nesterov=True)
adam = Adam(lr=self._xeta, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
self._model.compile(optimizer=adam, loss=weighted_mse_x)
history = self._model.fit_generator(
generator=batch_generator_ae(S, self._beta, self._n_batch, True),
nb_epoch=self._num_iter,
samples_per_epoch=S.shape[0] // self._n_batch,
verbose=1,
callbacks=[callbacks.TerminateOnNaN()])
loss = history.history['loss']
# Get embedding for all points
if loss[0] == np.inf or np.isnan(loss[0]):
print('Model diverged. Assigning random embeddings')
self._Y = np.random.randn(self._node_num, self._d)
else:
self._Y = model_batch_predictor(self._autoencoder, S,
self._n_batch)
t2 = time()
# Save the autoencoder and its weights
if (self._weightfile is not None):
saveweights(self._encoder, self._weightfile[0])
saveweights(self._decoder, self._weightfile[1])
if (self._modelfile is not None):
savemodel(self._encoder, self._modelfile[0])
savemodel(self._decoder, self._modelfile[1])
if (self._savefilesuffix is not None):
saveweights(self._encoder,
'encoder_weights_' + self._savefilesuffix + '.hdf5')
saveweights(self._decoder,
'decoder_weights_' + self._savefilesuffix + '.hdf5')
savemodel(self._encoder,
'encoder_model_' + self._savefilesuffix + '.json')
savemodel(self._decoder,
'decoder_model_' + self._savefilesuffix + '.json')
# Save the embedding
np.savetxt('embedding_' + self._savefilesuffix + '.txt', self._Y)
return self._Y, (t2 - t1)
def getModel(self,
n_objects,
object_dim=19,
relation_dim=1,
timestep_diff=100):
if n_objects in self.Nets.keys():
return self.Nets[n_objects]
n_relations = n_objects * (n_objects - 1)
#Inputs
objects = Input(shape=(timestep_diff, n_objects, object_dim),
name='objects')
sender_relations = Input(shape=(timestep_diff, n_objects, n_relations),
name='sender_relations')
receiver_relations = Input(shape=(timestep_diff, n_objects,
n_relations),
name='receiver_relations')
permuted_senders_rel = Permute((1, 3, 2))(sender_relations)
permuted_receiver_rel = Permute((1, 3, 2))(receiver_relations)
relation_info = Input(shape=(timestep_diff, n_relations, relation_dim),
name='relation_info')
propagation = Input(shape=(timestep_diff, n_objects, 100),
name='propagation')
# Getting sender and receiver objects
senders = dot([permuted_senders_rel, objects], axes=(3, 2))
receivers = dot([permuted_receiver_rel, objects], axes=(3, 2))
# Getting specific features of objects for relationNetwork
get_attributes = Lambda(lambda x: x[:, :, :, 7:13],
output_shape=(timestep_diff, n_relations, 6))
get_pos = Lambda(lambda x: x[:, :, :, :3],
output_shape=(timestep_diff, n_relations, 3))
get_orientation = Lambda(lambda x: x[:, :, :, 3:7],
output_shape=(timestep_diff, n_relations, 4))
get_vel = Lambda(lambda x: x[:, :, :, 13:16],
output_shape=(timestep_diff, n_relations, 3))
get_control = Lambda(lambda x: x[:, :, :, 16:19],
output_shape=(timestep_diff, n_relations, 3))
# Getting specific features of objects for objectNetwork
get_attributes2 = Lambda(lambda x: x[:, :, :, 7:13],
output_shape=(timestep_diff, n_objects, 6))
get_pose = Lambda(lambda x: x[:, :, :, :7],
output_shape=(timestep_diff, n_objects, 7))
get_vel2 = Lambda(lambda x: x[:, :, :, 13:16],
output_shape=(timestep_diff, n_objects, 3))
get_control2 = Lambda(lambda x: x[:, :, :, 16:19],
output_shape=(timestep_diff, n_objects, 3))
if (self.set_weights):
rm = RelationalModel((
timestep_diff,
n_relations,
), 32 + relation_dim, [150, 150, 150, 150], self.relnet, True)
om = ObjectModel((
timestep_diff,
n_objects,
), 19, [100, 100], self.objnet, True)
rmp = RelationalModel((
timestep_diff,
n_relations,
), 350, [150, 150, 100], self.relnetp, True)
omp = ObjectModel((
timestep_diff,
n_objects,
), 300, [100, 100], self.objnetp, True)
rme = RecurrentRelationalModel_many(n_objects, 100, [100, 7],
self.rmepart1, self.rmepart2,
True, timestep_diff, 0)
else:
rm = RelationalModel((
timestep_diff,
n_relations,
), 32 + relation_dim, [150, 150, 150, 150])
om = ObjectModel((
timestep_diff,
n_objects,
), 19, [100, 100])
rmp = RelationalModel((
timestep_diff,
n_relations,
), 350, [150, 150, 100])
omp = ObjectModel((
timestep_diff,
n_objects,
), 300, [100, 100])
rme = RecurrentRelationalModel_many(n_objects, 100, [100, 7], None,
None, False, timestep_diff, 0)
self.set_weights = True
self.relnet = rm.getRelnet()
self.objnet = om.getObjnet()
self.relnetp = rmp.getRelnet()
self.objnetp = omp.getObjnet()
self.rmepart1, self.rmepart2 = rme.getRelnet()
r_att = get_attributes(receivers)
s_att = get_attributes(senders)
r_pos = get_pos(receivers)
s_pos = get_pos(senders)
r_orientation = get_orientation(receivers)
s_orientation = get_orientation(senders)
r_vel = get_vel(receivers)
s_vel = get_vel(senders)
r_control = get_control(receivers)
s_control = get_control(senders)
r_posvel = Concatenate()([r_pos, r_vel])
s_posvel = Concatenate()([s_pos, s_vel])
# Getting dynamic state differences.
dif_rs = Subtract()([r_posvel, s_posvel])
# Creating Input of Relation Network
rel_vector_wo_prop = Concatenate()([
relation_info, dif_rs, r_att, s_att, r_control, s_control,
r_orientation, s_orientation
])
obj_vector_wo_er = Concatenate()([
get_vel2(objects),
get_pose(objects),
get_attributes2(objects),
get_control2(objects)
])
rel_encoding = Activation('relu')(rm(rel_vector_wo_prop))
obj_encoding = Activation('relu')(om(obj_vector_wo_er))
rel_encoding = Dropout(0.1)(rel_encoding)
obj_encoding = Dropout(0.1)(obj_encoding)
prop = propagation
for _ in range(1):
senders_prop = dot([permuted_senders_rel, prop], axes=(2, 1))
receivers_prop = dot([permuted_receiver_rel, prop], axes=(2, 1))
rmp_vector = Concatenate()(
[rel_encoding, senders_prop, receivers_prop])
x = rmp(rmp_vector)
effect_receivers = Activation('relu')(dot([receiver_relations, x],
axes=(2, 1)))
omp_vector = Concatenate()([obj_encoding, effect_receivers,
prop]) #
x = omp(omp_vector)
prop = Activation('relu')(Add()([x, prop]))
x = Permute((2, 1, 3))(prop)
predicted = rme(x)
start_step = lookstart
prediction_mask = Input(shape=(n_objects, timestep_diff - start_step,
7),
name='prediction_mask')
predicted = Lambda(lambda x: x[:, :, :, :] * prediction_mask,
output_shape=(n_objects, timestep_diff - start_step,
7),
name='target')(predicted)
model = Model(inputs=[
objects, sender_relations, receiver_relations, relation_info,
propagation
],
outputs=[predicted])
adam = optimizers.Adam(lr=0.0001)
model.compile(optimizer=adam, loss='mse')
self.Nets[n_objects] = model
return model
default=16,
help='number of embedded vectors')
args = parser.parse_args()
dim = args.embedding_dim
nb_samples = args.nb_samples
x_train = np.random.randn(nb_samples, dim)
# Model -------------------------------------
input_shape = (dim, )
x = Input(shape=input_shape, name='encoder_input')
x1 = VQVAELayer(dim, args.num_embedding, commitment_cost, name="vq1")(x)
x2 = Lambda(lambda x1: x + K.stop_gradient(x1 - x))(x1)
stage1_error = Subtract()([x, x2])
x3 = VQVAELayer(dim, args.num_embedding, commitment_cost,
name="vq2")(stage1_error)
x4 = Lambda(lambda x3: stage1_error + K.stop_gradient(x3 - stage1_error))(x3)
x5 = Add()([x2, x4])
vqvae = Model(x, x5)
loss = vq_vae_loss_wrapper(commitment_cost, x1, x, x3, stage1_error)
vqvae.compile(loss=loss, optimizer='adam')
vqvae.summary()
plot_model(vqvae, to_file='vq_vae_demo_2stage.png', show_shapes=True)
history = vqvae.fit(x_train,
x_train,
batch_size=batch_size,
y = Dense(1)(x)
model = Model(inputs=inputs, outputs=y)
return model
base_model = basic_model(input_dims=512)
inp1 = Input(shape=(512, ), dtype='float32', name='Correct_Paragraph'
) # Always some variation of True paragraph and Question
inp2 = Input(shape=(512, ), dtype='float32', name='Original_Paragraph'
) # Some variation of False + True paragraph and Question
y1 = base_model(inp1)
y2 = base_model(inp2)
y = Subtract()([y1, y2])
y_out = Activation("sigmoid")(y)
final_model = Model([inp1, inp2], y_out)
final_model.compile(optimizer='Adadelta',
loss='binary_crossentropy',
metrics=['binary_accuracy'])
# Load weights
print("Loading weights...")
final_model.load_weights("absolute_diff_model_weight_0_drop_1st.h5")
base_model1 = basic_model(input_dims=512)
inp11 = Input(shape=(512, ), dtype='float32', name='Correct_Paragraph'
) # Always some variation of True paragraph and Question
inp21 = Input(shape=(512, ), dtype='float32', name='Original_Paragraph'
def build_model(word2idx):
print('build model...')
source_input = Input(batch_shape=(None, seq_length))
target_input = Input(batch_shape=(None, seq_length))
embedding_layer = Embedding(len(word2idx),
embedding_size,
input_length=seq_length)
source = embedding_layer(source_input)
target = embedding_layer(target_input)
source_outputs = []
target_outputs = []
all_filter_num = len(filter_sizes) * filter_num
for filter_size in filter_sizes:
conv = Conv1D(filter_num,
filter_size,
activation='relu',
kernel_initializer='he_uniform',
bias_initializer='he_uniform')
max_pool = MaxPooling1D(seq_length - filter_size + 1)
reshape = Reshape([filter_num])
source_conv = conv(source)
target_conv = conv(target)
source_sdv = reshape(max_pool(source_conv))
target_sdv = reshape(max_pool(target_conv))
source_outputs.append(source_sdv)
target_outputs.append(target_sdv)
mask = Masking()
gru = GRU(filter_num, dropout=drop_out_rate, recurrent_dropout=0.2)
source_outputs.append(gru(mask(source)))
target_outputs.append(gru(mask(target)))
source_conc = Concatenate()(source_outputs)
target_conc = Concatenate()(target_outputs)
abs = Lambda(lambda x: kb.abs(x))
h_sub = abs(Subtract()([source_conc, target_conc]))
h_mul = Multiply()([source_conc, target_conc])
w1 = Dense(all_filter_num,
activation='tanh',
kernel_regularizer=regularizers.l2(regularizer_rate),
bias_regularizer=regularizers.l2(regularizer_rate))
w2 = Dense(all_filter_num,
activation='tanh',
kernel_regularizer=regularizers.l2(regularizer_rate),
bias_regularizer=regularizers.l2(regularizer_rate))
sdv = Add()([w1(h_sub), w2(h_mul)])
output = Dense(all_filter_num,
activation='tanh',
kernel_regularizer=regularizers.l2(regularizer_rate),
bias_regularizer=regularizers.l2(regularizer_rate))(sdv)
output = Dropout(drop_out_rate)(output)
logits = Dense(class_num,
activation='softmax',
kernel_regularizer=regularizers.l2(regularizer_rate),
bias_regularizer=regularizers.l2(regularizer_rate))(output)
model = Model(inputs=[source_input, target_input], outputs=logits)
return model
def __init__(self, dims1, dims2, bn, init, nu1, nu2, alpha, gamma, n_clusters, *hyper_dict, **kwargs):
''' Initialize the SDNE class
Args:
d: dimension of the embedding
beta: penalty parameter in matrix B of 2nd order objective
alpha: weighing hyperparameter for 1st order objective
nu1: L1-reg hyperparameter
nu2: L2-reg hyperparameter
K: number of hidden layers in encoder/decoder
n_units: vector of length K-1 containing #units in hidden layers
of encoder/decoder, not including the units in the
embedding layer
rho: bounding ratio for number of units in consecutive layers (< 1)
n_iter: number of sgd iterations for first embedding (const)
xeta: sgd step size parameter
n_batch: minibatch size for SGD
modelfile: Files containing previous encoder and decoder models
weightfile: Files containing previous encoder and decoder weights
'''
self.alpha = alpha
self.gamma = gamma
self.dims1 = dims1
self.dims2 = dims2
self.gene_num = self.dims1[0]
self.cell_num = self.dims2[0]
self.n_stacks1 = len(self.dims1) -1
self.n_stacks2 = len(self.dims2) -1
# Generate encoder, decoder and autoencoder
# If cannot use previous step information, initialize new models
if bn:
self.autoencoder1, self.encoder1 = autoencoder_bn(self.dims1, nu1=nu1, nu2=nu2, init=init)
self.autoencoder2, self.encoder2 = autoencoder(self.dims2, nu1=nu1, nu2=nu2, init=init)
else:
self.autoencoder1, self.encoder1 = autoencoder(self.dims1, nu1=nu1, nu2=nu2, init=init)
self.autoencoder2, self.encoder2 = autoencoder(self.dims2, nu1=nu1, nu2=nu2, init=init)
# Initialize self.model
# Input
x1 = Input(shape=(self.gene_num,), name='x1_in')
x2 = Input(shape=(self.cell_num,), name='x2_in')
# Process inputs
x_hat1 = self.autoencoder1(x1)
x_hat2 = self.autoencoder2(x2)
y1 = self.encoder1(x1)
y2 = self.encoder2(x2)
# Outputs
x_diff1 = Subtract()([x_hat1, x1])
x_diff2 = Subtract()([x_hat2, x2])
y_diff = Subtract()([y2, y1])
def cosine_distance(vests):
x, y = vests
# x = K.l2_normalize(x, axis=-1)
# y = K.l2_normalize(y, axis=-1)
return -K.mean(x * y, axis=-1, keepdims=True)
def cos_dist_output_shape(shapes):
shape1, shape2 = shapes
return (shape1[0], 1)
distance = Lambda(cosine_distance, output_shape=cos_dist_output_shape)([y1, y2])
# Model
# self.pre_model = Model(input=x_in, output=[x_diff1, x_diff2])
self.model = Model(input=[x1, x2], output=[x_diff1, x_diff2, y_diff])
input_shape = (20000, )
base_network = create_base_model(input_shape)
base_network.summary()
from keras.layers import Input, Subtract, Lambda, Dense
from keras.optimizers import RMSprop, Adam
from keras.models import Model
input_a = Input(shape=(input_shape))
input_b = Input(shape=(input_shape))
processed_a = base_network(input_a)
processed_b = base_network(input_b)
distance = Subtract()([processed_a, processed_b])
distance = Lambda(lambda x: np.absolute(x))(distance)
predictions = Dense(1, activation='sigmoid')(distance)
model = Model(inputs=[input_a, input_b], outputs=predictions)
rms = RMSprop(lr=1e-4, rho=0.9, epsilon=1e-08)
model.compile(loss='binary_crossentropy',
optimizer=Adam(lr=1e-4),
metrics=['accuracy'])
print(model.summary())
'''
bias_initializer='he_normal',
name="rsscnn_fc2_1")(rsscnn_flattend2)
rsscnn_fc2_2 = Dense(4096,
activation='relu',
kernel_initializer='he_normal',
bias_initializer='he_normal',
name="rsscnn_fc2_2")(rsscnn_fc2_1)
rsscnn_rank_right = Dense(1,
activation='linear',
kernel_regularizer=regularizers.l2(0.01),
kernel_initializer='he_normal',
bias_initializer='he_normal',
name="rsscnn_fc2_3")(rsscnn_fc2_2)
# # calculate the ranking difference between the image 1 and 2
rss_cnn_output = Subtract(name="rsscnn_output")(
[rsscnn_rank_right, rsscnn_rank_left])
# construct the model
model = Model(inputs=[image_input1, image_input2],
outputs=[ss_cnn_output, rss_cnn_output])
model.summary()
# compile the model
sgd = optimizers.SGD(lr=0.01, clipnorm=1.)
model.compile(optimizer=sgd,
loss={
'sscnn_output': 'binary_crossentropy',
'rsscnn_output': "squared_hinge"
},
loss_weights={
'sscnn_output': 1.,
def m_bilstm_attention_esim_add_feature_model(max_features, max_len_1, max_len_2, feature_size, embedding_matrix):
# 1. lstm attention
inp1 = Input(shape=(max_len_1,))
x1 = Embedding(max_features, 200, weights=[embedding_matrix], trainable=False)(inp1)
x1 = Bidirectional(LSTM(40, return_sequences=True))(x1)
atten_x1 = Attention(max_len_1)(x1)
inp2 = Input(shape=(max_len_2,))
x2 = Embedding(max_features, 200, weights=[embedding_matrix], trainable=False)(inp2)
x2 = Bidirectional(LSTM(40, return_sequences=True))(x2)
atten_x2 = Attention(max_len_2)(x2)
avg_pool_1 = GlobalAveragePooling1D()(x1)
max_pool_1 = GlobalMaxPooling1D()(x1)
avg_pool_2 = GlobalAveragePooling1D()(x2)
max_pool_2 = GlobalMaxPooling1D()(x2)
# 2. ESIM
e = Dot(axes=2)([x1, x2])
e1 = Softmax(axis=2)(e)
e2 = Softmax(axis=1)(e)
e1 = Lambda(K.expand_dims, arguments={'axis' : 3})(e1)
e2 = Lambda(K.expand_dims, arguments={'axis' : 3})(e2)
_x1 = Lambda(K.expand_dims, arguments={'axis' : 1})(x2)
_x1 = Multiply()([e1, _x1])
_x1 = Lambda(K.sum, arguments={'axis' : 2})(_x1)
_x2 = Lambda(K.expand_dims, arguments={'axis' : 2})(x1)
_x2 = Multiply()([e2, _x2])
_x2 = Lambda(K.sum, arguments={'axis' : 1})(_x2)
m1 = Concatenate()([x1, _x1, Subtract()([x1, _x1]), Multiply()([x1, _x1])])
m2 = Concatenate()([x2, _x2, Subtract()([x2, _x2]), Multiply()([x2, _x2])])
y1 = Bidirectional(LSTM(40, return_sequences=True))(m1)
y2 = Bidirectional(LSTM(40, return_sequences=True))(m2)
mx1 = Lambda(K.max, arguments={'axis' : 1})(y1)
av1 = Lambda(K.mean, arguments={'axis' : 1})(y1)
mx2 = Lambda(K.max, arguments={'axis' : 1})(y2)
av2 = Lambda(K.mean, arguments={'axis' : 1})(y2)
# 3. feats
inp3 = Input(shape=(feature_size,))
# x3 = BatchNormalization()(inp3)
x3 = Dense(256, activation="relu")(inp3)
# 4. concat
conc = concatenate([atten_x1, avg_pool_1, max_pool_1, atten_x2, avg_pool_2, max_pool_2, mx1, av1, mx2, av2, x3])
# conc = concatenate([avg_pool_1, avg_pool_2])
conc = Dropout(0.2)(conc)
conc = BatchNormalization()(conc)
conc = Dense(256, activation="relu")(conc)
conc = Dropout(0.2)(conc)
conc = Dense(64, activation="relu")(conc)
conc = Dropout(0.2)(conc)
outp = Dense(2, activation="sigmoid")(conc)
model_nn = Model(inputs=[inp1, inp2, inp3], outputs=outp)
model_nn.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model_nn
def train_model(self, sentences_pair, feat, scores, embedding_meta_data, model_save_directory='./'):
tokenizer, embedding_matrix = embedding_meta_data['tokenizer'], embedding_meta_data['embedding_matrix']
train_data_x1, train_data_x2, train_scores, leaks_train, feat_train, \
val_data_x1, val_data_x2, val_scores, leaks_val, feat_val = embedding.create_train_dev_set(tokenizer,
sentences_pair, feat,
scores,
self.max_sequence_length,
self.validation_split_ratio)
if train_data_x1 is None:
print("-----Failure: Unable to train model-----")
return None
nb_words = len(tokenizer.word_index) + 1
# Creating word embedding layer
embedding_layer = Embedding(nb_words, self.embedding_dim, weights=[embedding_matrix],
input_length=self.max_sequence_length, trainable=False)
# Creating LSTM Encoders
lstm_layer1 = Bidirectional(
LSTM(150, kernel_initializer='random_uniform', bias_initializer='zeros', activation='sigmoid',
kernel_regularizer=l2(0.001), recurrent_regularizer=l2(0.001), bias_regularizer=l2(0.001)))
lstm_layer2 = Bidirectional(
LSTM(150, kernel_initializer='random_uniform', bias_initializer='zeros', activation='sigmoid',
kernel_regularizer=l2(0.001), recurrent_regularizer=l2(0.001), bias_regularizer=l2(0.001)))
# lstm_layer1 = Bidirectional(
# LSTM(150, kernel_initializer='random_uniform', bias_initializer='zeros', activation='sigmoid'))
# lstm_layer2 = Bidirectional(
# LSTM(150, kernel_initializer='random_uniform', bias_initializer='zeros', activation='sigmoid'))
# Setting LSTM Encoder layer for Second Sentence
sequence_2_input = Input(shape=(self.max_sequence_length,), dtype='int32') # Input 1
embedded_sequences_2 = embedding_layer(sequence_2_input)
x2 = lstm_layer2(embedded_sequences_2)
x2 = BatchNormalization()(x2)
x2 = Dropout(0.4)(x2)
x2 = Dense(50, activation='sigmoid')(x2)
# Setting LSTM Encoder layer for First Sentence
sequence_1_input = Input(shape=(self.max_sequence_length,), dtype='int32') # Input 2
embedded_sequences_1 = embedding_layer(sequence_1_input)
x1 = Subtract()([embedded_sequences_1, embedded_sequences_2]) # dist = v1 - v2
x1 = lstm_layer1(x1)
x1 = BatchNormalization()(x1)
x1 = Dropout(0.4)(x1)
x1 = Dense(50, activation='sigmoid')(x1)
# Create feature engineering input
feat_input = Input(shape=(5,)) # Input 3
feat_dense = Dense(125, activation='sigmoid')(feat_input)
feat_dense = Dense(125, activation='sigmoid')(feat_dense)
# feat_dense = Dense(125, activation='sigmoid')(feat_dense)
# feat_dense = Dense(125, activation='sigmoid')(feat_dense)
feat_dense = Dense(125, activation='sigmoid', kernel_regularizer=l2(0.01), bias_regularizer=l2(0.01))(feat_dense)
feat_dense = Dense(125, activation='sigmoid', kernel_regularizer=l2(0.01), bias_regularizer=l2(0.01))(feat_dense)
# Creating leaks input
leaks_input = Input(shape=(leaks_train.shape[1],)) # Input 4
leaks_dense_layer = Dense(self.number_dense_units, activation=self.activation_function)
leaks_dense = leaks_dense_layer(leaks_input)
leaks_dense = Dense(50, activation='sigmoid')(leaks_dense)
# Merging two LSTM encodes vectors from sentences to
# pass it to dense layer applying dropout and batch normalisation
merged = concatenate([x1, x2, feat_dense, leaks_dense])
merged = Dense(125, activation='sigmoid')(merged)
# merged = Dense(125, activation='sigmoid')(merged)
# merged = Dense(125, activation='sigmoid')(merged)
merged = Dense(125, activation='sigmoid', kernel_regularizer=l2(0.01), bias_regularizer=l2(0.01))(merged)
merged = Dense(125, activation='sigmoid', kernel_regularizer=l2(0.01), bias_regularizer=l2(0.01))(merged)
merged = Dense(25, activation='sigmoid')(merged)
merged = BatchNormalization()(merged)
merged = Dropout(0.5)(merged)
# preds = Dense(3, activation='softmax')(merged) # 3 classes
preds = Dense(5, activation='softmax')(merged) # 5 classes
model = Model(inputs=[sequence_2_input, sequence_1_input, feat_input, leaks_input], outputs=preds)
opt = keras.optimizers.Adagrad(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['acc'])
STAMP = 'lstm_%d_%d_%.2f_%.2f' % (
self.number_lstm_units, self.number_dense_units, self.rate_drop_lstm, self.rate_drop_dense)
checkpoint_dir = model_save_directory + 'checkpoints/' + str(int(time.time())) + '/'
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
bst_model_path = checkpoint_dir + STAMP + '.h5'
model_checkpoint = ModelCheckpoint(bst_model_path, save_best_only=True, save_weights_only=False)
tensorboard = TensorBoard(log_dir=checkpoint_dir + "logs/{}".format(time.time()))
model.fit([train_data_x1, train_data_x2, feat_train, leaks_train], train_scores,
validation_data=([val_data_x1, val_data_x2, feat_val, leaks_val], val_scores),
epochs=150, batch_size=128, shuffle=True,
callbacks=[model_checkpoint, tensorboard])
return bst_model_path
embedding_layer = Embedding(len(word2idx),
embedding_size,
input_length=seq_length,
weights=[word_embeddings],
trainable=False)
source = embedding_layer(source_input)
target = embedding_layer(target_input)
mask = Masking()
lstm = LSTM(filter_num)
source_lstm = lstm(mask(source))
target_lstm = lstm(mask(target))
abs = Lambda(lambda x: kb.abs(x))
h_sub = abs(Subtract()([source_lstm, target_lstm]))
h_mul = Multiply()([source_lstm, target_lstm])
h_conc = Concatenate()([h_sub, h_mul])
logits = Dense(class_num, activation='softmax')(h_conc)
max_dev_pearson = 0.
max_test_pearson = 0.
model = Model(inputs=[source_input, target_input], outputs=logits)
model.compile(optimizer='Adam', loss=kl_distance, metrics=[pearson])
for epoch in range(epochs_num):
print('epoch num %s ' % epoch)
model.fit([train_sources, train_targets],
train_score_probs,
epochs=1,
source = embedding_layer(source_input)
target = embedding_layer(target_input)
avg_pool = AveragePooling1D(pool_size=seq_length)
reshape = Reshape([filter_num])
avg_source = reshape(avg_pool(source))
avg_target = reshape(avg_pool(target))
w = Dense(filter_num)
avg_source = w(avg_source)
avg_target = w(avg_target)
abs = Lambda(lambda x: kb.abs(x))
h_sub = abs(Subtract()([avg_source, avg_target]))
h_mul = Multiply()([avg_source, avg_target])
h_conc = Concatenate()([h_sub, h_mul])
logits = Dense(class_num, activation='softmax')(h_conc)
max_dev_pearson = 0.
max_test_pearson = 0.
model = Model(inputs=[source_input, target_input], outputs=logits)
model.compile(optimizer='Adam', loss=kl_distance, metrics=[pearson])
for epoch in range(epochs_num):
print('epoch num %s ' % epoch)
model.fit([train_sources, train_targets],
train_score_probs,
epochs=1,
conv2 = Conv2D(8, kernel_size=3, activation='relu')(conv1)
flatten = Flatten()(conv2)
V_stream_dense1 = Dense(4, activation='relu')(flatten)
#V_stream_dense2 = Dense(16, activation = 'relu')(V_stream_dense1)
V_stream_out = Dense(1, activation='linear')(V_stream_dense1)
V_stream = reshape_layer(V_stream_out, 4)
A_stream_dense1 = Dense(4, activation='relu')(flatten)
#A_stream_dense2 = Dense(16, activation = 'relu')(A_stream_dense1)
A_stream_out = Dense(4, activation='linear')(A_stream_dense1)
A_mean = Lambda(lambda x: K.mean(x, axis=1, keepdims=True))(A_stream_out)
#A_mean = reshape_layer(A_mean, 4)
A_stream = Subtract()([A_stream_out, A_mean])
Q_stream = Add()([V_stream, A_stream])
sgd = optimizers.SGD(lr=0.00001, clipnorm=10)
model = Model(inputs=[input_layer], outputs=[Q_stream])
model.load_weights('snake.h5')
model.compile(loss='mean_squared_error', optimizer=sgd)
print(model.summary())
model_copy = keras.models.clone_model(model)
model_copy.compile(optimizer=sgd, loss='mse')
model_copy.set_weights(model.get_weights())
# e greedy policy
def learn_embeddings(self, graphs):
self._node_num = graphs[0].number_of_nodes()
t1 = time()
###################################
# TensorFlow wizardry
config = tf.ConfigProto()
# Don't pre-allocate memory; allocate as-needed
config.gpu_options.allow_growth = True
# Only allow a total of half the GPU memory to be allocated
config.gpu_options.per_process_gpu_memory_fraction = 0.1
# Create a session with the above options specified.
KBack.tensorflow_backend.set_session(tf.Session(config=config))
###################################
# Generate encoder, decoder and autoencoder
self._num_iter = self._n_iter
self._encoder = get_encoder(self._node_num, self._d,
self._n_units,
self._nu1, self._nu2,
self._actfn)
self._decoder = get_decoder(self._node_num, self._d,
self._n_units,
self._nu1, self._nu2,
self._actfn)
self._autoencoder = get_autoencoder(self._encoder, self._decoder)
# Initialize self._model
# Input
x_in = Input(shape=(self._node_num,), name='x_in')
x_pred = Input(
shape=(self._node_num,),
name='x_pred'
)
# Process inputs
[x_hat, y] = self._autoencoder(x_in)
# Outputs
x_diff = Subtract()([x_hat, x_in])
# Objectives
def weighted_mse_x(y_true, y_pred):
""" Hack: This fn doesn't accept additional arguments.
We use y_true to pass them.
y_pred: Contains x_hat - x_pred
y_true: Contains b
"""
return KBack.sum(
KBack.square(y_pred * y_true[:, 0:self._node_num]),
axis=-1
)
# Model
self._model = Model(input=[x_in, x_pred], output=x_diff)
sgd = SGD(lr=self._xeta, decay=1e-5, momentum=0.99, nesterov=True)
adam = Adam(lr=self._xeta, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
self._model.compile(optimizer=sgd, loss=weighted_mse_x)
# tensorboard = TensorBoard(log_dir="logs/{}".format(time()))
# pdb.set_trace()
history = self._model.fit_generator(
generator=batch_generator_dynae(
graphs,
self._beta,
self._n_batch,
self._n_prev_graphs,
True
),
nb_epoch=self._num_iter,
samples_per_epoch=(
graphs[0].number_of_nodes() * self._n_prev_graphs
) // self._n_batch,
verbose=1
# callbacks=[tensorboard]
)
loss = history.history['loss']
# Get embedding for all points
if loss[0] == np.inf or np.isnan(loss[0]):
print('Model diverged. Assigning random embeddings')
self._Y = np.random.randn(self._node_num, self._d)
else:
self._Y, self._next_adj = model_batch_predictor_dynae(
self._autoencoder,
graphs[len(graphs) - self._n_prev_graphs:],
self._n_batch
)
t2 = time()
# Save the autoencoder and its weights
if self._weightfile is not None:
saveweights(self._encoder, self._weightfile[0])
saveweights(self._decoder, self._weightfile[1])
if self._modelfile is not None:
savemodel(self._encoder, self._modelfile[0])
savemodel(self._decoder, self._modelfile[1])
if self._savefilesuffix is not None:
saveweights(self._encoder,
'encoder_weights_' + self._savefilesuffix + '.hdf5')
saveweights(self._decoder,
'decoder_weights_' + self._savefilesuffix + '.hdf5')
savemodel(self._encoder,
'encoder_model_' + self._savefilesuffix + '.json')
savemodel(self._decoder,
'decoder_model_' + self._savefilesuffix + '.json')
# Save the embedding
np.savetxt('embedding_' + self._savefilesuffix + '.txt',
self._Y)
np.savetxt('next_pred_' + self._savefilesuffix + '.txt',
self._next_adj)
return self._Y, (t2 - t1)
def build_model(self, n_features):
"""
The method builds a new member of the ensemble and returns it.
"""
# derived parameters
self.hyperparameters['n_members'] = self.hyperparameters[
'n_segments'] * self.hyperparameters['n_members_segment']
# initialize optimizer and early stopping
self.optimizer = Adam(lr=self.hyperparameters['lr'],
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.,
amsgrad=False)
self.es = EarlyStopping(monitor=f'val_{self.loss_name}',
min_delta=0.0,
patience=self.hyperparameters['patience'],
verbose=1,
mode='min',
restore_best_weights=True)
inputs = Input(shape=(n_features, ))
h = GaussianNoise(self.hyperparameters['noise_in'],
name='noise_input')(inputs)
for i in range(self.hyperparameters['layers']):
h = Dense(self.hyperparameters['neurons'],
activation=self.hyperparameters['activation'],
kernel_regularizer=regularizers.l1_l2(
self.hyperparameters['l1_hidden'],
self.hyperparameters['l2_hidden']),
kernel_initializer='random_uniform',
bias_initializer='random_uniform',
name=f'hidden_{i}')(h)
h = Dropout(self.hyperparameters['dropout'],
name=f'hidden_dropout_{i}')(h)
median = Dense(1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
self.hyperparameters['l1_out'],
self.hyperparameters['l2_out']),
kernel_initializer='random_uniform',
bias_initializer='random_uniform',
name='median')(h)
qp1 = Dense(1,
activation='relu',
kernel_regularizer=regularizers.l1_l2(
self.hyperparameters['l1_out'],
self.hyperparameters['l2_out']),
kernel_initializer='random_uniform',
bias_initializer='random_uniform',
name='qp1')(h)
qp2 = Dense(1,
activation='relu',
kernel_regularizer=regularizers.l1_l2(
self.hyperparameters['l1_out'],
self.hyperparameters['l2_out']),
kernel_initializer='random_uniform',
bias_initializer='random_uniform',
name='qp2')(h)
qm1 = Dense(1,
activation='relu',
kernel_regularizer=regularizers.l1_l2(
self.hyperparameters['l1_out'],
self.hyperparameters['l2_out']),
kernel_initializer='random_uniform',
bias_initializer='random_uniform',
name='qm1')(h)
qm2 = Dense(1,
activation='relu',
kernel_regularizer=regularizers.l1_l2(
self.hyperparameters['l1_out'],
self.hyperparameters['l2_out']),
kernel_initializer='random_uniform',
bias_initializer='random_uniform',
name='qm2')(h)
qp1 = Add()([median, qp1])
qp2 = Add()([qp1, qp2])
qm1 = Subtract()([median, qm1])
qm2 = Subtract()([qm1, qm2])
out = Concatenate()([qm2, qm1, median, qp1, qp2])
out = GaussianNoise(self.hyperparameters['noise_out'],
name='noise_out')(out)
model = Model(inputs=inputs, outputs=out)
return model
def dd_net(ldct_img, is_training=True):
net = ldct_img
num_filter = 16
# ---A1 Layer-----------------------
h_conv1 = Conv2D(16, (7, 7), padding='same', use_bias=True,
strides=(1, 1))(net)
a1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(h_conv1)
# images 256 X 256
d1 = denseblock(a1)
a1 = BatchNormalization()(d1)
a1 = Activation('relu')(a1)
h_conv1_T = Conv2D(16, (1, 1), strides=(1, 1), use_bias=True)(a1)
# ----A2 Layer---------------------
a2 = MaxPooling2D((2, 2), strides=(2, 2), padding='same')(h_conv1_T)
# images 128 X 128 d
d2 = denseblock(a2)
a2 = BatchNormalization()(d2)
a2 = Activation('relu')(a2)
h_conv2_T = Conv2D(16, (1, 1), strides=(1, 1), use_bias=True)(a2)
# images 128 X 128
# # ----A3 Layer----------------------
a3 = MaxPooling2D((2, 2), strides=(2, 2), padding='same')(h_conv2_T)
# images 64 X 64
d3 = denseblock(a3)
a3 = BatchNormalization()(d3)
a3 = Activation('relu')(a3)
h_conv3_T = Conv2D(16, (1, 1), strides=(1, 1), use_bias=True)(a3)
# ----A4 Layer----------------------
a4 = MaxPooling2D((2, 2), strides=(2, 2), padding='same')(h_conv3_T)
# images 32 X 3
d4 = denseblock(a4)
a4 = BatchNormalization()(d4)
a4 = Activation('relu')(a4)
h_conv4_T = Conv2D(16, (1, 1), strides=(1, 1), use_bias=True)(a4)
# #----B1 Layer-----------------------
b1 = UpSampling2D((2, 2), interpolation="nearest")(h_conv4_T)
# images 64 X 64
b1 = concatenate([b1, h_conv3_T])
b1 = Conv2DTranspose(num_filter * 2, (5, 5),
padding='same',
strides=(1, 1))(b1)
b1 = Activation('relu')(b1)
b1 = BatchNormalization()(b1)
b1 = Conv2DTranspose(16, (1, 1), padding='same', strides=(1, 1))(b1)
b1 = Activation('relu')(b1)
b1 = BatchNormalization()(b1)
# #----B2 Layer-----------------------
b2 = UpSampling2D((2, 2), interpolation="nearest")(b1)
# images 128 X 128
b2 = concatenate([b2, h_conv2_T])
b2 = Conv2DTranspose(num_filter * 2, (5, 5),
padding='same',
strides=(1, 1))(b2)
b2 = Activation('relu')(b2)
b2 = BatchNormalization()(b2)
b2 = Conv2DTranspose(16, (1, 1), padding='same', strides=(1, 1))(b2)
b2 = Activation('relu')(b2)
b2 = BatchNormalization()(b2)
#----B3 Layer------------------------conv6
b3 = UpSampling2D((2, 2), interpolation="nearest")(b2)
# images 256 X 256
b3 = concatenate([b3, h_conv1_T])
b3 = Conv2DTranspose(num_filter * 2, (5, 5),
padding='same',
strides=(1, 1))(b3)
b3 = Activation('relu')(b3)
b3 = BatchNormalization()(b3)
b3 = Conv2DTranspose(16, (1, 1), padding='same', strides=(1, 1))(b3)
b3 = Activation('relu')(b3)
b3 = BatchNormalization()(b3)
#----B4 Layer-------------------------
b4 = UpSampling2D((2, 2), interpolation="nearest")(b3)
# images 512 X 512
b4 = concatenate([b4, h_conv1])
b4 = Conv2DTranspose(num_filter * 2, (5, 5),
padding='same',
strides=(1, 1))(b4)
b4 = Activation('relu')(b4)
# b4 = BatchNormalization()(b4)
output_img = Conv2DTranspose(1, (1, 1), strides=(1, 1))(b4)
# output_img = Activation('relu')(output_img) # in paper but DIDN'T CONVERGE
# ------ end B4 layer
denoised_image = Subtract()([net, output_img])
return denoised_image
def __init__(self, **kwargs):
super(ScaleShift, self).__init__(**kwargs)
def call(self, inputs):
z, shift, log_scale = inputs
z = K.exp(log_scale) * z + shift
logdet = -K.sum(K.mean(log_scale, 0))
self.add_loss(logdet)
return z
z_shift = Dense(z_dim)(x)
z_log_scale = Dense(z_dim)(x)
u = Lambda(lambda z: K.random_normal(shape=K.shape(z)))(z_shift)
z = ScaleShift()([u, z_shift, z_log_scale])
x_recon = decoder([y, z])
x_out = Subtract()([x_in, x_recon])
recon_loss = 0.5 * K.sum(K.mean(x_out**2, 0)) + 0.5 * np.log(2*np.pi) * np.prod(K.int_shape(x_out)[1:])
z_loss = 0.5 * K.sum(K.mean(z**2, 0)) - 0.5 * K.sum(K.mean(u**2, 0))
vae_loss = recon_loss + z_loss
vae = Model([x_in, y], x_out)
vae.add_loss(vae_loss)
vae.compile(optimizer=Adam(1e-4))
def sample(path):
n = 9
figure = np.zeros((img_dim*n, img_dim*n, 3))
for i in range(n):
for j in range(n):
def build_pconv_unet(self, train_bn=True, lr=0.0002):
# ENCODER
def encoder_layer(img_in, mask_in, filters, kernel_size, bn=True):
conv, mask = PConv2D(filters,
kernel_size,
strides=2,
padding='same')([img_in, mask_in])
if bn:
conv = BatchNormalization(name='EncBN' +
str(encoder_layer.counter))(
conv, training=train_bn)
conv = Activation('relu')(conv)
encoder_layer.counter += 1
return conv, mask
# DECODER
def decoder_layer(img_in,
mask_in,
e_conv,
e_mask,
filters,
kernel_size,
bn=True):
up_img = UpSampling2D(size=(2, 2))(img_in)
up_mask = UpSampling2D(size=(2, 2))(mask_in)
concat_img = Concatenate(axis=3)([e_conv, up_img])
concat_mask = Concatenate(axis=3)([e_mask, up_mask])
conv, mask = PConv2D(filters, kernel_size,
padding='same')([concat_img, concat_mask])
if bn:
conv = BatchNormalization()(conv)
conv = LeakyReLU(alpha=0.2)(conv)
return conv, mask
# Setup the model inputs / outputs
with tf.device("/cpu:0"):
# INPUTS
inputs_img = Input((self.img_rows, self.img_cols, 3))
inputs_mask = Input((self.img_rows, self.img_cols, 3))
encoder_layer.counter = 0
e_conv1, e_mask1 = encoder_layer(inputs_img,
inputs_mask,
96,
7,
bn=False) # 64*1.5
e_conv2, e_mask2 = encoder_layer(e_conv1, e_mask1, 192,
5) # 128*1.5
e_conv3, e_mask3 = encoder_layer(e_conv2, e_mask2, 384,
5) # 256*1.5
e_conv4, e_mask4 = encoder_layer(e_conv3, e_mask3, 384, 3)
e_conv5, e_mask5 = encoder_layer(e_conv4, e_mask4, 384, 3)
e_conv6, e_mask6 = encoder_layer(e_conv5, e_mask5, 384, 3)
e_conv7, e_mask7 = encoder_layer(e_conv6, e_mask6, 384, 3)
e_conv8, e_mask8 = encoder_layer(e_conv7, e_mask7, 384, 3)
d_conv9, d_mask9 = decoder_layer(e_conv8, e_mask8, e_conv7,
e_mask7, 384, 3)
d_conv10, d_mask10 = decoder_layer(d_conv9, d_mask9, e_conv6,
e_mask6, 384, 3)
d_conv11, d_mask11 = decoder_layer(d_conv10, d_mask10, e_conv5,
e_mask5, 384, 3)
d_conv12, d_mask12 = decoder_layer(d_conv11, d_mask11, e_conv4,
e_mask4, 384, 3)
d_conv13, d_mask13 = decoder_layer(d_conv12, d_mask12, e_conv3,
e_mask3, 384, 3)
d_conv14, d_mask14 = decoder_layer(d_conv13, d_mask13, e_conv2,
e_mask2, 192, 3)
d_conv15, d_mask15 = decoder_layer(d_conv14, d_mask14, e_conv1,
e_mask1, 96, 3)
d_conv16, d_mask16 = decoder_layer(d_conv15,
d_mask15,
inputs_img,
inputs_mask,
3,
3,
bn=False)
x = Conv2D(3, 1, activation='sigmoid')(d_conv16)
ones = Lambda(lambda x: K.expand_dims(K.ones(K.int_shape(x)[1:]), 0
))(inputs_mask)
in_mask = Subtract()([ones, inputs_mask])
x_inmask = Multiply(name="in_mask")([x, in_mask])
x_outmask = Multiply(name="out_mask")([inputs_img, inputs_mask])
outputs = Add(name="last")([x_inmask, x_outmask])
cpu_model = Model(inputs=[inputs_img, inputs_mask],
outputs=outputs)
model = keras.utils.multi_gpu_model(cpu_model, gpus=4)
# Compile the model
model.compile(optimizer=Adam(lr=lr), loss=self.loss_total(inputs_mask))
return model
def build_model(image_size,
n_classes,
mode='training',
l2_regularization=0.0,
min_scale=0.1,
max_scale=0.9,
scales=None,
aspect_ratios_global=[0.5, 1.0, 2.0],
aspect_ratios_per_layer=None,
two_boxes_for_ar1=True,
steps=None,
offsets=None,
clip_boxes=False,
variances=[1.0, 1.0, 1.0, 1.0],
coords='centroids',
normalize_coords=False,
subtract_mean=None,
divide_by_stddev=None,
swap_channels=False,
confidence_thresh=0.01,
iou_threshold=0.45,
top_k=200,
nms_max_output_size=400,
return_predictor_sizes=False):
'''
Build a Keras model with SSD architecture, see references.
The model consists of convolutional feature layers and a number of convolutional
predictor layers that take their input from different feature layers.
The model is fully convolutional.
The implementation found here is a smaller version of the original architecture
used in the paper (where the base network consists of a modified VGG-16 extended
by a few convolutional feature layers), but of course it could easily be changed to
an arbitrarily large SSD architecture by following the general design pattern used here.
This implementation has 7 convolutional layers and 4 convolutional predictor
layers that take their input from layers 4, 5, 6, and 7, respectively.
Most of the arguments that this function takes are only needed for the anchor
box layers. In case you're training the network, the parameters passed here must
be the same as the ones used to set up `SSDBoxEncoder`. In case you're loading
trained weights, the parameters passed here must be the same as the ones used
to produce the trained weights.
Some of these arguments are explained in more detail in the documentation of the
`SSDBoxEncoder` class.
Note: Requires Keras v2.0 or later. Training currently works only with the
TensorFlow backend (v1.0 or later).
Arguments:
image_size (tuple): The input image size in the format `(height, width, channels)`.
n_classes (int): The number of positive classes, e.g. 20 for Pascal VOC, 80 for MS COCO.
mode (str, optional): One of 'training', 'inference' and 'inference_fast'. In 'training' mode,
the model outputs the raw prediction tensor, while in 'inference' and 'inference_fast' modes,
the raw predictions are decoded into absolute coordinates and filtered via confidence thresholding,
non-maximum suppression, and top-k filtering. The difference between latter two modes is that
'inference' follows the exact procedure of the original Caffe implementation, while
'inference_fast' uses a faster prediction decoding procedure.
l2_regularization (float, optional): The L2-regularization rate. Applies to all convolutional layers.
min_scale (float, optional): The smallest scaling factor for the size of the anchor boxes as a fraction
of the shorter side of the input images.
max_scale (float, optional): The largest scaling factor for the size of the anchor boxes as a fraction
of the shorter side of the input images. All scaling factors between the smallest and the
largest will be linearly interpolated. Note that the second to last of the linearly interpolated
scaling factors will actually be the scaling factor for the last predictor layer, while the last
scaling factor is used for the second box for aspect ratio 1 in the last predictor layer
if `two_boxes_for_ar1` is `True`.
scales (list, optional): A list of floats containing scaling factors per convolutional predictor layer.
This list must be one element longer than the number of predictor layers. The first `k` elements are the
scaling factors for the `k` predictor layers, while the last element is used for the second box
for aspect ratio 1 in the last predictor layer if `two_boxes_for_ar1` is `True`. This additional
last scaling factor must be passed either way, even if it is not being used. If a list is passed,
this argument overrides `min_scale` and `max_scale`. All scaling factors must be greater than zero.
aspect_ratios_global (list, optional): The list of aspect ratios for which anchor boxes are to be
generated. This list is valid for all predictor layers. The original implementation uses more aspect ratios
for some predictor layers and fewer for others. If you want to do that, too, then use the next argument instead.
aspect_ratios_per_layer (list, optional): A list containing one aspect ratio list for each predictor layer.
This allows you to set the aspect ratios for each predictor layer individually. If a list is passed,
it overrides `aspect_ratios_global`.
two_boxes_for_ar1 (bool, optional): Only relevant for aspect ratio lists that contain 1. Will be ignored otherwise.
If `True`, two anchor boxes will be generated for aspect ratio 1. The first will be generated
using the scaling factor for the respective layer, the second one will be generated using
geometric mean of said scaling factor and next bigger scaling factor.
steps (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be
either ints/floats or tuples of two ints/floats. These numbers represent for each predictor layer how many
pixels apart the anchor box center points should be vertically and horizontally along the spatial grid over
the image. If the list contains ints/floats, then that value will be used for both spatial dimensions.
If the list contains tuples of two ints/floats, then they represent `(step_height, step_width)`.
If no steps are provided, then they will be computed such that the anchor box center points will form an
equidistant grid within the image dimensions.
offsets (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be
either floats or tuples of two floats. These numbers represent for each predictor layer how many
pixels from the top and left boarders of the image the top-most and left-most anchor box center points should be
as a fraction of `steps`. The last bit is important: The offsets are not absolute pixel values, but fractions
of the step size specified in the `steps` argument. If the list contains floats, then that value will
be used for both spatial dimensions. If the list contains tuples of two floats, then they represent
`(vertical_offset, horizontal_offset)`. If no offsets are provided, then they will default to 0.5 of the step size,
which is also the recommended setting.
clip_boxes (bool, optional): If `True`, clips the anchor box coordinates to stay within image boundaries.
variances (list, optional): A list of 4 floats >0. The anchor box offset for each coordinate will be divided by
its respective variance value.
coords (str, optional): The box coordinate format to be used internally by the model (i.e. this is not the input format
of the ground truth labels). Can be either 'centroids' for the format `(cx, cy, w, h)` (box center coordinates, width,
and height), 'minmax' for the format `(xmin, xmax, ymin, ymax)`, or 'corners' for the format `(xmin, ymin, xmax, ymax)`.
normalize_coords (bool, optional): Set to `True` if the model is supposed to use relative instead of absolute coordinates,
i.e. if the model predicts box coordinates within [0,1] instead of absolute coordinates.
subtract_mean (array-like, optional): `None` or an array-like object of integers or floating point values
of any shape that is broadcast-compatible with the image shape. The elements of this array will be
subtracted from the image pixel intensity values. For example, pass a list of three integers
to perform per-channel mean normalization for color images.
divide_by_stddev (array-like, optional): `None` or an array-like object of non-zero integers or
floating point values of any shape that is broadcast-compatible with the image shape. The image pixel
intensity values will be divided by the elements of this array. For example, pass a list
of three integers to perform per-channel standard deviation normalization for color images.
swap_channels (list, optional): Either `False` or a list of integers representing the desired order in which the input
image channels should be swapped.
confidence_thresh (float, optional): A float in [0,1), the minimum classification confidence in a specific
positive class in order to be considered for the non-maximum suppression stage for the respective class.
A lower value will result in a larger part of the selection process being done by the non-maximum suppression
stage, while a larger value will result in a larger part of the selection process happening in the confidence
thresholding stage.
iou_threshold (float, optional): A float in [0,1]. All boxes that have a Jaccard similarity of greater than `iou_threshold`
with a locally maximal box will be removed from the set of predictions for a given class, where 'maximal' refers
to the box's confidence score.
top_k (int, optional): The number of highest scoring predictions to be kept for each batch item after the
non-maximum suppression stage.
nms_max_output_size (int, optional): The maximal number of predictions that will be left over after the NMS stage.
return_predictor_sizes (bool, optional): If `True`, this function not only returns the model, but also
a list containing the spatial dimensions of the predictor layers. This isn't strictly necessary since
you can always get their sizes easily via the Keras API, but it's convenient and less error-prone
to get them this way. They are only relevant for training anyway (SSDBoxEncoder needs to know the
spatial dimensions of the predictor layers), for inference you don't need them.
Returns:
model: The Keras SSD model.
predictor_sizes (optional): A Numpy array containing the `(height, width)` portion
of the output tensor shape for each convolutional predictor layer. During
training, the generator function needs this in order to transform
the ground truth labels into tensors of identical structure as the
output tensors of the model, which is in turn needed for the cost
function.
References:
https://arxiv.org/abs/1512.02325v5
'''
n_predictor_layers = 3 # The number of predictor conv layers in the network
n_classes += 1 # Account for the background class.
l2_reg = l2_regularization # Make the internal name shorter.
img_height, img_width, img_channels = image_size[0], image_size[1], image_size[2]
############################################################################
# Get a few exceptions out of the way.
############################################################################
if aspect_ratios_global is None and aspect_ratios_per_layer is None:
raise ValueError("`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified.")
if aspect_ratios_per_layer: # TSL: None
if len(aspect_ratios_per_layer) != n_predictor_layers:
raise ValueError("It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}.".format(n_predictor_layers, len(aspect_ratios_per_layer)))
if (min_scale is None or max_scale is None) and scales is None:
raise ValueError("Either `min_scale` and `max_scale` or `scales` need to be specified.")
if scales:
if len(scales) != n_predictor_layers+1:
raise ValueError("It must be either scales is None or len(scales) == {}, but len(scales) == {}.".format(n_predictor_layers+1, len(scales)))
else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale`
scales = np.linspace(min_scale, max_scale, n_predictor_layers+1)
if len(variances) != 4: # We need one variance value for each of the four box coordinates
raise ValueError("4 variance values must be pased, but {} values were received.".format(len(variances)))
variances = np.array(variances)
if np.any(variances <= 0):
raise ValueError("All variances must be >0, but the variances given are {}".format(variances))
if (not (steps is None)) and (len(steps) != n_predictor_layers):
raise ValueError("You must provide at least one step value per predictor layer.")
if (not (offsets is None)) and (len(offsets) != n_predictor_layers):
raise ValueError("You must provide at least one offset value per predictor layer.")
############################################################################
# Compute the anchor box parameters.
############################################################################
# Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers.
if aspect_ratios_per_layer: # TSL: None
aspect_ratios = aspect_ratios_per_layer
else:
aspect_ratios = [aspect_ratios_global] * n_predictor_layers
# Compute the number of boxes to be predicted per cell for each predictor layer.
# We need this so that we know how many channels the predictor layers need to have.
if aspect_ratios_per_layer:
n_boxes = []
for ar in aspect_ratios_per_layer:
if (1 in ar) & two_boxes_for_ar1:
n_boxes.append(len(ar) + 1) # +1 for the second box for aspect ratio 1
else:
n_boxes.append(len(ar))
else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer
if (1 in aspect_ratios_global) & two_boxes_for_ar1:
n_boxes = len(aspect_ratios_global) + 1
else:
n_boxes = len(aspect_ratios_global)
n_boxes = [n_boxes] * n_predictor_layers
if steps is None:
steps = [None] * n_predictor_layers
if offsets is None:
offsets = [None] * n_predictor_layers
############################################################################
# Define functions for the Lambda layers below.
############################################################################
def identity_layer(tensor):
return tensor
def input_mean_normalization(tensor):
return tensor - np.array(subtract_mean)
def input_stddev_normalization(tensor):
return tensor / np.array(divide_by_stddev)
def input_channel_swap(tensor):
if len(swap_channels) == 3:
return K.stack([tensor[...,swap_channels[0]], tensor[...,swap_channels[1]], tensor[...,swap_channels[2]]], axis=-1)
elif len(swap_channels) == 4:
return K.stack([tensor[...,swap_channels[0]], tensor[...,swap_channels[1]], tensor[...,swap_channels[2]], tensor[...,swap_channels[3]]], axis=-1)
############################################################################
# Build the network.
############################################################################
x = Input(shape=(img_height, img_width, img_channels))
# The following identity layer is only needed so that the subsequent lambda layers can be optional.
x1 = Lambda(identity_layer, output_shape=(img_height, img_width, img_channels), name='identity_layer')(x)
if not (subtract_mean is None):
x1 = Lambda(input_mean_normalization, output_shape=(img_height, img_width, img_channels), name='input_mean_normalization')(x1)
if not (divide_by_stddev is None):
x1 = Lambda(input_stddev_normalization, output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization')(x1)
if swap_channels:
x1 = Lambda(input_channel_swap, output_shape=(img_height, img_width, img_channels), name='input_channel_swap')(x1)
conv1 = Conv2D(32, (5, 5), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1')(x1)
conv1 = BatchNormalization(axis=3, momentum=0.99, name='bn1')(conv1) # Tensorflow uses filter format [filter_height, filter_width, in_channels, out_channels], hence axis = 3
conv1 = ELU(name='elu1')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2), name='pool1')(conv1)
conv2 = Conv2D(48, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2')(pool1)
conv2 = BatchNormalization(axis=3, momentum=0.99, name='bn2')(conv2)
conv2 = ELU(name='elu2')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2), name='pool2')(conv2)
conv3 = Conv2D(64, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3')(pool2)
conv3 = BatchNormalization(axis=3, momentum=0.99, name='bn3')(conv3)
conv3 = ELU(name='elu3')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2), name='pool3')(conv3)
conv4 = Conv2D(64, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4')(pool3)
conv4 = BatchNormalization(axis=3, momentum=0.99, name='bn4')(conv4)
conv4 = ELU(name='elu4')(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2), name='pool4')(conv4)
conv5 = Conv2D(48, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5')(pool4)
base_model = Model(x, [conv3, conv4, conv5], name='base_model')
x_test = Input(shape=(img_height, img_width, img_channels), name='input_test')
x_temp = Input(shape=(img_height, img_width, img_channels), name='input_temp')
test_features = base_model(x_test)
temp_features = base_model(x_temp)
shared_features = []
for i in range(3):
test_feat = test_features[i]
temp_feat = temp_features[i]
shared_feat1 = Subtract()([test_feat, temp_feat])
shared_feat2 = Subtract()([temp_feat, test_feat])
share_feat = Concatenate()([shared_feat1, shared_feat2])
shared_features.append(share_feat)
# The next part is to add the convolutional predictor layers on top of the base network
# that we defined above. Note that I use the term "base network" differently than the paper does.
# To me, the base network is everything that is not convolutional predictor layers or anchor
# box layers. In this case we'll have four predictor layers, but of course you could
# easily rewrite this into an arbitrarily deep base network and add an arbitrary number of
# predictor layers on top of the base network by simply following the pattern shown here.
# Build the convolutional predictor layers on top of conv layers 4, 5, 6, and 7.
# We build two predictor layers on top of each of these layers: One for class prediction (classification), one for box coordinate prediction (localization)
# We precidt `n_classes` confidence values for each box, hence the `classes` predictors have depth `n_boxes * n_classes`
# We predict 4 box coordinates for each box, hence the `boxes` predictors have depth `n_boxes * 4`
# Output shape of `classes`: `(batch, height, width, n_boxes * n_classes)`
classes3 = Conv2D(n_boxes[0] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes3')(shared_features[0])
classes4 = Conv2D(n_boxes[1] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes4')(shared_features[1])
classes5 = Conv2D(n_boxes[2] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes5')(shared_features[2])
# Output shape of `boxes`: `(batch, height, width, n_boxes * 4)`
boxes3 = Conv2D(n_boxes[0] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes3')(shared_features[0])
boxes4 = Conv2D(n_boxes[1] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes4')(shared_features[1])
boxes5 = Conv2D(n_boxes[2] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes5')(shared_features[2])
# Generate the anchor boxes
# Output shape of `anchors`: `(batch, height, width, n_boxes, 8)` #TSL: 8-- cx, cy, w, h, 1.0, 1.0, 1.0, 1.0, this is a Constant tensor and will only be initialized once.
# TSL: aspect_ratios all the same as [0.5, 1, 2]
# TSL: scales=[0.08, 0.16, 0.32, 0.64, 0.96]
# TSL: the following function is used to generate anchors for each predictor. n_boxes is len(aspect_ratios) + 1 (addtional boxe)
anchors3 = AnchorBoxes(img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0],
two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0],
clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors3')(boxes3)
anchors4 = AnchorBoxes(img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1],
two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1],
clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors4')(boxes4)
anchors5 = AnchorBoxes(img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2],
two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2],
clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors5')(boxes5)
# Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, n_classes)`
# We want the classes isolated in the last axis to perform softmax on them
classes3_reshaped = Reshape((-1, n_classes), name='classes3_reshape')(classes3)
classes4_reshaped = Reshape((-1, n_classes), name='classes4_reshape')(classes4)
classes5_reshaped = Reshape((-1, n_classes), name='classes5_reshape')(classes5)
# Reshape the box coordinate predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)`
# We want the four box coordinates isolated in the last axis to compute the smooth L1 loss
boxes3_reshaped = Reshape((-1, 4), name='boxes3_reshape')(boxes3)
boxes4_reshaped = Reshape((-1, 4), name='boxes4_reshape')(boxes4)
boxes5_reshaped = Reshape((-1, 4), name='boxes5_reshape')(boxes5)
# Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)`
anchors3_reshaped = Reshape((-1, 8), name='anchors3_reshape')(anchors3)
anchors4_reshaped = Reshape((-1, 8), name='anchors4_reshape')(anchors4)
anchors5_reshaped = Reshape((-1, 8), name='anchors5_reshape')(anchors5)
# Concatenate the predictions from the different layers and the assosciated anchor box tensors
# Axis 0 (batch) and axis 2 (n_classes or 4, respectively) are identical for all layer predictions,
# so we want to concatenate along axis 1
# Output shape of `classes_concat`: (batch, n_boxes_total, n_classes)
classes_concat = Concatenate(axis=1, name='classes_concat')([classes3_reshaped,
classes4_reshaped,
classes5_reshaped])
# Output shape of `boxes_concat`: (batch, n_boxes_total, 4)
boxes_concat = Concatenate(axis=1, name='boxes_concat')([boxes3_reshaped,
boxes4_reshaped,
boxes5_reshaped])
# Output shape of `anchors_concat`: (batch, n_boxes_total, 8)
anchors_concat = Concatenate(axis=1, name='anchors_concat')([anchors3_reshaped,
anchors4_reshaped,
anchors5_reshaped])
# The box coordinate predictions will go into the loss function just the way they are,
# but for the class predictions, we'll apply a softmax activation layer first
classes_softmax = Activation('softmax', name='classes_softmax')(classes_concat)
# Concatenate the class and box coordinate predictions and the anchors to one large predictions tensor
# Output shape of `predictions`: (batch, n_boxes_total, n_classes + 4 + 8)
predictions = Concatenate(axis=2, name='predictions')([classes_softmax, boxes_concat, anchors_concat])
if mode == 'training':
model = Model(inputs=[x_test, x_temp], outputs=predictions)
elif mode == 'inference':
decoded_predictions = DecodeDetections(confidence_thresh=confidence_thresh,
iou_threshold=iou_threshold,
top_k=top_k,
nms_max_output_size=nms_max_output_size,
coords=coords,
normalize_coords=normalize_coords,
img_height=img_height,
img_width=img_width,
name='decoded_predictions')(predictions)
model = Model(inputs=[x_test, x_temp], outputs=decoded_predictions)
elif mode == 'inference_fast':
decoded_predictions = DecodeDetectionsFast(confidence_thresh=confidence_thresh,
iou_threshold=iou_threshold,
top_k=top_k,
nms_max_output_size=nms_max_output_size,
coords=coords,
normalize_coords=normalize_coords,
img_height=img_height,
img_width=img_width,
name='decoded_predictions')(predictions)
model = Model(inputs=[x_test, x_temp], outputs=decoded_predictions)
else:
raise ValueError("`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'.".format(mode))
if return_predictor_sizes:
# The spatial dimensions are the same for the `classes` and `boxes` predictor layers.
predictor_sizes = np.array([classes3._keras_shape[1:3],
classes4._keras_shape[1:3],
classes5._keras_shape[1:3]])
return model, predictor_sizes
else:
return model
input_length=wordMAXLEN,
weights=[W],
mask_zero=False,
name='embedding',
trainable=False)(word_input)
print(embed.shape)
'vector gating'
linembed = Dense(300, activation='sigmoid')(embed)
print('lineembed', linembed)
# gating = Lambda(lambda x,y: (1.0 - y)*x+y*x)(embed,linembed)
# gating=(1.0 - linembed)*embed+linembed*embed
# gating=MyLayer(300)(linembed,embed,char_embed)
t = Lambda(lambda x: K.ones_like(x, dtype='float32'))(linembed)
merged1 = merge([linembed, char_embed], name='merged1', mode='mul')
sub = Subtract()([t, linembed])
merged2 = merge([embed, sub], name='merged2', mode='mul')
gating = merge([merged1, merged2], name='gating', mode='sum')
# gating = (1.0 - linembed) *embed +linembed * char_embed
enc = Bidirectional(LSTM(150, recurrent_dropout=0.25))(gating)
fc = Dense(64, activation="relu")(enc)
dropout = Dropout(0.25)(fc)
output = Dense(3, activation='softmax')(dropout)
model = Model(inputs=[word_input, char_input],
outputs=output,
name='output')
model.compile(
def learn_embedding(self,
graph=None,
edge_f=None,
is_weighted=False,
no_python=False):
if not graph and not edge_f:
raise Exception('graph/edge_f needed')
if not graph:
graph = graph_util.loadGraphFromEdgeListTxt(edge_f)
S = nx.to_scipy_sparse_matrix(graph)
t1 = time()
S = (S + S.T) / 2
self._node_num = len(graph.nodes)
# Generate encoder, decoder and autoencoder
self._num_iter = self._n_iter
# If cannot use previous step information, initialize new models
self._encoder = get_encoder(self._node_num, self._d, self._K,
self._n_units, self._nu1, self._nu2,
self._actfn)
self._decoder = get_decoder(self._node_num, self._d, self._K,
self._n_units, self._nu1, self._nu2,
self._actfn)
self._autoencoder = get_autoencoder(self._encoder, self._decoder)
# Initialize self._model
# Input
x_in = Input(shape=(2 * self._node_num, ), name='x_in')
x1 = Lambda(lambda x: x[:, 0:self._node_num],
output_shape=(self._node_num, ))(x_in)
x2 = Lambda(lambda x: x[:, self._node_num:2 * self._node_num],
output_shape=(self._node_num, ))(x_in)
# Process inputs
[x_hat1, y1] = self._autoencoder(x1)
[x_hat2, y2] = self._autoencoder(x2)
# Outputs
x_diff1 = Subtract()([x_hat1, x1])
# merge([x_hat1, x1],
# mode=lambda ab: ab[0] - ab[1],
# output_shape=lambda L: L[1])
x_diff2 = Subtract()([x_hat2, x2])
# merge([x_hat2, x2],
# mode=lambda ab: ab[0] - ab[1],
# output_shape=lambda L: L[1])
y_diff = Subtract()([y2, y1])
# merge([y2, y1],
# mode=lambda ab: ab[0] - ab[1],
# output_shape=lambda L: L[1])
# Objectives
def weighted_mse_x(y_true, y_pred):
''' Hack: This fn doesn't accept additional arguments.
We use y_true to pass them.
y_pred: Contains x_hat - x
y_true: Contains [b, deg]
'''
return KBack.sum(KBack.square(
y_pred * y_true[:, 0:self._node_num]),
axis=-1) / y_true[:, self._node_num]
def weighted_mse_y(y_true, y_pred):
''' Hack: This fn doesn't accept additional arguments.
We use y_true to pass them.
y_pred: Contains y2 - y1
y_true: Contains s12
'''
min_batch_size = KBack.shape(y_true)[0]
return KBack.reshape(KBack.sum(KBack.square(y_pred), axis=-1),
[min_batch_size, 1]) * y_true
# Model
self._model = Model(input=x_in, output=[x_diff1, x_diff2, y_diff])
sgd = SGD(lr=self._xeta, decay=1e-5, momentum=0.99, nesterov=True)
# adam = Adam(lr=self._xeta, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
self._model.compile(
optimizer=sgd,
loss=[weighted_mse_x, weighted_mse_x, weighted_mse_y],
loss_weights=[1, 1, self._alpha])
self._model.fit_generator(
generator=batch_generator_sdne(S, self._beta, self._n_batch, True),
nb_epoch=self._num_iter,
samples_per_epoch=S.nonzero()[0].shape[0] // self._n_batch,
verbose=1)
# Get embedding for all points
self._Y = model_batch_predictor(self._autoencoder, S, self._n_batch)
t2 = time()
# Save the autoencoder and its weights
if (self._weightfile is not None):
saveweights(self._encoder, self._weightfile[0])
saveweights(self._decoder, self._weightfile[1])
if (self._modelfile is not None):
savemodel(self._encoder, self._modelfile[0])
savemodel(self._decoder, self._modelfile[1])
if (self._savefilesuffix is not None):
saveweights(self._encoder,
'encoder_weights_' + self._savefilesuffix + '.hdf5')
saveweights(self._decoder,
'decoder_weights_' + self._savefilesuffix + '.hdf5')
savemodel(self._encoder,
'encoder_model_' + self._savefilesuffix + '.json')
savemodel(self._decoder,
'decoder_model_' + self._savefilesuffix + '.json')
# Save the embedding
np.savetxt('embedding_' + self._savefilesuffix + '.txt', self._Y)
return self._Y, (t2 - t1)