def qualitative_CNN(self, vocab_size, emb_dim, max_len, nb_filters):
self.max_len = max_len
max_features = vocab_size
filter_lengths = [3, 4, 5]
print("Build model...")
self.qual_model = Graph()
self.qual_conv_set = {}
'''Embedding Layer'''
self.qual_model.add_input(
name='input', input_shape=(max_len,), dtype=int)
self.qual_model.add_node(Embedding(max_features, emb_dim, input_length=max_len, weights=self.model.nodes['sentence_embeddings'].get_weights()),
name='sentence_embeddings', input='input')
'''Convolution Layer & Max Pooling Layer'''
for i in filter_lengths:
model_internal = Sequential()
model_internal.add(
Reshape(dims=(1, max_len, emb_dim), input_shape=(max_len, emb_dim)))
self.qual_conv_set[i] = Convolution2D(nb_filters, i, emb_dim, activation="relu", weights=self.model.nodes[
'unit_' + str(i)].layers[1].get_weights())
model_internal.add(self.qual_conv_set[i])
model_internal.add(MaxPooling2D(pool_size=(max_len - i + 1, 1)))
model_internal.add(Flatten())
self.qual_model.add_node(
model_internal, name='unit_' + str(i), input='sentence_embeddings')
self.qual_model.add_output(
name='output_' + str(i), input='unit_' + str(i))
self.qual_model.compile(
'rmsprop', {'output_3': 'mse', 'output_4': 'mse', 'output_5': 'mse'})
def __init__(self, output_dimesion, vocab_size, dropout_rate, emb_dim, max_len, nb_filters, init_W=None):
self.max_len = max_len
max_features = vocab_size
vanila_dimension = 200
projection_dimension = output_dimesion
filter_lengths = [3, 4, 5]
self.model = Graph()
'''Embedding Layer'''
self.model.add_input(name='input', input_shape=(max_len,), dtype=int)
if init_W is None:
self.model.add_node(Embedding(
max_features, emb_dim, input_length=max_len), name='sentence_embeddings', input='input')
else:
self.model.add_node(Embedding(max_features, emb_dim, input_length=max_len, weights=[
init_W / 20]), name='sentence_embeddings', input='input')
'''Convolution Layer & Max Pooling Layer'''
for i in filter_lengths:
model_internal = Sequential()
model_internal.add(
Reshape(dims=(1, self.max_len, emb_dim), input_shape=(self.max_len, emb_dim)))
model_internal.add(Convolution2D(
nb_filters, i, emb_dim, activation="relu"))
model_internal.add(MaxPooling2D(
pool_size=(self.max_len - i + 1, 1)))
model_internal.add(Flatten())
self.model.add_node(model_internal, name='unit_' +
str(i), input='sentence_embeddings')
'''Dropout Layer'''
self.model.add_node(Dense(vanila_dimension, activation='tanh'),
name='fully_connect', inputs=['unit_' + str(i) for i in filter_lengths])
self.model.add_node(Dropout(dropout_rate),
name='dropout', input='fully_connect')
'''Projection Layer & Output Layer'''
self.model.add_node(Dense(projection_dimension, activation='tanh'),
name='projection', input='dropout')
# Output Layer
self.model.add_output(name='output', input='projection')
# loss function by oneself
self.model.compile('rmsprop', {'output': 'mse'})
def __init__(self, mode) :
self.io_dim = Codec.n_chars
with open('config/semantic.json') as file :
semantic_config = json.load(file)
self.feature_dim = semantic_config['vectorDim']
self.seq_len = semantic_config['seqLength']
self.mode = mode
self.model = SeqModel()
self.encoder = SeqContainer()
self.encoder.add(TimeDistributedDense(
input_dim = self.io_dim,
input_length = self.seq_len,
output_dim = self.feature_dim,
activation = 'sigmoid'))
self.encoder.add(GRU(
input_dim = self.feature_dim,
input_length = self.seq_len,
output_dim = self.feature_dim,
activation = 'sigmoid',
inner_activation = 'hard_sigmoid',
truncate_gradient = self.seq_len,
return_sequences = True))
self.encoder.add(GRU(
input_dim = self.feature_dim,
input_length = self.seq_len,
output_dim = self.feature_dim,
activation = 'sigmoid',
inner_activation = 'hard_sigmoid',
truncate_gradient = self.seq_len,
return_sequences = False))
self.model.add(self.encoder)
if mode == 'train' :
self.decoder = SeqContainer()
self.decoder.add(SimpleRNN(
input_dim = self.feature_dim,
input_length = self.seq_len,
output_dim = self.feature_dim,
activation = 'sigmoid',
truncate_gradient = self.seq_len,
return_sequences = True))
self.decoder.add(TimeDistributedDense(
input_dim = self.feature_dim,
input_length = self.seq_len,
output_dim = self.io_dim,
activation = 'sigmoid'))
self.model.add(RepeatVector(self.seq_len, input_shape = (self.feature_dim,)))
self.model.add(self.decoder)
def main():
data = load_data()
print 'building model'
SEQ_LENGTH = data.shape[1]
N_INPUT = 88
N_HIDDEN = 160
Z_SIZE = 8
net_z_given_x = SequentialLayer()
net_z_given_x.add(LSTM(N_HIDDEN, return_sequences=True,
input_length=SEQ_LENGTH, input_dim=N_INPUT))
net_z_given_x.add(LSTM(Z_SIZE*2, return_sequences=True))
net_x_given_z = SequentialLayer()
net_x_given_z.add(LSTM(N_HIDDEN, return_sequences=True,
input_length=SEQ_LENGTH, input_dim=Z_SIZE))
net_x_given_z.add(LSTM(N_INPUT*2, return_sequences=True))
sampler_z_given_x = NormalSampler(axis_to_split=2)
prob_z_given_x = NormalProbCalculator(axis_to_split=2)
prob_x_given_z = NormalProbCalculator(axis_to_split=2)
variational = VariationalWrapper(1, SEQ_LENGTH, Z_SIZE,
net_z_given_x, prob_z_given_x, sampler_z_given_x,
net_x_given_z, prob_x_given_z)
model = Sequential()
model.add(variational)
# optimizer = optimizers.SGD(lr=0.01)
optimizer = optimizers.RMSprop()
model.compile(loss='mae',
optimizer=optimizer,)
for iteration in range(500):
print iteration
model.fit(data, np.zeros((data.shape[0], SEQ_LENGTH)),
batch_size=128, nb_epoch=10)
if iteration % 10 == 0:
print "saving weights"
model.save_weights('D:/scratch/keras_model_simple.hdf5', overwrite=True)
print "done saving weights"
class Model() :
# mode = 'encode' || 'train'
def __init__(self, mode) :
self.io_dim = Codec.n_chars
with open('config/semantic.json') as file :
semantic_config = json.load(file)
self.feature_dim = semantic_config['vectorDim']
self.seq_len = semantic_config['seqLength']
self.mode = mode
self.model = SeqModel()
self.encoder = SeqContainer()
self.encoder.add(TimeDistributedDense(
input_dim = self.io_dim,
input_length = self.seq_len,
output_dim = self.feature_dim,
activation = 'sigmoid'))
self.encoder.add(GRU(
input_dim = self.feature_dim,
input_length = self.seq_len,
output_dim = self.feature_dim,
activation = 'sigmoid',
inner_activation = 'hard_sigmoid',
truncate_gradient = self.seq_len,
return_sequences = True))
self.encoder.add(GRU(
input_dim = self.feature_dim,
input_length = self.seq_len,
output_dim = self.feature_dim,
activation = 'sigmoid',
inner_activation = 'hard_sigmoid',
truncate_gradient = self.seq_len,
return_sequences = False))
self.model.add(self.encoder)
if mode == 'train' :
self.decoder = SeqContainer()
self.decoder.add(SimpleRNN(
input_dim = self.feature_dim,
input_length = self.seq_len,
output_dim = self.feature_dim,
activation = 'sigmoid',
truncate_gradient = self.seq_len,
return_sequences = True))
self.decoder.add(TimeDistributedDense(
input_dim = self.feature_dim,
input_length = self.seq_len,
output_dim = self.io_dim,
activation = 'sigmoid'))
self.model.add(RepeatVector(self.seq_len, input_shape = (self.feature_dim,)))
self.model.add(self.decoder)
def _load_weights(self, path) :
with h5py.File(path, 'r') as file :
group = file['/weights']
n_layers = group.attrs.get('n_layers')[0]
weights = []
for i in range(n_layers) :
layer_weights = file['/weights/layer_' + str(i)][()]
weights.append(layer_weights)
return weights
def load(self) :
encoder_weights = self._load_weights('data/encoder.hdf5')
self.encoder.set_weights(encoder_weights)
if self.mode == 'train' :
decoder_weights = self._load_weights('data/decoder.hdf5')
self.decoder.set_weights(decoder_weights)
def _save_weights(self, weights, path) :
with h5py.File(path, 'w') as file :
group = file.create_group('weights')
n_layers = len(weights)
group.attrs.create('n_layers', np.array([n_layers]))
for i, layer_weights in enumerate(weights) :
group.create_dataset('layer_' + str(i), data = layer_weights)
def save(self) :
if self.mode != 'train' :
raise Exception('invalid mode')
encoder_weights = self.encoder.get_weights()
decoder_weights = self.decoder.get_weights()
self._save_weights(encoder_weights, 'data/encoder.hdf5')
self._save_weights(decoder_weights, 'data/decoder.hdf5')
def compile(self) :
self.model.compile(loss = 'categorical_crossentropy', optimizer = Adadelta(clipnorm = 1.))
# in_data & out_data numpy bool array of shape (n_sample, seq_len, io_dim)
# return train (loss, accuracy)
def train(self, in_data, out_data) :
if self.mode != 'train' :
raise Exception('invalid mode')
return self.model.train_on_batch(in_data, out_data, accuracy = True)
# in_data & out_data numpy bool array of shape (n_sample, seq_len, io_dim)
# return the evaluation (loss, accuracy)
def evaluate(self, in_data, out_data) :
if self.mode != 'train' :
raise Exception('invalid mode')
return self.model.test_on_batch(in_data, out_data, accuracy = True)
# sequence : numpy bool array of shape (seq_len, io_dim)
# return : numpy float32 array of shape (feature_dim)
def encode(self, sequence) :
if self.mode != 'encode' :
raise Exception('invalid mode')
input_sequences = np.ndarray((1, self.seq_len, self.io_dim), dtype = np.bool)
input_sequences[0] = sequence
return self.model.predict(input_sequences)[0]
