def qualitative_CNN(self, vocab_size, emb_dim, max_len, nb_filters): self.max_len = max_len max_features = vocab_size filter_lengths = [3, 4, 5] print("Build model...") self.qual_model = Graph() self.qual_conv_set = {} '''Embedding Layer''' self.qual_model.add_input( name='input', input_shape=(max_len,), dtype=int) self.qual_model.add_node(Embedding(max_features, emb_dim, input_length=max_len, weights=self.model.nodes['sentence_embeddings'].get_weights()), name='sentence_embeddings', input='input') '''Convolution Layer & Max Pooling Layer''' for i in filter_lengths: model_internal = Sequential() model_internal.add( Reshape(dims=(1, max_len, emb_dim), input_shape=(max_len, emb_dim))) self.qual_conv_set[i] = Convolution2D(nb_filters, i, emb_dim, activation="relu", weights=self.model.nodes[ 'unit_' + str(i)].layers[1].get_weights()) model_internal.add(self.qual_conv_set[i]) model_internal.add(MaxPooling2D(pool_size=(max_len - i + 1, 1))) model_internal.add(Flatten()) self.qual_model.add_node( model_internal, name='unit_' + str(i), input='sentence_embeddings') self.qual_model.add_output( name='output_' + str(i), input='unit_' + str(i)) self.qual_model.compile( 'rmsprop', {'output_3': 'mse', 'output_4': 'mse', 'output_5': 'mse'})
def __init__(self, output_dimesion, vocab_size, dropout_rate, emb_dim, max_len, nb_filters, init_W=None): self.max_len = max_len max_features = vocab_size vanila_dimension = 200 projection_dimension = output_dimesion filter_lengths = [3, 4, 5] self.model = Graph() '''Embedding Layer''' self.model.add_input(name='input', input_shape=(max_len,), dtype=int) if init_W is None: self.model.add_node(Embedding( max_features, emb_dim, input_length=max_len), name='sentence_embeddings', input='input') else: self.model.add_node(Embedding(max_features, emb_dim, input_length=max_len, weights=[ init_W / 20]), name='sentence_embeddings', input='input') '''Convolution Layer & Max Pooling Layer''' for i in filter_lengths: model_internal = Sequential() model_internal.add( Reshape(dims=(1, self.max_len, emb_dim), input_shape=(self.max_len, emb_dim))) model_internal.add(Convolution2D( nb_filters, i, emb_dim, activation="relu")) model_internal.add(MaxPooling2D( pool_size=(self.max_len - i + 1, 1))) model_internal.add(Flatten()) self.model.add_node(model_internal, name='unit_' + str(i), input='sentence_embeddings') '''Dropout Layer''' self.model.add_node(Dense(vanila_dimension, activation='tanh'), name='fully_connect', inputs=['unit_' + str(i) for i in filter_lengths]) self.model.add_node(Dropout(dropout_rate), name='dropout', input='fully_connect') '''Projection Layer & Output Layer''' self.model.add_node(Dense(projection_dimension, activation='tanh'), name='projection', input='dropout') # Output Layer self.model.add_output(name='output', input='projection') # loss function by oneself self.model.compile('rmsprop', {'output': 'mse'})
def __init__(self, mode) : self.io_dim = Codec.n_chars with open('config/semantic.json') as file : semantic_config = json.load(file) self.feature_dim = semantic_config['vectorDim'] self.seq_len = semantic_config['seqLength'] self.mode = mode self.model = SeqModel() self.encoder = SeqContainer() self.encoder.add(TimeDistributedDense( input_dim = self.io_dim, input_length = self.seq_len, output_dim = self.feature_dim, activation = 'sigmoid')) self.encoder.add(GRU( input_dim = self.feature_dim, input_length = self.seq_len, output_dim = self.feature_dim, activation = 'sigmoid', inner_activation = 'hard_sigmoid', truncate_gradient = self.seq_len, return_sequences = True)) self.encoder.add(GRU( input_dim = self.feature_dim, input_length = self.seq_len, output_dim = self.feature_dim, activation = 'sigmoid', inner_activation = 'hard_sigmoid', truncate_gradient = self.seq_len, return_sequences = False)) self.model.add(self.encoder) if mode == 'train' : self.decoder = SeqContainer() self.decoder.add(SimpleRNN( input_dim = self.feature_dim, input_length = self.seq_len, output_dim = self.feature_dim, activation = 'sigmoid', truncate_gradient = self.seq_len, return_sequences = True)) self.decoder.add(TimeDistributedDense( input_dim = self.feature_dim, input_length = self.seq_len, output_dim = self.io_dim, activation = 'sigmoid')) self.model.add(RepeatVector(self.seq_len, input_shape = (self.feature_dim,))) self.model.add(self.decoder)
def main(): data = load_data() print 'building model' SEQ_LENGTH = data.shape[1] N_INPUT = 88 N_HIDDEN = 160 Z_SIZE = 8 net_z_given_x = SequentialLayer() net_z_given_x.add(LSTM(N_HIDDEN, return_sequences=True, input_length=SEQ_LENGTH, input_dim=N_INPUT)) net_z_given_x.add(LSTM(Z_SIZE*2, return_sequences=True)) net_x_given_z = SequentialLayer() net_x_given_z.add(LSTM(N_HIDDEN, return_sequences=True, input_length=SEQ_LENGTH, input_dim=Z_SIZE)) net_x_given_z.add(LSTM(N_INPUT*2, return_sequences=True)) sampler_z_given_x = NormalSampler(axis_to_split=2) prob_z_given_x = NormalProbCalculator(axis_to_split=2) prob_x_given_z = NormalProbCalculator(axis_to_split=2) variational = VariationalWrapper(1, SEQ_LENGTH, Z_SIZE, net_z_given_x, prob_z_given_x, sampler_z_given_x, net_x_given_z, prob_x_given_z) model = Sequential() model.add(variational) # optimizer = optimizers.SGD(lr=0.01) optimizer = optimizers.RMSprop() model.compile(loss='mae', optimizer=optimizer,) for iteration in range(500): print iteration model.fit(data, np.zeros((data.shape[0], SEQ_LENGTH)), batch_size=128, nb_epoch=10) if iteration % 10 == 0: print "saving weights" model.save_weights('D:/scratch/keras_model_simple.hdf5', overwrite=True) print "done saving weights"
class Model() : # mode = 'encode' || 'train' def __init__(self, mode) : self.io_dim = Codec.n_chars with open('config/semantic.json') as file : semantic_config = json.load(file) self.feature_dim = semantic_config['vectorDim'] self.seq_len = semantic_config['seqLength'] self.mode = mode self.model = SeqModel() self.encoder = SeqContainer() self.encoder.add(TimeDistributedDense( input_dim = self.io_dim, input_length = self.seq_len, output_dim = self.feature_dim, activation = 'sigmoid')) self.encoder.add(GRU( input_dim = self.feature_dim, input_length = self.seq_len, output_dim = self.feature_dim, activation = 'sigmoid', inner_activation = 'hard_sigmoid', truncate_gradient = self.seq_len, return_sequences = True)) self.encoder.add(GRU( input_dim = self.feature_dim, input_length = self.seq_len, output_dim = self.feature_dim, activation = 'sigmoid', inner_activation = 'hard_sigmoid', truncate_gradient = self.seq_len, return_sequences = False)) self.model.add(self.encoder) if mode == 'train' : self.decoder = SeqContainer() self.decoder.add(SimpleRNN( input_dim = self.feature_dim, input_length = self.seq_len, output_dim = self.feature_dim, activation = 'sigmoid', truncate_gradient = self.seq_len, return_sequences = True)) self.decoder.add(TimeDistributedDense( input_dim = self.feature_dim, input_length = self.seq_len, output_dim = self.io_dim, activation = 'sigmoid')) self.model.add(RepeatVector(self.seq_len, input_shape = (self.feature_dim,))) self.model.add(self.decoder) def _load_weights(self, path) : with h5py.File(path, 'r') as file : group = file['/weights'] n_layers = group.attrs.get('n_layers')[0] weights = [] for i in range(n_layers) : layer_weights = file['/weights/layer_' + str(i)][()] weights.append(layer_weights) return weights def load(self) : encoder_weights = self._load_weights('data/encoder.hdf5') self.encoder.set_weights(encoder_weights) if self.mode == 'train' : decoder_weights = self._load_weights('data/decoder.hdf5') self.decoder.set_weights(decoder_weights) def _save_weights(self, weights, path) : with h5py.File(path, 'w') as file : group = file.create_group('weights') n_layers = len(weights) group.attrs.create('n_layers', np.array([n_layers])) for i, layer_weights in enumerate(weights) : group.create_dataset('layer_' + str(i), data = layer_weights) def save(self) : if self.mode != 'train' : raise Exception('invalid mode') encoder_weights = self.encoder.get_weights() decoder_weights = self.decoder.get_weights() self._save_weights(encoder_weights, 'data/encoder.hdf5') self._save_weights(decoder_weights, 'data/decoder.hdf5') def compile(self) : self.model.compile(loss = 'categorical_crossentropy', optimizer = Adadelta(clipnorm = 1.)) # in_data & out_data numpy bool array of shape (n_sample, seq_len, io_dim) # return train (loss, accuracy) def train(self, in_data, out_data) : if self.mode != 'train' : raise Exception('invalid mode') return self.model.train_on_batch(in_data, out_data, accuracy = True) # in_data & out_data numpy bool array of shape (n_sample, seq_len, io_dim) # return the evaluation (loss, accuracy) def evaluate(self, in_data, out_data) : if self.mode != 'train' : raise Exception('invalid mode') return self.model.test_on_batch(in_data, out_data, accuracy = True) # sequence : numpy bool array of shape (seq_len, io_dim) # return : numpy float32 array of shape (feature_dim) def encode(self, sequence) : if self.mode != 'encode' : raise Exception('invalid mode') input_sequences = np.ndarray((1, self.seq_len, self.io_dim), dtype = np.bool) input_sequences[0] = sequence return self.model.predict(input_sequences)[0]