def RMDN_train(hidden_states, n_mixtures, input_shape, summary=True): """ Function that returns RMDN model for training. Here the recurrent layer has return_sequences=True and is stateless. :param hidden_states: dimension of the LSTM state. :param n_mixtures: number of output mixture densities. :param input_shape: the input shape like (time_steps, c3d_encoding). :param summary: optional, whether or not to print summary. :return: a Keras model. """ time_steps, _ = input_shape sequence_in = Input(shape=input_shape, name='input') state = LSTM(output_dim=hidden_states, return_sequences=True, name='recurrent_module')(sequence_in) state = Dropout(0.5)(state) # Mixture Density Inference # mixture components weights weight = TimeDistributedDense(output_dim=n_mixtures * 1, name='output_weight')(state) weight = Reshape(target_shape=(time_steps, n_mixtures))(weight) weight = Activation('softmax')(weight) weight = Reshape(target_shape=(time_steps, n_mixtures, 1))(weight) # gaussian mean mu = TimeDistributedDense(output_dim=n_mixtures * 2, name='output_mean')(state) mu = Reshape(target_shape=(time_steps, n_mixtures, 2))(mu) mu = Activation('relu')(mu) # this must become linear # variance sigma = TimeDistributedDense(output_dim=n_mixtures * 2, name='output_var')(state) sigma = Reshape(target_shape=(time_steps, n_mixtures, 2))(sigma) sigma = Lambda(lambda x: K.exp(x) + 1, output_shape=(time_steps, n_mixtures, 2))(sigma) # correlation ro = TimeDistributedDense(output_dim=n_mixtures, name='output_corr')(state) ro = Reshape(target_shape=(time_steps, n_mixtures, 1))(ro) ro = Activation('tanh')(ro) md = merge([weight, mu, sigma, ro], mode='concat', concat_axis=-1) model = Model(input=sequence_in, output=md) if summary: model.summary() return model
def create_temporal_sequential_model(): model = Sequential() model.add( GRU(32, input_shape=(timesteps, input_dim), return_sequences=True)) model.add(TimeDistributedDense(nb_classes)) model.add(Activation('softmax')) return model
def model_define(): model = Sequential() model.add( TimeDistributed(Convolution2D(nb_filter, 3, 3, border_mode="same"), input_shape=[lstm_output_size, 1, 172, 224])) model.add(TimeDistributed(Activation("relu"))) model.add(TimeDistributed(MaxPooling2D(pool_size=pool_size))) print(model.output_shape) # model.add(TimeDistributed(Convolution2D(1, 3, 3, border_mode="same"))) model.add(TimeDistributed(Activation("relu"))) model.add(TimeDistributed(MaxPooling2D(pool_size=pool_size))) print(model.output_shape) # model.add(TimeDistributed(Flatten())) print(model.output_shape) # model.add(LSTM(256, return_sequences=True, # input_shape=(lstm_output_size, 172*224))) model.add( LSTM(256, return_sequences=True, input_shape=(lstm_output_size, (172 / 4) * (224 / 4)))) print(model.output_shape) model.add(LSTM(128, return_sequences=True)) print(model.output_shape) model.add(LSTM(32, return_sequences=True)) print(model.output_shape) model.add(TimeDistributedDense(2)) model.add(Activation('linear')) print(model.output_shape) return model
def __init__(self, input_dim, maxlen, lstm_size, output_dim): Model.__init__(self) self.classifier = None self.maxlen = maxlen model = Sequential() model.add(Masking(input_shape=(None, input_dim))) # model.add(Embedding(max_features, 128, input_length=maxlen)) model.add( LSTM(lstm_size, input_dim=input_dim)) # try using a GRU instead, for fun #model.add(Dense(lstm_size, output_dim)) model.add(Dense(lstm_size, input_dim=lstm_size)) model.add(Activation('relu')) model.add(RepeatVector(maxlen)) model.add( LSTM(lstm_size, input_dim=lstm_size, return_sequences=True)) # try using a GRU instead, for fun model.add( TimeDistributedDense(output_dim=output_dim, input_dim=lstm_size)) model.add(Activation('softmax')) # try using different optimizers and different optimizer configs model.compile(loss='categorical_crossentropy', optimizer='rmsprop', sample_weight_mode="temporal") self.classifier = model
def __init__(self, input_dim=None,maxlen=None, lstm_size=None, output_dim=None, load_from_file = None, weight_file = None, temp="."): Model.__init__(self) if load_from_file is None: self.classifier = None self.maxlen=maxlen model = Sequential() model.add(Masking(input_shape=(None,input_dim))) # model.add(Embedding(max_features, 128, input_length=maxlen)) lstm = LSTM(input_dim=input_dim, output_dim=lstm_size//2,return_sequences=True) gru = GRU(input_dim=input_dim, output_dim=lstm_size//2, go_backwards=True, return_sequences=True) # original examples was 128, we divide by 2 because results will be concatenated brnn = Bidirectional(forward=lstm, backward=gru, return_sequences=True) model.add(brnn) # try using another Bidirectional RNN inside the Bidirectional RNN. Inception meets callback hell. lstm2 = LSTM(input_dim=lstm_size, output_dim=lstm_size,return_sequences=True) gru2 = GRU(input_dim=lstm_size, output_dim=lstm_size, go_backwards=True, return_sequences=True) # original examples was 128, we divide by 2 because results will be concatenated brnn2 = Bidirectional(forward=lstm2, backward=gru2, return_sequences=True) # model.add(LSTM(lstm_size, input_dim=input_dim, return_sequences=True)) # try using a GRU instead, for fun #model.add(Dense(lstm_size, output_dim)) model.add(brnn2) model.add(TimeDistributedDense(output_dim=output_dim, input_dim=lstm_size*2)) model.add(Activation('softmax')) # try using different optimizers and different optimizer configs model.compile(loss='categorical_crossentropy', optimizer='rmsprop',sample_weight_mode="temporal") self.classifier=model else: self.classifier = model_from_json(load_from_file) self.classifier.load_weights(temp + "/" + 'my_model_weights.h5')
def build(self): dim_data = self.size_of_input_data_dim nb_time_step = self.size_of_input_timesteps financial_time_series_input = Input(shape=(nb_time_step, dim_data)) lstm_layer_1 = LSTM(output_dim=nb_hidden_units, dropout_U=dropout, dropout_W=dropout, inner_activation='sigmoid', W_regularizer=l2(l2_norm_alpha), b_regularizer=l2(l2_norm_alpha), activation='tanh', return_sequences=True) lstm_layer_2 = LSTM(output_dim=nb_hidden_units, dropout_U=dropout, dropout_W=dropout, inner_activation='sigmoid', W_regularizer=l2(l2_norm_alpha), b_regularizer=l2(l2_norm_alpha), activation='tanh', return_sequences=True) h1 = lstm_layer_1(financial_time_series_input) h2 = lstm_layer_2(h1) time_series_predictions = TimeDistributedDense(1)(h2) self.model = Model( financial_time_series_input, time_series_predictions, name="deep rnn for financial time series forecasting")
def test_sequence_to_sequence(): ''' Apply a same Dense layer for each element of time dimension of the input and make predictions of the output sequence elements. This does not make use of the temporal structure of the sequence (see TimeDistributedDense for more details) ''' np.random.seed(1337) (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=500, nb_test=200, input_shape=(3, 5), output_shape=(3, 5), classification=False) model = Sequential() model.add( TimeDistributedDense(y_train.shape[-1], input_shape=(X_train.shape[1], X_train.shape[2]))) model.compile(loss='hinge', optimizer='rmsprop') history = model.fit(X_train, y_train, nb_epoch=20, batch_size=16, validation_data=(X_test, y_test), verbose=0) assert (history.history['val_loss'][-1] < 0.8)
def create_temporal_graph_model(): model = Graph() model.add_input(name='input', input_shape=(timesteps, input_dim)) model.add_node(GRU(32, return_sequences=True), name='d1', input='input') model.add_node(TimeDistributedDense(nb_classes, activation='softmax'), name='d2', input='d1') model.add_output(name='output', input='d2') return model
def get_language_model(): ''' returns language model''' language_model = Sequential() language_model.add(Embedding(vocab_size, 4096, input_length = max_caption_length)) language_model.add(GRU(output_dim=4096, return_sequences=True)) language_model.add(TimeDistributedDense(4096)) return language_model
def test_masked_temporal(): ''' Confirm that even with masking on both inputs and outputs, cross-entropies are of the expected scale. In this task, there are variable length inputs of integers from 1-9, and a random subset of unmasked outputs. Each of these outputs has a 50% probability of being the input number unchanged, and a 50% probability of being 2*input%10. The ground-truth best cross-entropy loss should, then be -log(0.5) = 0.69 ''' np.random.seed(55318) model = Sequential() model.add(Embedding(10, 20, mask_zero=True, input_length=20)) model.add(TimeDistributedDense(10)) model.add(Activation('softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', sample_weight_mode='temporal') X = np.random.random_integers(1, 9, (50000, 20)) for rowi in range(X.shape[0]): padding = np.random.random_integers(X.shape[1] / 2) X[rowi, :padding] = 0 # 50% of the time the correct output is the input. # The other 50% of the time it's 2 * input % 10 y = (X * np.random.random_integers(1, 2, X.shape)) % 10 Y = np.zeros((y.size, 10), dtype='int32') for i, target in enumerate(y.flat): Y[i, target] = 1 Y = Y.reshape(y.shape + (10, )) # Mask 50% of the outputs via sample weights sample_weight = np.random.random_integers(0, 1, y.shape) print('X shape:', X.shape) print('Y shape:', Y.shape) print('sample_weight shape:', Y.shape) history = model.fit(X, Y, validation_split=0.05, sample_weight=None, verbose=1, nb_epoch=2) ground_truth = -np.log(0.5) assert (np.abs(history.history['val_loss'][-1] - ground_truth) < 0.06)
def build_sampling_model(): print('Build model...') model = Sequential() model.add( Embedding(vocab_size, vocab_size, mask_zero=True, init='identity', batch_input_shape=(1, 1))) model.add(LSTM(512, return_sequences=True, stateful=True)) model.add(Dropout(0.2)) model.add(LSTM(512, return_sequences=True, stateful=True)) model.add(Dropout(0.2)) model.add(TimeDistributedDense(vocab_size, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='rmsprop') return model
def model_define(): model = Sequential() model.add( LSTM(256, return_sequences=True, input_shape=(lstm_output_size, 172 * 224))) print(model.output_shape) model.add(LSTM(128, return_sequences=True)) print(model.output_shape) model.add(LSTM(32, return_sequences=True)) print(model.output_shape) model.add(TimeDistributedDense(2)) model.add(Activation('linear')) print(model.output_shape) return model
def getModel(self, params, weight=None ): lstm_cell_size = params['lstm_cell_size'] print "params['embeddings_dim'] = ", params['embeddings_dim'] print "lstm_cell_size= ", lstm_cell_size inp = Input(shape=(params['inp_length'],), dtype='int32', name="inp") embedding = Embedding(input_dim = params['vocab_size']+1, output_dim = params['embeddings_dim'], input_length = params['inp_length'], dropout=0.2, mask_zero=True, trainable=True) (inp) lstm_out = LSTM(lstm_cell_size, return_sequences=True)(embedding) out = TimeDistributedDense(params['vocab_size'], activation='softmax')(lstm_out) model = Model(input=[inp], output=[out]) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'] ) print model.summary() return model
def train_breaker(datafilename, sentence_num=1000, puncs=u',,.。!!??', \ RNN=recurrent.GRU, HIDDEN_SIZE=128, EPOCH_SIZE=10, validate=True): wordtable = WordTable() wordtable.parse(datafilename, sentence_num) X, Y = [], [] for line in open(datafilename).readlines()[:sentence_num]: line = line.strip().decode('utf-8') line = re.sub(ur'(^[{0}]+)|([{0}]+$)'.format(puncs),'',line) words = wordtable.encode(re.sub(ur'[{0}]'.format(puncs),'',line)) breaks = re.sub(ur'0[{0}]+'.format(puncs),'1',re.sub(ur'[^{0}]'.format(puncs),'0',line)) if len(words) >= 30 and len(words) <= 50 and breaks.count('1') >= 4: x = np.zeros((len(words), wordtable.capacity), dtype=np.bool) y = np.zeros((len(breaks), 2), dtype=np.bool) for idx in xrange(len(words)): x[idx][words[idx]] = True y[idx][int(breaks[idx])] = True X.append(x) Y.append(y) print 'total sentence: ', len(X) if validate: # Set apart 10% for validation split_at = len(X) - len(X)/10 X_train, X_val = X[:split_at], X[split_at:] y_train, y_val = Y[:split_at], Y[split_at:] else: X_train, y_train = X, Y model = Graph() model.add_input(name='input', input_shape=(None, wordtable.capacity)) model.add_node(RNN(HIDDEN_SIZE, return_sequences=True), name='forward', input='input') model.add_node(TimeDistributedDense(2, activation='softmax'), name='softmax', input='forward') model.add_output(name='output', input='softmax') model.compile('adam', {'output': 'categorical_crossentropy'}) for epoch in xrange(EPOCH_SIZE): print "epoch: ", epoch for idx, (seq, label) in enumerate(zip(X_train, y_train)): loss, accuracy = model.train_on_batch({'input':np.array([seq]), 'output':np.array([label])}, accuracy=True) if idx % 20 == 0: print "\tidx={0}, loss={1}, accuracy={2}".format(idx, loss, accuracy) if validate: _Y, _P = [], [] for (seq, label) in zip(X_val, y_val): y = label.argmax(axis=-1) p = model.predict({'input':np.array([seq])})['output'][0].argmax(axis=-1) _Y.extend(list(y)) _P.extend(list(p)) _Y, _P = np.array(_Y), np.array(_P) print "should break right: ", ((_P == 1)*(_Y == 1)).sum() print "should break wrong: ", ((_P == 0)*(_Y == 1)).sum() print "should not break right: ", ((_P == 0)*(_Y == 0)).sum() print "should not break wrong: ", ((_P == 1)*(_Y == 0)).sum() with open('wordtable_json.txt','w') as wordtable_file: wordtable_file.write(wordtable.to_json()) with open('model_json.txt','w') as model_file: model_file.write(model.to_json()) model.save_weights('model_weights.h5', overwrite=True)
border_mode='same', W_regularizer=l2(l2value))(input) pool1 = AveragePooling1D(pool_length=5, stride=1, border_mode='same')(conv1) conv2 = Convolution1D(120, 5, activation='tanh', border_mode='same', W_regularizer=l2(l2value))(pool1) pool2 = AveragePooling1D(pool_length=5, stride=1, border_mode='same')(conv2) conv3 = Convolution1D(160, 4, activation='tanh', border_mode='same', W_regularizer=l2(l2value))(pool2) output = TimeDistributedDense(8, activation='softmax', name='output', W_regularizer=l2(l2value))(conv3) model = Model(input=input, output=output) adam = Adam(lr=lr) model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['weighted_accuracy']) model.summary() best_model_file = './onehotcnn' + str(lr) + str(nep) + str( l2value) + '.h5' #+str(hsize)+'tanh.h5' best_model = ModelCheckpoint(best_model_file, monitor='val_weighted_accuracy', verbose=2, save_best_only=True) # and trained it via: model.fit(traindata,
features_size = 100 output_size = nchar batch_len = 128 N_SAMPLE_PER_EPOCH = 10000000 NUMBER_OF_EPOCH = 10 # кол-во повторов тренировки по одному набору данных model = Sequential() rnn_layer = LSTM(input_dim=input_size, output_dim=features_size, activation='tanh', return_sequences=True) model.add(rnn_layer) model.add(TimeDistributedDense(output_dim=output_size)) model.add(Activation('softmax')) #sgd = SGD(lr=0.05, decay=1e-6, momentum=0.9, nesterov=True) print 'Compiling the model...' #model.compile(loss='categorical_crossentropy', optimizer=sgd) model.compile(loss='categorical_crossentropy', optimizer='rmsprop') print('Start training...') total_session_count = 0 total_sample_count = 0 # накапливаем последовательности разной длины в отдельных списках len2list_of_seq = {}
if not config.rnn_bidirectional: model.add_node(rnn_layer(config.rnn_size, config), input=prev, name='rnn') model.add_node(Dropout(config.after_rnn_dropout), input='rnn', name='dropout') else: # BRNN following https://github.com/fchollet/keras/blob/master/examples/imdb_bidirectional_lstm.py. Alternative approach: https://github.com/EderSantana/seya/blob/master/seya/layers/recurrent.py#L17-L48 model.add_node(rnn_layer(config.rnn_size/2, config), input=prev, name='forward') model.add_node(rnn_layer(config.rnn_size/2, config, go_backwards=True), input=prev, name='backward') model.add_node(Dropout(config.after_rnn_dropout), inputs=['forward', 'backward'], name='dropout') # Output model.add_node(TimeDistributedDense(data.output_size), input='dropout', name='tddense') model.add_node(Activation('softmax'), input='tddense', name='softmax') model.add_output(input='softmax', name='output') model.compile(optimizer=optimizer, loss={ 'output': config.loss }) def make_sequences(a, size=config.num_steps): """Reshape (d1, d2, ...) array into (d1/size, size, d2, ...). Pad with zeros if d1 % size is non-zero. """ if a.shape[0] % size != 0: pad_size = size - (a.shape[0] % size) warn('make_sequences: padding with %d zeros' % pad_size) pad = np.zeros((pad_size,) + a.shape[1:], dtype=a.dtype)
return model print "VGG loading" image_model = VGG_16('vgg16_weights.h5') image_model.trainable = False print "VGG loaded" # let's load the weights from a save file. # image_model.load_weights('weight_file.h5') # next, let's define a RNN model that encodes sequences of words # into sequences of 128-dimensional word vectors. print "Text model loading" language_model = Sequential() language_model.add(Embedding(vocab_size, 256, input_length=max_caption_len)) language_model.add(GRU(output_dim=128, return_sequences=True)) language_model.add(TimeDistributedDense(128)) print "Text model loaded" # let's repeat the image vector to turn it into a sequence. print "Repeat model loading" image_model.add(RepeatVector(max_caption_len)) print "Repeat model loaded" # the output of both models will be tensors of shape (samples, max_caption_len, 128). # let's concatenate these 2 vector sequences. print "Merging" model = Sequential() model.add(Merge([image_model, language_model], mode='concat', concat_axis=-1)) # let's encode this vector sequence into a single vector model.add(GRU(256, return_sequences=False)) # which will be used to compute a probability # distribution over what the next word in the caption should be! model.add(Dense(vocab_size))
def build_model(): # design the deepaclstm model main_input = Input(shape=(700, ), dtype='float32', name='main_input') #main_input = Masking(mask_value=23)(main_input) x = Embedding(output_dim=21, input_dim=21, input_length=700)(main_input) auxiliary_input = Input(shape=(700, 21), name='aux_input') #24 #auxiliary_input = Masking(mask_value=0)(auxiliary_input) print main_input.get_shape() print auxiliary_input.get_shape() concat = merge([x, auxiliary_input], mode='concat', concat_axis=-1) conv1_features = Convolution1D(42, 1, activation='relu', border_mode='same', W_regularizer=l2(0.001))(concat) # print 'conv1_features shape', conv1_features.get_shape() conv1_features = Reshape((700, 42, 1))(conv1_features) conv2_features = Convolution2D(42, 3, 1, activation='relu', border_mode='same', W_regularizer=l2(0.001))(conv1_features) # print 'conv2_features.get_shape()', conv2_features.get_shape() conv2_features = Reshape((700, 42 * 42))(conv2_features) conv2_features = Dropout(0.5)(conv2_features) conv2_features = Dense(400, activation='relu')(conv2_features) #, activation='tanh', inner_activation='sigmoid',dropout_W=0.5,dropout_U=0.5 lstm_f1 = LSTM(output_dim=300, return_sequences=True, inner_activation='sigmoid', dropout_W=0.5, dropout_U=0.5)(conv2_features) lstm_b1 = LSTM(output_dim=300, return_sequences=True, go_backwards=True, inner_activation='sigmoid', dropout_W=0.5, dropout_U=0.5)(conv2_features) lstm_f2 = LSTM(output_dim=300, return_sequences=True, inner_activation='sigmoid', dropout_W=0.5, dropout_U=0.5)(lstm_f1) lstm_b2 = LSTM(output_dim=300, return_sequences=True, go_backwards=True, inner_activation='sigmoid', dropout_W=0.5, dropout_U=0.5)(lstm_b1) concat_features = merge([lstm_f2, lstm_b2, conv2_features], mode='concat', concat_axis=-1) concat_features = Dropout(0.4)(concat_features) protein_features = Dense(600, activation='relu')(concat_features) # protein_features = TimeDistributedDense(600,activation='relu')(concat_features) # protein_features = TimeDistributedDense(100,activation='relu', W_regularizer=l2(0.001))(protein_features) main_output = TimeDistributedDense(8, activation='softmax', name='main_output')(protein_features) model = Model(input=[main_input, auxiliary_input], output=[main_output]) adam = Adam(lr=0.003) model.compile(optimizer=adam, loss={'main_output': 'categorical_crossentropy'}, metrics=['weighted_accuracy']) model.summary() return model
if s[12] > 0: ns[3] = s[12] elif s[14] > 0: ns[3] = s[14] else: ns[3] = 0 nt.append(ns) tdata.append(nt) tdata = np.array(tdata) x_train, x_test, _, _ = train_test_split(tdata, tdata, test_size=0.22, random_state=0) # model input_dim = 4 inputs = Input(shape=(x_train.shape[1], input_dim)) encoded = GRU(16, activation='relu', return_sequences=True)(inputs) encoded = TimeDistributedDense(16)(encoded) encoded = TimeDistributedDense(2)(encoded) encoded = Flatten()(encoded) decoded = Dense(128)(encoded) decoded = Reshape((32,4))(decoded) decoded = GRU(input_dim, return_sequences=True, activation='hard_sigmoid')(decoded) m = Model(inputs, decoded) e = Model(inputs, encoded) m.compile(optimizer='adadelta', loss='binary_crossentropy') m.summary() m.fit(x_train, x_train, nb_epoch=500, batch_size=256, shuffle=True, validation_data=(x_test, x_test)) tt = x_test[0] rr = np.around(m.predict(np.array([x_test[0]])))[0] print(x_test[0])
def main(argv): batch_size = 128 nb_epochs = 10 n_hidden = 128 patience = 100 train_data_size = 10000 test_data_size = 5000 T = 1000 #Delay length input_len = 10 #Input length category_size = 8 #Category size learning_rate = 0.001 learning_rate_natGrad = 0.00001 #None clipnorm = 1.0 savefile = "testing.txt" model = "uRNN_keras" out_every_t = True unitary_impl = "full" #ASB2016"#full"#ASB2016" #full, otherwise unitary_init = 'ASB2016' #ASB2016' #or it can be svd?, or just use ASB2016. histfile = 'exp/long_run' nb_classes = category_size + 2 # --- Create data -------------------- data_set, data_param = copyingProblemData(train_data_size, test_data_size, T, input_len, category_size) print("Done constructing data....") train_x = np.array(data_set['train']['Input']) train_y = np.array(data_set['train']['Output']) test_x = np.array(data_set['test']['Input']) test_y = np.array(data_set['test']['Output']) s_train_x = theano.shared(train_x) s_train_y = theano.shared(train_y) s_test_x = theano.shared(test_x) s_test_y = theano.shared(test_y) print(train_x.shape) print("Classes:", nb_classes) # --- Create theano graph and compute gradients ---------------------- if (model == 'uRNN_keras'): epsilon = 1e-5 model = Sequential() model.add( uRNN(output_dim=n_hidden, inner_init=unitary_init, unitary_impl=unitary_impl, input_shape=train_x.shape[1:], consume_less='gpu', epsilon=epsilon, return_sequences=True)) model.add(TimeDistributedDense(nb_classes)) model.add(Activation('softmax')) if (model == 'complex_RNN'): #This is currently broke still model = Sequential() model.add( complex_RNN_wrapper(output_dim=nb_classes, hidden_dim=n_hidden, unitary_impl=unitary_impl, input_shape=train_x.shape[1:])) model.add(Activation('softmax')) if (model == 'LSTM'): model = Sequential() model.add( LSTM(n_hiddne, return_sequences=True, input_shape=train_x.shape[1:])) model.add(TimeDistributedDense(nb_classes)) model.add(Activation('softmax')) #Setting up the model rmsprop = RMSprop_and_natGrad(lr=learning_rate, clipnorm=clipnorm, lr_natGrad=learning_rate_natGrad) model.compile(loss='categorical_crossentropy', optimizer=rmsprop, metrics=['accuracy']) history = LossHistory(histfile) checkpointer = keras.callbacks.ModelCheckpoint(filepath=savefile, verbose=1, save_best_only=True) earlystopping = keras.callbacks.EarlyStopping(monitor='val_loss', patience=patience, verbose=1, mode='auto') #make sure the experiment directory to hold results exists if not os.path.exists('exp'): os.makedirs('exp') print(model.summary()) #Now for the actual methods. print("X:", train_x.shape) print("Y:", train_y.shape) model.fit(train_x, train_y, nb_epoch=nb_epochs, verbose=1, batch_size=batch_size, validation_data=(test_x, test_y), callbacks=[history, checkpointer, earlystopping]) print("Done fitting!") scores = model.evaluate(train_x, train_y, verbose=0) print('Test loss:', scores[0]) print('Test accuracy:', scores[1]) # add test scores to history history_load = cPickle.load(open(histfile, 'rb')) history_load.update({'test_loss': scores[0], 'test_acc': scores[1]}) cPickle.dump(history_load, open(histfile, 'wb'))
x_train, x_test, y_train, y_test = train_test_split(set_x, set_y, test_size=0.25, random_state=0) # create sample weights from y_train nonbin_y_train = lb.inverse_transform(y_train) count = dict(zip(*np.unique(nonbin_y_train, return_counts=True))) tt = nonbin_y_train.shape[0] samples_weights = [] for lbl in nonbin_y_train: samples_weights.append(count[lbl] / float(tt)) samples_weights = 1 - np.array(samples_weights) # keras GRU m = Sequential() m.add(GRU(15, input_shape=(128, 15), activation='relu', return_sequences=True)) m.add(TimeDistributedDense(15, activation='relu')) m.add(Flatten()) m.add(Dense(8, activation='hard_sigmoid')) m.compile(optimizer='adadelta', loss='binary_crossentropy') m.summary() # # GRU Learning m.fit(x_train, y_train, sample_weight=samples_weights, nb_epoch=100, batch_size=128, shuffle=True, validation_data=(x_test, y_test)) # gru_predictions = m.predict(c0_themes)
def flow_prediction(method=""): record = [[[[0] * ranget for j in xrange(rangey)] for i in xrange(rangex)] for d in xrange(8)] for d, filename in enumerate(sorted( glob.glob(r"../data/pos_hour_user#/*"))): print filename for line in fileinput.input(filename): part = line.strip().split(" ") px, py, t, c = int(part[0].split(",")[0]), int( part[0].split(",")[1]), int(part[1]), int(part[2]) record[d][px][py][t] = c fileinput.close() record = filter(lambda x: sum(x) >= 10 * ranget, [ record[d][i][j] for d in xrange(8) for i in xrange(rangex) for j in xrange(rangey) ]) if method == "RNN": from keras.models import Graph from keras.layers import recurrent, Dropout, TimeDistributedDense normal = [[1. * record[r][t] / max(record[r]) for t in xrange(ranget)] for r in xrange(len(record))] X_train = np.array([[[p] for p in r][:-1] for r in normal]) y_train = np.array([[[p] for p in r][1:] for r in normal]) EPOCH_SIZE = 5 HIDDEN_SIZE = 256 RNN = recurrent.GRU # Replace with SimpleRNN, LSTM, GRU model = Graph() model.add_input(name='input', input_shape=(ranget - 1, 1)) model.add_node(RNN(HIDDEN_SIZE, return_sequences=True), name='forward_l1', input='input') model.add_node(TimeDistributedDense(1), name='dense', input='forward_l1') model.add_output(name='output', input='dense') model.compile('adam', {'output': 'mean_squared_error'}) model.fit({ 'input': X_train, 'output': y_train }, nb_epoch=EPOCH_SIZE, show_accuracy=True) y_pred = model.predict({'input': X_train})['output'] y_error_total, y_real_total = np.zeros((23, )), np.zeros((23, )) for r in xrange(len(record)): y_error_total += abs( np.reshape(max(record[r]) * y_pred[r], (23, )) - np.array(record[r][1:])) y_real_total += np.array(record[r][1:]) print 1. * y_error_total / y_real_total if method == "ARIMA": from statsmodels.tsa.arima_model import ARIMA normal = np.array(normal)[np.random.choice(len(record), 100)] y_error_total, y_real_total = np.zeros((23, )), np.zeros((23, )) for x in normal: try: model = ARIMA(np.array(x), order=(2, 0, 1)).fit(disp=False) y_error_total += abs(model.predict(1, 23) - np.array(x[1:])) y_real_total += np.array(x[1:]) except: continue print 1. * y_error_total / y_real_total
seq.add( Convolution1D(2, 5, activation='tanh', border_mode='same', W_regularizer=l2(0.001))) # 80 seq.add(AveragePooling1D(pool_length=5, stride=1, border_mode='same')) seq.add( Convolution1D(2, 4, activation='tanh', border_mode='same', W_regularizer=l2(0.001))) # 80 seq.add( TimeDistributedDense(8, activation='softmax', name='output', W_regularizer=l2(0.001))) adam = Adam(lr=lr) seq.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy']) seq.summary() #best_model_file = './onehotcnn'+str(lr)+str(nep)+'.h5' #best_model = ModelCheckpoint(best_model_file, monitor='val_output_acc', verbose=1, save_best_only=True) # and trained it via: seq.fit(traindata, trainlabel, nb_epoch=nep, batch_size=2, validation_data=(testdata, testlabel))
def build_model(): main_input = Input(shape=(700, 21), name='main_input') concat = main_input conv1_features = Convolution1D(42, 1, activation='relu', border_mode='same', W_regularizer=l2(0.001))(concat) # print 'conv1_features shape', conv1_features.get_shape() conv1_features = Reshape((700, 42, 1))(conv1_features) conv2_features = Convolution2D(42, 3, 1, activation='relu', border_mode='same', W_regularizer=l2(0.001))(conv1_features) # print 'conv2_features.get_shape()', conv2_features.get_shape() conv2_features = Reshape((700, 42 * 42))(conv2_features) conv2_features = Dropout(0.5)(conv2_features) conv2_features = Dense(400, activation='relu')(conv2_features) #, activation='tanh', inner_activation='sigmoid',dropout_W=0.5,dropout_U=0.5 lstm_f1 = LSTM(output_dim=300, return_sequences=True, inner_activation='sigmoid', dropout_W=0.5, dropout_U=0.5)(conv2_features) lstm_b1 = LSTM(output_dim=300, return_sequences=True, go_backwards=True, inner_activation='sigmoid', dropout_W=0.5, dropout_U=0.5)(conv2_features) lstm_f2 = LSTM(output_dim=300, return_sequences=True, inner_activation='sigmoid', dropout_W=0.5, dropout_U=0.5)(lstm_f1) lstm_b2 = LSTM(output_dim=300, return_sequences=True, go_backwards=True, inner_activation='sigmoid', dropout_W=0.5, dropout_U=0.5)(lstm_b1) concat_features = merge([lstm_f2, lstm_b2, conv2_features], mode='concat', concat_axis=-1) concat_features = Dropout(0.4)(concat_features) protein_features = Dense(600, activation='relu')(concat_features) # protein_features = TimeDistributedDense(600,activation='relu')(concat_features) # protein_features = TimeDistributedDense(100,activation='relu', W_regularizer=l2(0.001))(protein_features) main_output = TimeDistributedDense(8, activation='softmax', name='main_output')(protein_features) deepaclstm = Model(input=[main_input], output=[main_output]) adam = Adam(lr=0.003) deepaclstm.compile(optimizer=adam, loss={'main_output': 'categorical_crossentropy'}, metrics=['weighted_accuracy']) deepaclstm.summary() return deepaclstm
activation='tanh', inner_activation='sigmoid', dropout_W=0.5, dropout_U=0.5)(f1) f4 = GRU(output_dim=200, return_sequences=True, activation='tanh', inner_activation='sigmoid', go_backwards=True, dropout_W=0.5, dropout_U=0.5)(f2) cf_feature = merge([f3, f4, d_output], mode='concat', concat_axis=2) cf_feature = Dropout(0.4)(cf_feature) f_input = Dense(600, activation='relu')(cf_feature) main_output = TimeDistributedDense(8, activation='softmax', name='main_output')(f_input) model = Model(input=[main_input, auxiliary_input], output=[main_output]) adam = Adam(lr=0.003) model.compile( optimizer=adam, loss={'main_output': 'categorical_crossentropy'}, # loss_weights={'main_output': 1}, metrics=['weighted_accuracy']) model.summary() print "####### look at data's shape#########" print train_hot.shape, trainpssm.shape, trainlabel.shape, test_hot.shape, testpssm.shape, testlabel.shape, val_hot.shape, valpssm.shape, vallabel.shape earlyStopping = EarlyStopping(monitor='val_weighted_accuracy', patience=5,
i += 1 sample_weight.append(temp) sample_weight = np.asarray(sample_weight) X_sample_weight, X_sample_weight_rest = train_test_split(sample_weight, test_size=0.1, random_state=47) sample_weight_val, sample_weight_test = train_test_split(X_sample_weight_rest, test_size=0.5, random_state=47) #X_sample_weight=np.concatenate((X_sample_weight,sample_weight_val)) #model.add(Reshape((100,100),input_shape=(10000,))) model.add(LSTM(60, return_sequences=True, input_shape=(timesteps, data_dim))) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(Dropout(0.2)) model.add(TimeDistributedDense(28, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', sample_weight_mode="temporal") totaltrain = np.asarray(totaltrain) totallabel = np.asarray(totallabel) print len(x_val) x_train = np.asarray(x_train) x_test = np.asarray(x_test) x_val = np.asarray(x_val) y_train = np.asarray(y_train) y_val = np.asarray(y_val) y_test = np.asarray(y_test) print y_train.shape x_train = x_train[:, :100, :] y_train = y_train[:, :100, :]
print(y_train.shape) print('Build model...') model = Sequential() # "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE # note: in a situation where your input sequences have a variable length, # use input_shape=(None, nb_feature). model.add(RNN(HIDDEN_SIZE, input_shape=(MAXLEN, len(chars)))) # For the decoder's input, we repeat the encoded input for each time step model.add(RepeatVector(DIGITS + 1)) # The decoder RNN could be multiple layers stacked or a single layer for _ in range(LAYERS): model.add(RNN(HIDDEN_SIZE, return_sequences=True)) # For each of step of the output sequence, decide which character should be chosen model.add(TimeDistributedDense(len(chars))) model.add(Activation('softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) # Train the model each generation and show predictions against the validation dataset for iteration in range(1, 200): print() print('-' * 50) print('Iteration', iteration) model.fit(X_train, y_train, batch_size=BATCH_SIZE, nb_epoch=1, validation_data=(X_val, y_val)) ### # Select 10 samples from the validation set at random so we can visualize errors
print 'Loaded Inception model' # Turn off training on base model layers for layer in image_model.layers: layer.trainable = False # Add on a dense layer to non-linearize features; Feature size is 2048 x = Dense(2048, activation='relu')(image_model.get_layer('flatten').output) x = Dropout(0.5)(x) # Build the label LSTM model print "Loading label model" label_model = Sequential() label_model.add(Embedding(NUM_LABELS, 256, input_length=MAX_LABELS)) label_model.add(LSTM(output_dim=128, return_sequences=True)) label_model.add(TimeDistributedDense(128)) print "Label model loaded" # Repeat image feature vector to turn it into a sequence print "Repeat model loading" x = RepeatVector(MAX_LABELS)(x) # image_model.add(RepeatVector(MAX_LABELS)) print "Repeat model loaded" img_model = Model(input=image_model.input, output=x) # Merge image features and label features by concatenation print "Merging image and label features" model = Sequential() model.add(Merge([img_model, label_model], mode='concat', concat_axis=-1))
go_backwards=True, dropout_W=0.5)(f) f = merge([d, e], mode='concat') d = GRU(output_dim=300, return_sequences=True, activation='tanh', inner_activation='sigmoid', dropout_W=0.5)(f) e = GRU(output_dim=300, return_sequences=True, activation='tanh', inner_activation='sigmoid', go_backwards=True, dropout_W=0.5)(f) f = merge([d, e, x], mode='concat') f = TimeDistributedDense(200, activation='relu', W_regularizer=l2(0.001))(f) f = TimeDistributedDense(200, activation='relu', W_regularizer=l2(0.001))(f) main_output = TimeDistributedDense(8, activation='softmax', name='main_output')(f) auxiliary_output = TimeDistributedDense(4, activation='softmax', name='aux_output')(f) model = Model(input=[main_input, auxiliary_input], output=[main_output, auxiliary_output]) adam = Adam(lr=0.0003) model.compile(optimizer=adam, loss={ 'main_output': 'categorical_crossentropy', 'aux_output': 'categorical_crossentropy' }, loss_weights={