def _create_decoder(encoder_outputs, encoder_state_h, encoder_state_c, step): decoder_inputs = tf.repeat( input=tf.expand_dims(encoder_state_h, axis=1), repeats=step.hyperparams['output_length'], axis=1 ) decoder = LSTM( units=step.hyperparams['hidden_dim'], activation='relu', return_state=False, return_sequences=True ) decoder_outputs = decoder(decoder_inputs, initial_state=[encoder_state_h, encoder_state_c]) decoder_outputs = _create_attention_mechanism( decoder_outputs=decoder_outputs, encoder_outputs=encoder_outputs, step=step ) # (batch_size, output_length, output_dim) return decoder_outputs
def build_model(input_dim, max_length, vocab_size, embedding_dim, embedding_matrix): inputs1 = Input(shape=(input_dim, )) fe1 = Dropout(0.5)(inputs1) fe2 = Dense(256, activation='relu')(fe1) inputs2 = Input(shape=(max_length, )) se1 = Embedding(vocab_size, embedding_dim, mask_zero=False)(inputs2) se2 = Dropout(0.5)(se1) se3 = LSTM(256)(se2) # decoder1 = Concatenate()([fe2, se3]) decoder1 = add([fe2, se3]) decoder2 = Dense(256, activation='relu')(decoder1) # attention_probs = Dense(256, activation='softmax', name='attention_probs')(decoder2) # attention_mul = merge.Multiply()([decoder1, attention_probs]) outputs = Dense(vocab_size, activation='softmax')(decoder2) model = Model(inputs=[inputs1, inputs2], outputs=outputs) model.layers[2].set_weights([embedding_matrix]) model.layers[2].trainable = False model.compile(loss='categorical_crossentropy', optimizer='adam') return model
def create_model(): model = Sequential() model.add( LSTM(16, input_shape=(2, 2), return_sequences=True, activation='tanh')) model.add(LSTM(32, return_sequences=True, activation='tanh')) model.add(LSTM(64, return_sequences=True, activation='tanh')) model.add(LSTM(128, return_sequences=True, activation='tanh')) model.add(LSTM(64, return_sequences=True, activation='tanh')) model.add(Dropout(.15)) model.add(LSTM(32, return_sequences=True, activation='tanh')) model.add(Dense(16, activation='sigmoid')) model.add(LSTM(8, return_sequences=True, activation='relu')) model.add(Dense(2, activation='softmax')) model.summary() rms = RMSprop(lr=.01) model.compile(optimizer=rms, loss='categorical_crossentropy', metrics=['accuracy']) if "weights.h5" in os.listdir(".") and train_mode == False: model.load_weights(str("weights.h5")) print("loading saved network: weights.h5") return model
def buildModel(classes, features, cellNeurons, cellDropout, denseDropout, denseNeurons, sequenceLength, stacked=False, bidirectional=False, l2=0.0): model = Sequential() model.add(TimeDistributed(Dense(features, kernel_initializer='identity', bias_initializer='zeros', name='customNn', activation=None), input_shape=(sequenceLength, features), name='td', trainable=False)) if bidirectional: if stacked: model.add(Bidirectional(LSTM(cellNeurons, recurrent_dropout=cellDropout, name='rnn', trainable=True, return_sequences=True, kernel_regularizer=regularizers.l2(l2)), merge_mode='concat')) model.add(Bidirectional(LSTM(cellNeurons, recurrent_dropout=cellDropout, name='rnn_2nd_layer', trainable=True, kernel_regularizer=regularizers.l2(l2)), merge_mode='concat')) else: model.add(Bidirectional(LSTM(cellNeurons, recurrent_dropout=cellDropout, name='rnn', trainable=True, kernel_regularizer=regularizers.l2(l2)), merge_mode='concat')) else: if stacked: model.add(LSTM(cellNeurons, recurrent_dropout=cellDropout, name='rnn', trainable=True, return_sequences=True, kernel_regularizer=regularizers.l2(l2))) model.add(LSTM(cellNeurons, recurrent_dropout=cellDropout, name='rnn_2nd_layer', trainable=True, kernel_regularizer=regularizers.l2(l2))) #model.add(Attention(name='attention', trainable=True)) else: model.add(LSTM(cellNeurons, recurrent_dropout=cellDropout, name='rnn', trainable=True, kernel_regularizer=regularizers.l2(l2))) model.add(Dense(denseNeurons, name='nn', trainable=True, kernel_regularizer=regularizers.l2(l2))) model.add(Dropout(denseDropout, name='nn_dropout', trainable=True)) model.add(Dense(classes, activation="softmax", name='output_softmax', trainable=True, kernel_regularizer=regularizers.l2(l2))) #model.summary() if onTpu: model.compile(loss="categorical_crossentropy", optimizer=tf.train.AdamOptimizer(learning_rate=0.001), metrics=["accuracy"]) multi_model = toTpuModel(model) else: multi_model = toMultiGpuModel(model) multi_model.compile(loss="categorical_crossentropy", optimizer=optimizers.Adam(lr=0.001), metrics=["accuracy"]) return model, multi_model
#每组采样300条数据用于训练 index0 = np.random.permutation(np.where(y == 0)[0]) index1 = np.random.permutation(np.where(y == 1)[0]) #准备训练数据 x_train = np.concatenate((x[index0[0:300]], x[index1[0:300]]), axis=0) y_train = np.concatenate((y[index0[0:300]], y[index1[0:300]]), axis=0) #准备测试数据 x_test = np.concatenate((x[index0[300:400]], x[index1[300:400]]), axis=0) y_test = np.concatenate((y[index0[300:400]], y[index1[300:400]]), axis=0) #定义神经网络 model = tf.keras.Sequential() model.add(LSTM(50, input_shape=(x_train.shape[1], x_train.shape[2]))) model.add(Dense(1024, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(128, activation='relu')) model.add(Dropout(0.25)) model.add(Dense(1, activation='sigmoid')) # 输出层 model.summary() # 定义优化器 sgd = tf.keras.optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) # 整合模型 model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])
# reshape into X=t and Y=t+1 lags = 3 X_train, y_train = prepare_data(train, lags) X_test, y_test = prepare_data(test, lags) # reshape input to be [samples, time steps, features] X_train = np.reshape(X_train, (X_train.shape[0], 1, X_train.shape[1])) X_test = np.reshape(X_test, (X_test.shape[0], 1, X_test.shape[1])) tbCallBack = tf.keras.callbacks.TensorBoard(log_dir='./tmp/model_graph', write_graph=True) # create and fit the LSTM network mdl = Sequential() mdl.add(Dense(4, input_shape=(1, lags), activation='relu')) mdl.add(LSTM(6, activation='relu')) mdl.add(Dense(1, activation='relu')) mdl.compile(loss='mean_squared_error', optimizer='adam') mdl.fit(X_train, y_train, epochs=10000, batch_size=10, verbose=2, callbacks=[tbCallBack]) # make predictions train_predict = mdl.predict(X_train) test_predict = mdl.predict(X_test) # invert transformation train_predict = scaler.inverse_transform(train_predict)
def keras_BiDirect_estimator(model_dir, config, learning_rate, dropout_rate, embedding_dim, embedding_path=None, word_index=None): model = models.Sequential() num_features = min(len(word_index) + 1, TOP_K) # If pre-trained embedding is used add weights to the embeddings layer # and set trainable to input is_embedding_trainable flag. if embedding_path != None: embedding_matrix = get_embedding_matrix(word_index, embedding_path, embedding_dim) is_embedding_trainable = True # set to False to freeze embedding weights model.add( Embedding(input_dim=num_features, output_dim=embedding_dim, input_length=MAX_SEQUENCE_LENGTH, weights=[embedding_matrix], trainable=is_embedding_trainable)) else: model.add( Embedding(input_dim=num_features, output_dim=embedding_dim, input_length=MAX_SEQUENCE_LENGTH)) model.add(SpatialDropout1D(dropout_rate)) model.add( Bidirectional( LSTM(embedding_dim * 2, dropout=dropout_rate, recurrent_dropout=0.2, return_sequences=True))) model.add( Bidirectional( LSTM(int(embedding_dim / 2), dropout=dropout_rate, recurrent_dropout=0.2))) model.add(Dropout(rate=dropout_rate)) model.add( Dense(len(CLASSES), kernel_regularizer=tf.keras.regularizers.l1(0.01), activation='softmax')) # Compile model with learning parameters. optimizer = tf.keras.optimizers.Adam(lr=learning_rate) model.compile( optimizer=optimizer, loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]) estimator = tf.keras.estimator.model_to_estimator(keras_model=model, model_dir=model_dir, config=config) return estimator
def __init__(self, fl, mode, hparams): """ Initialises new DNN model based on input features_dim, labels_dim, hparams :param features_dim: Number of input feature nodes. Integer :param labels_dim: Number of output label nodes. Integer :param hparams: Dict containing hyperparameter information. Dict can be created using create_hparams() function. hparams includes: hidden_layers: List containing number of nodes in each hidden layer. [10, 20] means 10 then 20 nodes. """ # self.features_dim = fl.features_c_dim # self.labels_dim = fl.labels_dim # Assuming that each task has only 1 dimensional output self.features_dim = fl.features_c_dim + 1 # 1 for the positional argument self.labels_dim = 1 self.numel = fl.labels.shape[1] + 1 self.hparams = hparams self.mode = mode self.normalise_labels = fl.normalise_labels self.labels_scaler = fl.labels_scaler features_in = Input(shape=(self.features_dim, ), name='main_features_c_input') # Selection of model if mode == 'ann': model = ann(self.features_dim, self.labels_dim, self.hparams) x = model(features_in) self.model = Model(inputs=features_in, outputs=x) elif mode == 'ann2': model_1 = ann(self.features_dim, 50, self.hparams) x = model_1(features_in) model_end = ann(50, 50, self.hparams) end = model_end(x) end_node = Dense(units=1, activation='linear', kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='output_layer')(end) model_2 = ann(50, self.labels_dim - 1, self.hparams) x = model_2(x) self.model = Model(inputs=features_in, outputs=[end_node, x]) elif mode == 'ann3': x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(0))(features_in) x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(1))(x) x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(2))(x) # x = BatchNormalization()(x) x = Dense(units=1, activation='linear', kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_set_19')(x) self.model = Model(inputs=features_in, outputs=x) elif mode == 'conv1': x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='shared' + str(1))(features_in) x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(1))(x) #x = BatchNormalization()(x) x = Dense(units=19, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_set_19')(x) #x = BatchNormalization()(x) x = Reshape(target_shape=(19, 1))(x) x = Conv1D(filters=hparams['filters'], kernel_size=3, strides=1, padding='same', activation='relu')(x) #x = BatchNormalization()(x) x = Conv1D(filters=hparams['filters'] * 2, kernel_size=3, strides=1, padding='same', activation='relu')(x) x = Conv1D(filters=hparams['filters'] * 4, kernel_size=3, strides=1, padding='same', activation='relu')(x) #x = Permute((2,1))(x) #x = GlobalAveragePooling1D()(x) x = TimeDistributed(Dense(1, activation='linear'))(x) x = Reshape(target_shape=(19, ))(x) self.model = Model(inputs=features_in, outputs=x) elif mode == 'conv2': x = Dense(units=10, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Shared_e_' + str(1))(features_in) x = Dense(units=10, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Shared_e_' + str(2))(x) end = Dense(units=10, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Dense_e_' + str(1))(x) end = Dense(units=10, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Dense_e_' + str(2))(end) end_node = Dense(units=1, activation='linear', kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='output_layer')(end) x = Dense(units=80, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(1))(x) x = Reshape(target_shape=(80, 1))(x) x = Conv1D(filters=8, kernel_size=3, strides=1, padding='same', activation='relu')(x) x = MaxPooling1D(pool_size=2)(x) x = Conv1D(filters=16, kernel_size=3, strides=1, padding='same', activation='relu')(x) x = MaxPooling1D(pool_size=2)(x) #x = Permute((2,1))(x) #x = GlobalAveragePooling1D()(x) x = TimeDistributed(Dense(1, activation='linear'))(x) x = Reshape(target_shape=(20, ))(x) self.model = Model(inputs=features_in, outputs=[end_node, x]) elif mode == 'lstm': x = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Shared_e_' + str(1))(features_in) x = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Shared_e_' + str(2))(x) end = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Dense_e_' + str(1))(x) end = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Dense_e_' + str(2))(end) end_node = Dense(units=1, activation='linear', kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='output_layer')(end) x = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(1))(x) x = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(2))(x) x = RepeatVector(n=20)(x) x = LSTM(units=30, activation='relu', return_sequences=True)(x) x = LSTM(units=30, activation='relu', return_sequences=True)(x) x = TimeDistributed(Dense(1))(x) x = Reshape(target_shape=(20, ))(x) ''' x = Permute((2,1))(x) x = GlobalAveragePooling1D()(x) ''' self.model = Model(inputs=features_in, outputs=[end_node, x]) optimizer = Adam(clipnorm=1) self.model.compile(optimizer=optimizer, loss='mean_squared_error')
train_ratio = 0.7 train_size = 7000 test_ratio = 0.3 test_size = 3000 x_train, x_test = x_in_rs[:train_size], x_in_rs[(train_size):(sample_size)] y_train, y_test = y_true[:train_size], y_true[(train_size):(sample_size)] data_dim = 4 timesteps = 1 data_out = 2 # Tensorflow model model = Sequential() # Input Layer model.add(LSTM(32, activation = 'tanh', return_sequences=True, input_shape=(timesteps, data_dim))) # model.add(Dense(32, kernel_initializer='normal', activation='relu', input_dim=data_dim)) # Hidden Layer model.add(Dense(64, kernel_initializer='normal', activation='relu')) # Hidden Layer model.add(Dense(128, kernel_initializer='normal', activation='relu')) # Hidden Layer model.add(Dense(256, kernel_initializer='normal', activation='relu')) # The Output Layer : model.add(Dense(data_out, kernel_initializer='normal',activation='linear')) # Compile the network : model.compile(loss='mae', optimizer='adam', metrics=[mae]) model.summary() # reshape x_train, x_test to fit input in LSTM layers
from sklearn.model_selection import train_test_split # 90%的样本用来训练,剩余10%用来测试 X_train, X_test, y_train, y_test = train_test_split(train_pad, train_target, test_size=0.1, random_state=12) # 查看训练样本,确认无误 print(reverse_tokens(X_train[35])) print('class: ', y_train[35]) # 用LSTM对样本进行分类 model = Sequential() # 模型第一层为embedding model.add(Embedding(num_words, embedding_dim, weights=[embedding_matrix], input_length=max_tokens, trainable=False)) model.add(Bidirectional(LSTM(units=64, return_sequences=True))) model.add(LSTM(units=16, return_sequences=False)) model.add(Dense(1, activation='sigmoid')) # 我们使用adam以0.001的learning rate进行优化 optimizer = Adam(lr=1e-3) model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) # 我们来看一下模型的结构,一共90k左右可训练的变量 model.summary() # 建立一个权重的存储点 path_checkpoint = 'sentiment_checkpoint.keras' checkpoint = ModelCheckpoint(filepath=path_checkpoint, monitor='val_loss', verbose=1, save_weights_only=True, save_best_only=True)
def __init__(self, fields=None, variables=None, hidden_layers=None, activation="tanh", output_activation="linear", rnn_type="SimpleRNN", recurrent_activation="tanh", kernel_initializer=None, bias_initializer=None, kernel_regularizer=None, bias_regularizer=None, dtype=None, trainable=True, **kwargs): # check data-type. if dtype is None: dtype = K.floatx() elif not K.floatx() == dtype: K.set_floatx(dtype) # prepare hidden layers. if hidden_layers is None: hidden_layers = [] else: hidden_layers = to_list(hidden_layers) # check for copy constructor. if all([x in kwargs for x in ('inputs', 'outputs', 'layers')]): self._inputs = kwargs['inputs'].copy() self._outputs = kwargs['outputs'].copy() self._layers = kwargs['layers'].copy() self._set_model() return # prepare kernel initializers. activations, def_biasinit, def_kerinit = \ prepare_default_activations_and_initializers( len(hidden_layers) * [activation] + [output_activation] ) if kernel_initializer is None: kernel_initializer = def_kerinit elif isinstance(kernel_initializer, (float, int)): kernel_initializer = default_weight_initializer( len(hidden_layers) * [activation] + [output_activation], 'constant', scale=kernel_initializer) else: kernel_initializer = [ kernel_initializer for l in len(hidden_layers) * [activation] + [output_activation] ] # prepare bias initializers. if bias_initializer is None: bias_initializer = def_biasinit elif isinstance(bias_initializer, (float, int)): bias_initializer = default_weight_initializer( len(hidden_layers) * [activation] + [output_activation], 'constant', scale=bias_initializer) else: bias_initializer = [ bias_initializer for l in len(hidden_layers) * [activation] + [output_activation] ] # prepare regularizers. kernel_regularizer = default_regularizer(kernel_regularizer) bias_regularizer = default_regularizer(bias_regularizer) # prepares fields. fields = to_list(fields) if all([isinstance(fld, str) for fld in fields]): output_fields = [ RNNField( name=fld, dtype=dtype, kernel_initializer=kernel_initializer[-1], bias_initializer=bias_initializer[-1], kernel_regularizer=kernel_regularizer, bias_regularizer=bias_regularizer, trainable=trainable, ) for fld in fields ] elif all([validations.is_field(fld) for fld in fields]): output_fields = fields else: raise TypeError('Please provide a "list" of field names of' + ' type "String" or "Field" objects.') # prepare inputs/outputs/layers. inputs = [] layers = [] variables = to_list(variables) if all([isinstance(var, RNNFunctional) for var in variables]): for var in variables: inputs += var.outputs # for var in variables: # for lay in var.layers: # layers.append(lay) else: raise TypeError( "Input error: Please provide a `list` of `Functional`s. \n" "Provided - {}".format(variables)) # prepare hidden layers. if hidden_layers is None: hidden_layers = [] else: hidden_layers = to_list(hidden_layers) # Check and convert activation functions to proper format. assert not isinstance(activation, list), \ 'Expected an activation function name not a "list". ' afunc = get_activation(activation) # Input layers. if len(inputs) == 1: net_input = inputs[0] else: layer = Concatenate(name=graph_unique_name('conct')) net_input = layer(inputs) # Define the networks. net = [net_input] assert len( hidden_layers) > 0, 'Minimum of 1 RNN hidden layer is needed.' # Adding hidden layers for nLay, nNeuron in enumerate(hidden_layers): if nLay < 1000: # First layer starts with RNN. if rnn_type == 'LSTM': layer = LSTM(nNeuron, return_sequences=True, recurrent_activation=recurrent_activation, kernel_initializer=kernel_initializer[nLay], bias_initializer=bias_initializer[nLay], kernel_regularizer=kernel_regularizer, bias_regularizer=bias_regularizer, trainable=trainable, dtype=dtype, unroll=True, name=graph_unique_name( "LSTM{:d}b_".format(nNeuron))) elif rnn_type == 'SimpleRNN': layer = SimpleRNN( nNeuron, return_sequences=True, kernel_initializer=kernel_initializer[nLay], bias_initializer=bias_initializer[nLay], kernel_regularizer=kernel_regularizer, bias_regularizer=bias_regularizer, trainable=trainable, dtype=dtype, unroll=True, name=graph_unique_name("SRNN{:d}b_".format(nNeuron))) else: raise ValueError('Invalid entry for `rnn_type` -- ' 'accepts from (`SimpleRNN`, `LSTM`).') else: # Add the dense layer. layer = Dense(nNeuron, kernel_initializer=kernel_initializer[nLay], bias_initializer=bias_initializer[nLay], kernel_regularizer=kernel_regularizer, bias_regularizer=bias_regularizer, trainable=trainable, dtype=dtype, name=graph_unique_name("D{:d}b".format(nNeuron))) layers.append(layer) net[-1] = layer(net[-1]) # Apply the activation. if afunc.__name__ != 'linear': layer = activations[nLay] layers.append(layer) net[-1] = layer(net[-1]) # store output layers. for out in output_fields: layers.append(out) # Assign to the output variable if len(net) == 1: net_output = net[0] else: raise ValueError("Legacy for Enrichment: Must be updated. ") layer = Concatenate(name=graph_unique_name("{}_".format("conct"))) net_output = layer(net) # check output activation functions. output_func = get_activation(output_activation) # Define the final outputs of each network outputs = [] for out in output_fields: # add the activation on the output. if output_func.__name__ != 'linear': layer = activations[-1] layers.append(layer) outputs.append(layer(out(net_output))) else: outputs.append(out(net_output)) self._inputs = inputs self._outputs = outputs self._layers = layers self._set_model()
scaler = MinMaxScaler(feature_range=(0, 1)) dataset = scaler.fit_transform(dataset) # split into train and test sets train_size = int(len(dataset) * 0.67) test_size = len(dataset) - train_size train, test = dataset[0:train_size, :], dataset[train_size:len(dataset), :] # reshape into X=t and Y=t+1 look_back = 1 trainX, trainY = create_dataset(train, look_back) testX, testY = create_dataset(test, look_back) # reshape input to be [samples, time steps, features] trainX = numpy.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1])) testX = numpy.reshape(testX, (testX.shape[0], 1, testX.shape[1])) # create and fit the LSTM network model = Sequential() model.add(LSTM(4, input_shape=(1, look_back))) model.add(Dense(1)) model.compile(loss='mean_squared_error', optimizer='adam') model.fit(trainX, trainY, epochs=100, batch_size=1, verbose=2) # make predictions trainPredict = model.predict(trainX) testPredict = model.predict(testX) # invert predictions trainPredict = scaler.inverse_transform(trainPredict) trainY = scaler.inverse_transform([trainY]) testPredict = scaler.inverse_transform(testPredict) testY = scaler.inverse_transform([testY]) # calculate root mean squared error trainScore = math.sqrt(mean_squared_error(trainY[0], trainPredict[:, 0])) print('Train Score: %.2f RMSE' % (trainScore)) testScore = math.sqrt(mean_squared_error(testY[0], testPredict[:, 0]))
#two categorical arrays, one for each side of the functional network #testY1, testY2 = np.hsplit(testY,2) #testY1 = to_categorical(testY1) #testY2 = to_categorical(testY2) valY1, valY2 = np.hsplit(valY,2) valY1 = to_categorical(valY1) valY2 = to_categorical(valY2) trainY1, trainY2 = np.hsplit(trainY,2) trainY1 = to_categorical(trainY1) trainY2 = to_categorical(trainY2) #build functional model visible =Input(shape=(trainX.shape[1],trainX.shape[2])) hidden1 = LSTM(32, return_sequences=True, name='firstLSTMLayer')(visible) hidden2 = LSTM(16, name='secondLSTMLayer',return_sequences=True)(hidden1) #left branch decides second agent action hiddenLeft = LSTM(10, name='leftBranch')(hidden2) agent2 = Dense(5,activation='softmax',name='agent2classifier')(hiddenLeft) #right branch decides third agent action hiddenRight = LSTM(10, name='rightBranch')(hidden2) agent3 = Dense(5,activation='softmax',name='agent3classifier')(hiddenRight) model = Model(inputs=visible,outputs=[agent2,agent3]) model.compile(optimizer='adam', loss={'agent2classifier': 'categorical_crossentropy', 'agent3classifier': 'categorical_crossentropy'}, metrics={'agent2classifier': ['acc'], 'agent3classifier': ['acc']})
vocab = pd.read_csv(os.path.join(training_dir, "vocab.csv"), header=None, names=None) # Labels are in the first column train_y = train_data.iloc[:, 0] train_x = train_data.iloc[:, 1:] max_answer_len = train_x.values.shape[1] # Build Model model = Sequential() model.add(Embedding(len(vocab), args.embedding_size, input_length=max_answer_len)) model.add(Dropout(args.dropout)) if args.flatten: model.add(Flatten()) model.add(Reshape((1, args.embedding_size * max_answer_len))) if args.lstm_dim_2: model.add(LSTM(args.lstm_dim_1, return_sequences=True)) model.add(LSTM(args.lstm_dim_2, return_sequences=False)) else: model.add(LSTM(args.lstm_dim_1, return_sequences=False)) model.add(Dropout(args.dropout)) model.add(Dense(1, activation="linear")) optimizer = AdamOptimizer() model.compile(optimizer=optimizer, loss='mean_squared_error', metrics=['acc']) # Train the model model.fit(train_x, train_y, epochs=args.epochs, verbose=0) # Validate test_y = test_data.iloc[:, 0] test_x = test_data.iloc[:, 1:]
# # Keras를 활용한 CNN-LSTM 모델 생성 및 학습 # In[61]: model1=Sequential() #input_shape=(36=오전 9시부터 5시까지의 15분 단위 데이터 갯수 ,7 = 일사량 예측을 위한 변수 갯수) model1.add(TimeDistributed(Conv1D(32, kernel_size=2, activation='relu'), input_shape=(xB.shape[1],xB.shape[2],1))) model1.add(BatchNormalization()) #배치 정규화를 통한 과적합 방지 model1.add(Dropout(0.3)) #과적합을 막기위한 dropout model1.add(TimeDistributed(MaxPooling1D(pool_size=2, strides=2))) #컨볼루션 연산을 통해 생성된 데이터의 지역적 특징 중 가장 큰 값을 반영하여 특징을 함축하는 MaxPooling층을 사용하여 지역적 특징을 연산합니다. #strides가 2이므로 입력 변수 갯수가 반으로 줄어듭니다. model1.add(Reshape((36,96), input_shape=(36,3,32))) #LSTM에 넣기 위해 3차원 데이터를 2차원으로 변환. model1.add(LSTM(32, input_shape=(36,96),return_sequences=True)) #stacked LSTM(2층 짜리) 구현 model1.add(Activation('relu')) #활성화 함수로 ReLu적용 model1.add(Dropout(0.3))#과적합을 막기위한 dropout model1.add(LSTM(32)) #두 번째 LSTM model1.add(Activation('relu'))#활성화 함수로 ReLu적용 model1.add(Dropout(0.3))#과적합을 막기위한 dropout model1.add(Dense(36))# 하루 단위 결과 출력 model1.compile(loss='mse', optimizer='adam') #손실함수 MSE, 최적화 함수 ADAM 적용 model1.summary() model1.fit(xB, yB, epochs=370, verbose=1) #기존의 기상청 데이터와 일사량데이터를 훈련데이터로 하여 학습 반복 횟수 370으로 설정하여 모델 학습 진행. # In[62]:
def model_build(FEATURE_NUM): # 模型构建层 ''' model = Sequential() # 序贯模型(Sequential)单输入单输出,一条路通到底,层与层之间只有相邻关系,没有跨层连接 # model.add(LSTM(30, input_shape=(TIMESERIES_LENGTH, 3))) # 单向,在此改双向 model.add(Bidirectional(GRU(30, return_sequences=True, input_shape=(TIMESERIES_LENGTH, 3)), merge_mode='concat')) # 双向GRU 第一层要指定数据输入的形状 # model.add(Bidirectional(LSTM(30, return_sequences=True, input_shape=(TIMESERIES_LENGTH, 3)), # merge_mode='concat')) # 双向LSTM 第一层要指定数据输入的形状,改双层此处多加一句return返回ndim=3的序列 model.add(BatchNormalization()) # 加入正则化 model.add(Dropout(0.2)) # 随机选取,防止过拟合 # model.add(Bidirectional(LSTM(30))) # 第二层双向LSTM model.add(Bidirectional(GRU(30))) # 第二层双向GRU model.add(BatchNormalization()) # 加入正则化 model.add(Dropout(0.2)) # 随机选取,防止过拟合 model.add(Dense(FEATURE_NUM, activation='softmax')) # 是Keras定义网络层的基本方法,有FEATURE_NUM个节点,激活函数是softmax model.add(Flatten()) # 比单向多出来一句 # Flatten层用来将输入“压平”,即把多维的输入一维化,常用在从卷积层到全连接层的过渡。Flatten不影响batch的大小 model.compile(optimizer=Adam(lr=0.001), loss='categorical_crossentropy', metrics=['accuracy']) model.build((None, 90, 3)) # 告诉模型输入的格式 其中90是lstm的time_step ,3是input_dim,none这里个人认为代表样本数 model.summary() ''' # 完整版 CNN-Att-BiLSTM ''' input = Input(shape=(TIMESERIES_LENGTH, 3)) # 1. 输入shape conv1 = Conv1D(filters=16, kernel_size=10, activation='relu')(input) # 2. for input1 主要用的卷积层 LSTM1 = Bidirectional(LSTM(30, return_sequences=True, input_shape=(TIMESERIES_LENGTH, 3)), merge_mode='concat')(conv1) # 3. 第一层双向GRU NormalOut = BatchNormalization()(LSTM1) # 4. 正则化,防止过拟合 DropOut = Dropout(0.2)(NormalOut) # 5. dropout,防止过拟合 attention_mul = attention_3d_block(DropOut) # 6. 注意力机制 BiLSTM2 = Bidirectional(LSTM(30))(attention_mul) # 7. 第二层双向GRU NormalOut = BatchNormalization()(BiLSTM2) # 8. 正则化,防止过拟合 DropOut = Dropout(0.2)(NormalOut) # 9. dropout,防止过拟合 ''' input = Input(shape=(TIMESERIES_LENGTH, 3)) # 1. 输入shape # conv1 = Conv1D(filters=48, kernel_size=6, strides=1, activation='relu')(input) # for input1 # conv1 = Conv1D(filters=16, kernel_size=10, activation='relu')(input) # 2. for input1 主要用的卷积层 # lstm1 = LSTM(30, return_sequences=True, input_shape=(TIMESERIES_LENGTH, 3))(conv1) # 单层LSTM # attention_mul = attention_3d_block(conv1) BiLSTM1 = Bidirectional(LSTM(30, return_sequences=True, input_shape=(TIMESERIES_LENGTH, 3)), merge_mode='concat')(input) # 3. 第一层双向LSTM # BiLSTM1 = Bidirectional(LSTM(30, return_sequences=True, input_shape=(TIMESERIES_LENGTH, 3)), merge_mode='concat')( # conv1) # 第一层双向LSTM # attention_mul = attention_3d_block(BiLSTM1) # 6. 注意力机制 # LSTM1 = LSTM(30, input_shape=(TIMESERIES_LENGTH, 3))(attention_mul) # 单层LSTM # BiLSTM2 = Bidirectional(LSTM(30, input_shape=(TIMESERIES_LENGTH, 3)), merge_mode='concat')(attention_mul) # 单层双向GRU # BiGRU1 = LSTM(30, input_shape=(TIMESERIES_LENGTH, 3))(input) # 单层LSTM # conv2 = Conv1D(filters=16, kernel_size=10, activation='relu')(attention_mul) NormalOut = BatchNormalization()(BiLSTM1) # 4. 正则化,防止过拟合 DropOut = Dropout(0.2)(NormalOut) # 5. dropout,防止过拟合 attention_mul = attention_3d_block(DropOut) # 6. 注意力机制 # # BiGRU2 = Bidirectional(GRU(30))(attention_mul) # 7. 第二层双向GRU BiLSTM12 = Bidirectional(LSTM(30))(attention_mul) # 第二层双向LSTM # # LSTMOut = LSTM(30, input_shape=(TIMESERIES_LENGTH, 3))(conv1) # NormalOut = BatchNormalization()(BiLSTM12) # 8. 正则化,防止过拟合 NormalOut = BatchNormalization()(BiLSTM12) # 正则化,防止过拟合 DropOut = Dropout(0.2)(NormalOut) # 9. dropout,防止过拟合 # 分界线 DenseOut = Dense(FEATURE_NUM, activation='softmax')( DropOut) # 10. 是Keras定义网络层的基本方法,有FEATURE_NUM个节点,激活函数是softmax model = Model(inputs=input, outputs=DenseOut) model.compile(optimizer=Adam(lr=0.001), loss='categorical_crossentropy', metrics=['accuracy']) # 交叉熵损失函数 model.build((None, 90, 3)) model.summary() return model
print(X_train.shape) print(X_test.shape) """(15, 40, 20) 15个样本 40个特征 每个特征用20词向量表示""" #拉直形状 (15, 40, 20)=>(15, 40*20) (6, 40, 20)=>(6, 40*20) X_train = X_train.reshape(len(y_train), maxlen * max_features) X_test = X_test.reshape(len(y_test), maxlen * max_features) #--------------------------------建模与训练------------------------------- model = Sequential() #构建Embedding层 128代表Embedding层的向量维度 model.add(Embedding(max_features, 128)) #构建LSTM层 model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2)) #构建全连接层 #注意上面构建LSTM层时只会得到最后一个节点的输出,如果需要输出每个时间点的结果需将return_sequences=True model.add(Dense(units=1, activation='sigmoid')) #模型可视化 model.summary() #激活神经网络 model.compile( optimizer='rmsprop', #RMSprop优化器 loss='binary_crossentropy', #二元交叉熵损失 metrics=['acc'] #计算误差或准确率 )
sys.exit(0) else: grid_df.loc[idx, 'timestamp'] = curr_time train_generator = crop_generator(input_path=tr_path, batch_size=batch_size, mode="train", do_shuffle=True, epsilon=0) val_generator = crop_generator(input_path=val_path, batch_size=batch_size, mode="val", do_shuffle=True, epsilon=0) # temporal part starts here temporal_input_layer = Input(batch_shape = (None, 23, 1), name='time_input_layer') # add an LSTM layer lstm_1 = LSTM(100, input_shape=(23,1), dropout=dropout_rate)(temporal_input_layer) # add a dense layer ts_output = Dense(32, activation="relu")(lstm_1) # temporal part ends here # and a softmax layer -- num_classes predictions = Dense(num_classes, activation='softmax')(ts_output) optimizer = Adam(lr=learning_rate) # this is the model we will train model = Model(inputs= temporal_input_layer, outputs=predictions)
def test_delete_channels_lstm(channel_index): layer = LSTM(9, return_sequences=True) recursive_test_helper(layer, channel_index)
EPOCHS = 10 EMBED_DIM = 64 BiRNN_UNITS = 200 vacab_size = vocabulary_lookuper.size() tag_size = tag_lookuper.size() # model = Sequential() # model.add(Embedding(vacab_size, EMBED_DIM, mask_zero=True)) # model.add(Bidirectional(LSTM(BiRNN_UNITS // 2, return_sequences=True))) # model.add(CRF(tag_size)) raw_input = layers.Input(shape=(MAX_LEN,)) embedding_layer = Embedding(vacab_size, EMBED_DIM, mask_zero=True)(raw_input) bilstm_layer = Bidirectional(LSTM(BiRNN_UNITS // 2, return_sequences=True))(embedding_layer) crf_layer = CRF( units=tag_size, transition_constraint_matrix=constraint_table ) dynamic_constraint_input = layers.Input(shape=(intent_number,)) output_layer = crf_layer([bilstm_layer, dynamic_constraint_input]) model = models.Model([raw_input, dynamic_constraint_input], output_layer) # print model summary model.summary()
def __init__(self, fl, mode, hparams): """ Initialises new DNN model based on input features_dim, labels_dim, hparams :param features_dim: Number of input feature nodes. Integer :param labels_dim: Number of output label nodes. Integer :param hparams: Dict containing hyperparameter information. Dict can be created using create_hparams() function. hparams includes: hidden_layers: List containing number of nodes in each hidden layer. [10, 20] means 10 then 20 nodes. """ self.features_dim = fl.features_c_dim self.labels_dim = fl.labels_dim # Assuming that each task has only 1 dimensional output self.hparams = hparams self.mode = mode self.normalise_labels = fl.normalise_labels self.labels_scaler = fl.labels_scaler features_in = Input(shape=(self.features_dim, ), name='main_features_c_input') # Selection of model if mode == 'ann': model = ann(self.features_dim, self.labels_dim, self.hparams) x = model(features_in) self.model = Model(inputs=features_in, outputs=x) elif mode == 'ann2': model_1 = ann(self.features_dim, 50, self.hparams) x = model_1(features_in) model_end = ann(50, 50, self.hparams) end = model_end(x) end_node = Dense(units=1, activation='linear', kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='output_layer')(end) model_2 = ann(50, self.labels_dim - 1, self.hparams) x = model_2(x) self.model = Model(inputs=features_in, outputs=[end_node, x]) elif mode == 'ann3': x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(0))(features_in) x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(1))(x) x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(2))(x) # x = BatchNormalization()(x) x = Dense(units=self.labels_dim, activation='linear', kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Final')(x) self.model = Model(inputs=features_in, outputs=x) elif mode == 'conv1': if fl.label_type == 'gf20': final_dim = 20 else: final_dim = 19 x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='shared' + str(1))(features_in) x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(1))(x) #x = BatchNormalization()(x) x = Dense(units=final_dim, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_set_19')(x) #x = BatchNormalization()(x) x = Reshape(target_shape=(final_dim, 1))(x) x = Conv1D(filters=hparams['filters'], kernel_size=3, strides=1, padding='same', activation='relu')(x) #x = BatchNormalization()(x) x = Conv1D(filters=hparams['filters'] * 2, kernel_size=3, strides=1, padding='same', activation='relu')(x) x = Conv1D(filters=hparams['filters'] * 4, kernel_size=3, strides=1, padding='same', activation='relu')(x) #x = Permute((2,1))(x) #x = GlobalAveragePooling1D()(x) x = TimeDistributed(Dense(1, activation='linear'))(x) x = Reshape(target_shape=(final_dim, ))(x) self.model = Model(inputs=features_in, outputs=x) elif mode == 'conv2': x = Dense(units=10, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Shared_e_' + str(1))(features_in) x = Dense(units=10, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Shared_e_' + str(2))(x) end = Dense(units=10, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Dense_e_' + str(1))(x) end = Dense(units=10, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Dense_e_' + str(2))(end) end_node = Dense(units=1, activation='linear', kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='output_layer')(end) x = Dense(units=80, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(1))(x) x = Reshape(target_shape=(80, 1))(x) x = Conv1D(filters=8, kernel_size=3, strides=1, padding='same', activation='relu')(x) x = MaxPooling1D(pool_size=2)(x) x = Conv1D(filters=16, kernel_size=3, strides=1, padding='same', activation='relu')(x) x = MaxPooling1D(pool_size=2)(x) #x = Permute((2,1))(x) #x = GlobalAveragePooling1D()(x) x = TimeDistributed(Dense(1, activation='linear'))(x) x = Reshape(target_shape=(20, ))(x) self.model = Model(inputs=features_in, outputs=[end_node, x]) elif mode == 'lstm': x = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Shared_e_' + str(1))(features_in) x = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Shared_e_' + str(2))(x) end = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Dense_e_' + str(1))(x) end = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Dense_e_' + str(2))(end) end_node = Dense(units=1, activation='linear', kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='output_layer')(end) x = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(1))(x) x = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(2))(x) x = RepeatVector(n=20)(x) x = LSTM(units=30, activation='relu', return_sequences=True)(x) x = LSTM(units=30, activation='relu', return_sequences=True)(x) x = TimeDistributed(Dense(1))(x) x = Reshape(target_shape=(20, ))(x) ''' x = Permute((2,1))(x) x = GlobalAveragePooling1D()(x) ''' self.model = Model(inputs=features_in, outputs=[end_node, x]) optimizer = Adam(learning_rate=hparams['learning_rate'], clipnorm=1) def weighted_mse(y_true, y_pred): loss_weights = np.sqrt(np.arange(1, 20)) #loss_weights = np.arange(1, 20) return K.mean(K.square(y_pred - y_true) * loss_weights, axis=-1) def haitao_error(y_true, y_pred): diff = K.abs( (y_true - y_pred) / K.reshape(K.clip(K.abs(y_true[:, -1]), K.epsilon(), None), (-1, 1))) return 100. * K.mean(diff, axis=-1) if hparams['loss'] == 'mape': self.model.compile(optimizer=optimizer, loss=MeanAbsolutePercentageError()) elif hparams['loss'] == 'haitao': self.model.compile(optimizer=optimizer, loss=haitao_error) elif hparams['loss'] == 'mse': self.model.compile(optimizer=optimizer, loss='mean_squared_error')
# Model variables gpus = 1 batch_size = 1024 * gpus n_epoch = 100 n_hidden = 50 # Define the shared model x = Sequential() x.add( Embedding(len(embeddings), embedding_dim, weights=[embeddings], input_shape=(max_seq_length * 2, ), trainable=False)) x.add(LSTM(50)) x.add(Dense(1, activation='sigmoid')) x.compile(loss='mean_squared_error', optimizer=tf.keras.optimizers.Adam(), metrics=['accuracy']) x.summary() model = x try: # Start trainings training_start_time = time() callbacks = [EarlyStopping(monitor='val_loss', patience=3)] malstm_trained = model.fit(X_train, Y_train, batch_size=batch_size,
def engine(): if request.method == 'GET': algo = request.args.get('algo') ptype = request.args.get('ptype') tick = request.args.get('tick') daysx = request.args.get('daysx') if algo == "Delta": algo = "adadelta" elif algo == "Meta": algo = "adam" elif algo == "Gradient": algo = "adagrad" #importing the packages #part 1 daysx = int(daysx) import datetime as dt import urllib.request, json import pandas as pd #3 import numpy as np #3 # import matplotlib.pyplot as plt # from matplotlib.pylab import rcParams from sklearn.preprocessing import MinMaxScaler #used for setting the output figure size # rcParams['figure.figsize'] = 20,10 #to normalize the given input data scaler = MinMaxScaler(feature_range=(0, 1)) #to read input data set (place the file name inside ' ') as shown below ticker = tick api_key = '3T9YAWICQG9J42JM' url_string = "https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=%s&outputsize=compact&apikey=%s" % ( ticker, api_key) todataframe = pd.DataFrame() with urllib.request.urlopen(url_string) as url: datax = json.loads(url.read().decode()) datax = datax['Time Series (Daily)'] df = pd.DataFrame( columns=['Date', 'Open', 'High', 'Low', 'Close', 'Volume']) for k, v in datax.items(): date = dt.datetime.strptime(k, '%Y-%m-%d') data_row = [ date.date(), float(v['3. low']), float(v['2. high']), float(v['4. close']), float(v['1. open']), float(v['5. volume']) ] # print(data_row) df.loc[-1, :] = data_row df.index = df.index + 1 todataframe = df #todataframe.head() #print(todataframe) #part 2 #todataframe['Date'] = pd.to_datetime(todataframe.Date,format='%Y-%m-%d') #todataframe.index = todataframe['Date'] #plt.figure(figsize=(16,8)) #plt.plot(todataframe['Close'], label='Closing Price') #part 3 from sklearn.preprocessing import MinMaxScaler from tensorflow.python.keras.layers import Dense, Dropout, LSTM from tensorflow.python.keras import Sequential #dataframe creation seriesdata = todataframe.sort_index(ascending=True, axis=0) new_seriesdata = pd.DataFrame(index=range(0, len(todataframe)), columns=['Date', ptype]) length_of_data = len(seriesdata) for i in range(0, length_of_data): new_seriesdata['Date'][i] = seriesdata['Date'][i] new_seriesdata[ptype][i] = seriesdata[ptype][i] #setting the index again new_seriesdata.index = new_seriesdata.Date new_seriesdata.drop('Date', axis=1, inplace=True) #creating train and test sets this comprises the entire data’s present in the dataset myseriesdataset = new_seriesdata.values totrain = myseriesdataset[0:new_seriesdata.size, :] tovalid = myseriesdataset[new_seriesdata.size:, :] #print(len(totrain)) #print(len(tovalid)) #part 4 scalerdata = MinMaxScaler(feature_range=(0, 1)) scale_data = scalerdata.fit_transform(myseriesdataset) x_totrain, y_totrain = [], [] length_of_totrain = len(totrain) for i in range(60, length_of_totrain): x_totrain.append(scale_data[i - 60:i, 0]) y_totrain.append(scale_data[i, 0]) x_totrain, y_totrain = np.array(x_totrain), np.array(y_totrain) x_totrain = np.reshape(x_totrain, (x_totrain.shape[0], x_totrain.shape[1], 1)) #LSTM neural network lstm_model = Sequential() lstm_model.add( LSTM(units=50, return_sequences=True, input_shape=(x_totrain.shape[1], 1))) lstm_model.add(LSTM(units=50)) lstm_model.add(Dense(1)) lstm_model.compile(loss='mean_squared_error', optimizer=algo) lstm_model.fit(x_totrain, y_totrain, epochs=3, batch_size=1, verbose=2) #predicting next data stock price myinputs = new_seriesdata[len(new_seriesdata) - (len(tovalid) + daysx) - 60:].values myinputs = myinputs.reshape(-1, 1) myinputs = scalerdata.transform(myinputs) tostore_test_result = [] for i in range(60, myinputs.shape[0]): tostore_test_result.append(myinputs[i - 60:i, 0]) tostore_test_result = np.array(tostore_test_result) tostore_test_result = np.reshape( tostore_test_result, (tostore_test_result.shape[0], tostore_test_result.shape[1], 1)) myclosing_priceresult = lstm_model.predict(tostore_test_result) myclosing_priceresult = scalerdata.inverse_transform( myclosing_priceresult) #part 5 #print(len(tostore_test_result)); # print(myclosing_priceresult); xm = myclosing_priceresult.tolist() return jsonify(xm)
def make_model(self, embedding_matrics, max_tokens, X_train, X_test, Y_train, Y_test): """ @ set up model :param embedding_matrics: :param max_tokens: :param X_train: :param X_test: :param Y_train: :param Y_test: :return: """ model = Sequential() # embedding层的作用其实就是查表,从而将高维稀疏矩阵转变为低纬稠密矩阵 model.add( Embedding(TCconfig.num_words, TCconfig.embedding_dim, weights=[embedding_matrics], input_length=max_tokens, trainable=False)) # model.add(Bidirectional(LSTM(units=128, return_sequences=True))) # model.add(Dropout(0.5)) model.add(Bidirectional(LSTM(units=64, return_sequences=True))) model.add(Dropout(0.5)) model.add(Bidirectional(LSTM(units=32, return_sequences=False))) model.summary() print('Y_train.shape[1]:', Y_train.shape[1]) model.add(Dense(Y_train.shape[1], activation='softmax')) optimizer = Adam(lr=1e-3) model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) model.summary() # defin callbacks path_checkpoint = os.path.join( envipath, 'Checkpoints/news_classifier_checkpoint.h5') checkpoint = ModelCheckpoint(filepath=path_checkpoint, monitor='val_loss', verbose=1, save_weights_only=False, save_best_only=True) earlystopping = EarlyStopping(monitor='val_loss', patience=3, verbose=1) lr_reduction = ReduceLROnPlateau(monitor='val_loss', factor=0.1, min_lr=1e-5, patience=0, verbose=1) tensorboard = TensorBoard(log_dir=os.path.join(envipath, 'Log')) callbacks = [earlystopping, lr_reduction, checkpoint, tensorboard] print('start to train........') logger.info('start to train........') # train model.fit(X_train, Y_train, validation_split=0.1, epochs=20, batch_size=64, shuffle=True, callbacks=callbacks) # print('Y_test:',Y_test) Y_test = to_categorical(Y_test, num_classes=self.classnames.values().__len__()) result = model.evaluate(X_test, Y_test) logger.info('accuracy:{}'.format(result[1])) print('accuracy:{}'.format(result[1])) # save model print('saving model is done...') logger.info('saving model is done')
""" Simple model with one temporal image input and one temporal tensor output """ from tensorflow.python.keras import Model from tensorflow.python.keras.layers import Input, Dense, Concatenate, Flatten, TimeDistributed, LSTM from export import export_current_model input = Input(shape=(None, 64, 64, 1)) x = TimeDistributed(Flatten())(input) x = TimeDistributed(Dense(10))(x) x = LSTM(10, return_sequences=True)(x) model = Model(inputs=[input], outputs=[x]) print(model.summary()) output_name = model.output.name.split(':')[0] print('Name of output node:', output_name) export_current_model('models/temporal_input_temporal_output.pb', output_name)
def __init__(self, units=1, name=None, rnn_type='SimpleRNN', activation=linear, kernel_initializer=default_kernel_initializer(), recurrent_initializer=default_kernel_initializer(), bias_initializer=default_bias_initializer(), kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, trainable=True, dtype=None,): if not dtype: dtype = floatx() elif not dtype == floatx(): set_floatx(dtype) assert isinstance(name, str), \ "Please provide a string for field name. " assert callable(activation), \ "Please provide a function handle for the activation. " # prepare initializers. if isinstance(kernel_initializer, (float, int)): kernel_initializer = default_constant_initializer(kernel_initializer) if isinstance(bias_initializer, (float, int)): bias_initializer = default_constant_initializer(bias_initializer) # prepare regularizers. kernel_regularizer = default_regularizer(kernel_regularizer) bias_regularizer = default_regularizer(bias_regularizer) if rnn_type == 'SimpleRNN': super(RNNField, self).__init__( SimpleRNN( units=units, activation=activation, return_sequences=True, kernel_initializer=kernel_initializer, recurrent_initializer=recurrent_initializer, bias_initializer=bias_initializer, kernel_regularizer=kernel_regularizer, recurrent_regularizer=recurrent_regularizer, bias_regularizer=bias_regularizer, trainable=trainable, dtype=dtype, unroll=True, name=name ) ) elif rnn_type == 'LSTM': super(RNNField, self).__init__( LSTM( units=units, activation=activation, return_sequences=True, kernel_initializer=kernel_initializer, recurrent_initializer=recurrent_initializer, bias_initializer=bias_initializer, kernel_regularizer=kernel_regularizer, recurrent_regularizer=recurrent_regularizer, bias_regularizer=bias_regularizer, trainable=trainable, dtype=dtype, unroll=True, name=name ) ) elif rnn_type == 'Dense': super(RNNField, self).__init__( Dense( units=units, activation=activation, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, kernel_regularizer=kernel_regularizer, bias_regularizer=bias_regularizer, trainable=trainable, dtype=dtype, name=name ) ) else: raise NotImplementedError('Supported RNNType: (SimpleRNN, LSTM, Dense)')
break trainy = np.append(trainy, temp) temp = np.zeros((38)) trainF.close() np.savetxt('trainX.csv', trainX, delimiter=',') np.savetxt('trainy.csv', trainy, delimiter=',') # trainX = np.loadtxt('trainX.csv', delimiter=',') # trainy = np.loadtxt('trainy.csv', delimiter=',') # testX = np.loadtxt('testX.csv', delimiter=',') # testy = np.loadtxt('testy.csv', delimiter=',') optimizer = tf.keras.optimizers.SGD(lr=1e-12, momentum=0.1) model = Sequential() model.add(LSTM(128, return_sequences=True, input_shape=(None, 128))) model.add(LSTM(30, input_shape=(128, 1))) model.add(Dense(10)) model.add(Dense(38, activation='softmax')) model.compile(optimizer='adam', loss='mean_squared_error', metrics=[ tf.metrics.MeanSquaredError(name='my_mse'), tf.metrics.AUC(name='my_auc'), ]) trainX = trainX.reshape(trainX.shape[0], 1, trainX.shape[1]) testX = testX.reshape(testX.shape[0], 1, testX.shape[1]) trainX = np.array(trainX).transpose() testX = np.array(testX).transpose() trainy = trainy.reshape(148572, 38)
from tensorflow.python.keras.layers import LSTM data = [[i for i in range(100)]] data = np.array(data, dtype=float) target = [[i for i in range(1, 101)]] target = np.array(target, dtype=float) data = data.reshape((1, 1, 100)) target = target.reshape((1, 1, 100)) x_test = [i for i in range(100, 200)] x_test = np.array(x_test).reshape((1, 1, 100)) y_test = [i for i in range(101, 201)] y_test = np.array(y_test).reshape(1, 1, 100) model = Sequential() model.add(LSTM(100, input_shape=(1, 100), return_sequences=True)) model.add(Dense(100)) model.compile(loss='mean_absolute_error', optimizer='adam', metrics=['accuracy']) model.fit(data, target, nb_epoch=10000, batch_size=1, verbose=2, validation_data=(x_test, y_test)) predict = model.predict(data) print(predict)
X_train = np.reshape(np.asarray(X_train), (1, np.asarray(X_train).shape[0], np.asarray(X_train).shape[1])) y_train = np.reshape(np.asarray(y_train), (1, np.asarray(y_train).shape[0], np.asarray(y_train).shape[1])) """Creating the LSTM Model and importing the required libraries""" from tensorflow.python.keras import Sequential from tensorflow import keras from tensorflow.python.keras.layers import LSTM, Dropout, Dense, Conv2D, MaxPooling2D, Flatten,TimeDistributed from keras.optimizers import Adam, SGD """Adam Optimizer is used and the loss function used here is Mean Sqaured error.""" model = Sequential() model.add(LSTM(units=64, return_sequences = True, input_shape=(None, np.asarray(X_train).shape[2]), name="LSTM1")) model.add(LSTM(units=64, return_sequences = True, name="LSTM2")) model.add(LSTM(units=32, return_sequences = True, name="LSTM3")) model.add(LSTM(units=32, return_sequences = True, name="LSTM4")) model.add(LSTM(units=16, name="LSTM5")) model.add(Dense(units=6, name="Dense1")) opt = keras.optimizers.Adam(learning_rate=0.0001) model.compile(optimizer=opt,loss= 'mean_squared_error') hist = model.fit(X_train,y_train,epochs=100,batch_size=32) model.summary() plt.plot(hist.history['loss']) plt.title('Model Loss') plt.ylabel('Traning Loss')
def siamese_model(input_shape=None, track_length=1, features=None, neighborhood_scale_size=10, reg=1e-5, init='he_normal', softmax=True, norm_method='std', filter_size=61): def compute_input_shape(feature): if feature == 'appearance': return input_shape elif feature == 'distance': return (None, 2) elif feature == 'neighborhood': return (None, 2 * neighborhood_scale_size + 1, 2 * neighborhood_scale_size + 1, 1) elif feature == 'regionprop': return (None, 3) else: raise ValueError('siamese_model.compute_input_shape: ' 'Unknown feature `{}`'.format(feature)) def compute_reshape(feature): if feature == 'appearance': return (64, ) elif feature == 'distance': return (2, ) elif feature == 'neighborhood': return (64, ) elif feature == 'regionprop': return (3, ) else: raise ValueError('siamese_model.compute_output_shape: ' 'Unknown feature `{}`'.format(feature)) def compute_feature_extractor(feature, shape): if feature == 'appearance': # This should not stay: channels_first/last should be used to # dictate size (1 works for either right now) N_layers = np.int(np.floor(np.log2(input_shape[1]))) feature_extractor = Sequential() feature_extractor.add(InputLayer(input_shape=shape)) # feature_extractor.add(ImageNormalization2D('std', filter_size=32)) for layer in range(N_layers): feature_extractor.add( Conv3D(64, (1, 3, 3), kernel_initializer=init, padding='same', kernel_regularizer=l2(reg))) feature_extractor.add(BatchNormalization(axis=channel_axis)) feature_extractor.add(Activation('relu')) feature_extractor.add(MaxPool3D(pool_size=(1, 2, 2))) feature_extractor.add(Reshape((-1, 64))) return feature_extractor elif feature == 'distance': return None elif feature == 'neighborhood': N_layers_og = np.int( np.floor(np.log2(2 * neighborhood_scale_size + 1))) feature_extractor_neighborhood = Sequential() feature_extractor_neighborhood.add( InputLayer(input_shape=(None, 2 * neighborhood_scale_size + 1, 2 * neighborhood_scale_size + 1, 1))) for layer in range(N_layers_og): feature_extractor_neighborhood.add( Conv3D(64, (1, 3, 3), kernel_initializer=init, padding='same', kernel_regularizer=l2(reg))) feature_extractor_neighborhood.add( BatchNormalization(axis=channel_axis)) feature_extractor_neighborhood.add(Activation('relu')) feature_extractor_neighborhood.add( MaxPool3D(pool_size=(1, 2, 2))) feature_extractor_neighborhood.add(Reshape((-1, 64))) return feature_extractor_neighborhood elif feature == 'regionprop': return None else: raise ValueError('siamese_model.compute_feature_extractor: ' 'Unknown feature `{}`'.format(feature)) if features is None: raise ValueError('siamese_model: No features specified.') if K.image_data_format() == 'channels_first': channel_axis = 1 input_shape = tuple([input_shape[0], None] + list(input_shape[1:])) else: channel_axis = -1 input_shape = tuple([None] + list(input_shape)) features = sorted(features) inputs = [] outputs = [] for feature in features: in_shape = compute_input_shape(feature) re_shape = compute_reshape(feature) feature_extractor = compute_feature_extractor(feature, in_shape) layer_1 = Input(shape=in_shape) layer_2 = Input(shape=in_shape) inputs.extend([layer_1, layer_2]) # apply feature_extractor if it exists if feature_extractor is not None: layer_1 = feature_extractor(layer_1) layer_2 = feature_extractor(layer_2) # LSTM on 'left' side of network since that side takes in stacks of features layer_1 = LSTM(64)(layer_1) layer_2 = Reshape(re_shape)(layer_2) outputs.append([layer_1, layer_2]) dense_merged = [] for layer_1, layer_2 in outputs: merge = Concatenate(axis=channel_axis)([layer_1, layer_2]) dense_merge = Dense(128)(merge) bn_merge = BatchNormalization(axis=channel_axis)(dense_merge) dense_relu = Activation('relu')(bn_merge) dense_merged.append(dense_relu) # Concatenate outputs from both instances merged_outputs = Concatenate(axis=channel_axis)(dense_merged) # Add dense layers dense1 = Dense(128)(merged_outputs) bn1 = BatchNormalization(axis=channel_axis)(dense1) relu1 = Activation('relu')(bn1) dense2 = Dense(128)(relu1) bn2 = BatchNormalization(axis=channel_axis)(dense2) relu2 = Activation('relu')(bn2) dense3 = Dense(3, activation='softmax')(relu2) # Instantiate model final_layer = dense3 model = Model(inputs=inputs, outputs=final_layer) return model