Пример #1
0
def build_model(layers):
    model = Sequential()

    model.add(LSTM(
        input_shape = (None,layers[0]),
        units=layers[1],
        return_sequences=True))
    model.add(Dropout(0.4))


    model.add(LSTM(
        input_shape = (layers[1], 1024),
        units=1024,
        return_sequences=True))
    model.add(Dropout(0.4))

    model.add(LSTM(
        1024,
        return_sequences=False))
    model.add(Dropout(0.4))


    model.add(Dense(
        units=layers[3]))
    model.add(Activation("linear"))
    start = time.time()
    model.compile(loss="mse", optimizer="rmsprop",metrics=['accuracy'])
    print("> Compilation Time : ", time.time() - start)
    
    return model
Пример #2
0
def lstm(trainData, trainMark, testData, embedding_dim, embedding_matrix, maxlen, output_len):
    # 填充数据,将每个序列长度保持一致
    trainData = list(sequence.pad_sequences(trainData, maxlen=maxlen,
                                            dtype='float64'))  # sequence返回的是一个numpy数组,pad_sequences用于填充指定长度的序列,长则阶段,短则补0,由于下面序号为0时,对应值也为0,因此可以这样
    testData = list(sequence.pad_sequences(testData, maxlen=maxlen,
                                           dtype='float64'))  # sequence返回的是一个numpy数组,pad_sequences用于填充指定长度的序列,长则阶段,短则补0

    # 建立lstm神经网络模型
    model = Sequential()  # 多个网络层的线性堆叠,可以通过传递一个layer的list来构造该模型,也可以通过.add()方法一个个的加上层
    # model.add(Dense(256, input_shape=(train_total_vova_len,)))   #使用全连接的输入层
    model.add(Embedding(len(embedding_matrix), embedding_dim, weights=[embedding_matrix], mask_zero=False,
                        input_length=maxlen))  # 指定输入层,将高维的one-hot转成低维的embedding表示,第一个参数大或等于0的整数,输入数据最大下标+1,第二个参数大于0的整数,代表全连接嵌入的维度
    # lstm层,也是比较核心的层
    model.add(LSTM(256))  # 256对应Embedding输出维度,128是输入维度可以推导出来
    model.add(Dropout(0.5))  # 每次在参数更新的时候以一定的几率断开层的链接,用于防止过拟合
    model.add(Dense(output_len))  # 全连接,这里用于输出层,1代表输出层维度,128代表LSTM层维度可以自行推导出来
    model.add(Activation('softmax'))  # 输出用sigmoid激活函数
    # 编译该模型,categorical_crossentropy(亦称作对数损失,logloss),adam是一种优化器,class_mode表示分类模式
    model.compile(loss='categorical_crossentropy', optimizer='sgd')

    # 正式运行该模型,我知道为什么了,因为没有补0!!每个array的长度是不一样的,因此才会报错
    X = np.array(list(trainData))  # 输入数据
    print("X:", X)
    Y = np.array(list(trainMark))  # 标签
    print("Y:", Y)
    # batch_size:整数,指定进行梯度下降时每个batch包含的样本数
    # nb_epoch:整数,训练的轮数,训练数据将会被遍历nb_epoch次
    model.fit(X, Y, batch_size=200, nb_epoch=10)  # 该函数的X、Y应该是多个输入:numpy list(其中每个元素为numpy.array),单个输入:numpy.array

    # 进行预测
    A = np.array(list(testData))  # 输入数据
    print("A:", A)
    classes = model.predict(A)  # 这个是预测的数据
    return classes
def deep_CNN(input_shape, label_class):
    #Two Convolutional Layers with Pooling Layer
    #config
    nb_filter = [250, 150, 50]
    nb_row = [1, 1]
    nb_col = [1, 1]
    nb_pool_size = [1, 1]

    #init
    model = Sequential()

    #First layers
    model.add(Convolution2D( nb_filter = 27, nb_row = 3, nb_col = 1,
        border_mode = 'valid', activation = 'relu', 
        input_shape=(input_shape[0],input_shape[1],input_shape[2])
        ))
    model.add(Convolution2D( nb_filter = 48, nb_row = 3, nb_col = 300,
        border_mode = 'valid', activation = 'relu'
        ))

    #pooling layer
    model.add(MaxPooling2D( pool_size=(21,1) ))

    #Fully Connected Layer with dropout
    model.add(Flatten())
    model.add(Dense(output_dim = 256, activation = 'relu'))
    model.add(Dropout(0.5))

    #Fully Connected Layer as output layer
    model.add(Dense(output_dim = label_class, activation='sigmoid'));
    adadelta = Adadelta(lr=1.0, rho=0.95, epsilon=1e-6)
    model.compile(loss='binary_crossentropy', class_mode = 'categorical',
            optimizer = adadelta)
    return model
Пример #4
0
def train():

    print('Build model...')
    model = Sequential()
    model.add(Embedding(max_features, 128, input_length=maxlen, dropout=0.2))
    model.add(LSTM(128, dropout_W=0.2, dropout_U=0.2))  # try using a GRU instead, for fun
    model.add(Dense(1))
    model.add(Activation('sigmoid'))

    # try using different optimizers and different optimizer configs
    model.compile(loss='binary_crossentropy',
                  optimizer='adam',
                  metrics=['accuracy'])

    print('Train...')
    print(X_train.shape)
    print(y_train.shape)
    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=15,
              validation_data=(X_test, y_test))
    score, acc = model.evaluate(X_test, y_test,
                                batch_size=batch_size)
    print('Test score:', score)
    print('Test accuracy:', acc)

    with open("save_weight_lstm.pickle", mode="wb") as f:
        pickle.dump(model.get_weights(),f)
def parallel_CNN():
    filter_shapes = [ [2, 300], [3, 300], [4, 300], [5, 300] ]
    pool_shapes = [ [25, 1], [24, 1], [23, 1], [22, 1] ]

    #Four Parallel Convolutional Layers with Four Pooling Layers
    model = Sequential()
    sub_models = []
    for i in range( len(pool_shapes) ):
        pool_shape = pool_shapes[i]
        filter_shape = filter_shapes[i]
        sub_model = Sequential()
        sub_model.add( Convolution2D(nb_filter = 512, nb_row = filter_shape[0], nb_col = filter_shape[1], 
                border_mode='valid', activation='relu',
                input_shape=(input_shape[0], input_shape[1], input_shape[2])
                ))
        sub_model.add( MaxPooling2D(pool_size=(pool_shape[1], pool_shape[1])) )
        sub_models.append(sub_model)
    model.add(Merge(sub_models, mode='concat'))

    #Fully Connected Layer with dropout
    model.add(Flatten())
    model.add(Dense(output_dim=256, activation='relu', input_dim=2048))
    model.add(Dropout(0.5))

    #Fully Connected Layer as output layer
    model.add( Dense(output_dim=label_num, activation='sigmoid', input_dim=256))
    adadelta = Adadelta(lr=1.0, rho=0.95, epsilon=1e-6)
    model.compile(loss='binary_crossentropy', class_mode = 'multi_label',
            optimizer=adadelta)

    return model
Пример #6
0
def train_model():
    # (X_train, Y_train, X_test, Y_test) = prapare_train()
    X_ = []
    with open('../data/train_matrix.out') as train_file:
        X_train = json.load(train_file)
        for x in X_train:
            a = len(x)
            print a/2
            x1 = x[:a/2]
            x2 = x[a/2:]
            x3 = []
            x3.append(x1)
            x3.append(x2)
            X_.append(x3)
    # X_test = pickle.load('../data/test_matrix.out')
    Y_train = [1,0,0]*3
    # Y_test = [1,0,0]*3
    # print len(X_train) - len(Y_train)
    # print len(X_test) - len(Y_test)
    model = Sequential()
    model = get_nn_model()
    model.compile(loss='binary_crossentropy',
                optimizer='adam',
                metrics=['accuracy'])
    # model.fit(X_train, Y_train,
    #       batch_size=batch_size,
    #       nb_epoch=nb_epoch,
    #       validation_data=(X_test, Y_test))
#2
    model.fit(X_, Y_train,
          batch_size=batch_size,
          nb_epoch=nb_epoch,
          validation_split = 0.2)
    print 'ok'
Пример #7
0
def model(X_train, X_test, y_train, y_test, maxlen, max_features):
    embedding_size = 300
    pool_length = 4
    lstm_output_size = 100
    batch_size = 200
    nb_epoch = 1

    model = Sequential()
    model.add(Embedding(max_features, embedding_size, input_length=maxlen))
    model.add(Dropout({{uniform(0, 1)}}))
    # Note that we use unnamed parameters here, which is bad style, but is used here
    # to demonstrate that it works. Always prefer named parameters.
    model.add(Convolution1D({{choice([64, 128])}},
                            {{choice([6, 8])}},
                            border_mode='valid',
                            activation='relu',
                            subsample_length=1))
    model.add(MaxPooling1D(pool_length=pool_length))
    model.add(LSTM(lstm_output_size))
    model.add(Dense(1))
    model.add(Activation('sigmoid'))

    model.compile(loss='binary_crossentropy',
                  optimizer='adam',
                  metrics=['accuracy'])

    print('Train...')
    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch,
              validation_data=(X_test, y_test))
    score, acc = model.evaluate(X_test, y_test, batch_size=batch_size)

    print('Test score:', score)
    print('Test accuracy:', acc)
    return {'loss': -acc, 'status': STATUS_OK, 'model': model}
Пример #8
0
def test_sequential_model_saving():
    model = Sequential()
    model.add(Dense(2, input_shape=(3,)))
    model.add(RepeatVector(3))
    model.add(TimeDistributed(Dense(3)))
    model.compile(loss=losses.MSE,
                  optimizer=optimizers.RMSprop(lr=0.0001),
                  metrics=[metrics.categorical_accuracy],
                  sample_weight_mode='temporal')
    x = np.random.random((1, 3))
    y = np.random.random((1, 3, 3))
    model.train_on_batch(x, y)

    out = model.predict(x)
    _, fname = tempfile.mkstemp('.h5')
    save_model(model, fname)

    new_model = load_model(fname)
    os.remove(fname)

    out2 = new_model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

    # test that new updates are the same with both models
    x = np.random.random((1, 3))
    y = np.random.random((1, 3, 3))
    model.train_on_batch(x, y)
    new_model.train_on_batch(x, y)
    out = model.predict(x)
    out2 = new_model.predict(x)
    assert_allclose(out, out2, atol=1e-05)
Пример #9
0
def cnn():
    model = Sequential()

    # **Worth taking into consideration that our image size is tiny (8x8), convolution may work much better for
    # **with 1792 sipms

    # kernal size is 3x3, 32 filters, padding is same.
    # Same padding works better, this is probably because same padding makes it easier for network no to retain as
    # much information as possible around the edges.
    model.add(Convolution2D(32,3,3,border_mode='same',input_shape=(1, nsipm, nsipm)))
    model.add(Activation('relu'))
    model.add(Convolution2D(32,3,3,border_mode='same', input_shape=(32, nsipm, nsipm)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Flatten())
    model.add(Dense(output_dim=128))
    model.add(Activation('relu'))
    model.add(Dense(output_dim=64))
    model.add(Activation('relu'))
    model.add(Dense(output_dim=2*N_ELpts))
    model.add(Activation('sigmoid'))

    # Nadam optimizer is a safe choice at least for deep networks. It is adam optimizer with Nesterov
    # Momentum. Nesterov Momentum takes into account future expected future gradient gradient, unlike traditional Mom.
    model.compile(loss='mse', optimizer=Nadam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004))
    N_layers = 'cnn'
    return model,N_layers
Пример #10
0
def test_TensorBoard_with_ReduceLROnPlateau(tmpdir):
    import shutil
    np.random.seed(np.random.randint(1, 1e7))
    filepath = str(tmpdir / 'logs')

    (X_train, y_train), (X_test, y_test) = get_test_data(num_train=train_samples,
                                                         num_test=test_samples,
                                                         input_shape=(input_dim,),
                                                         classification=True,
                                                         num_classes=num_class)
    y_test = np_utils.to_categorical(y_test)
    y_train = np_utils.to_categorical(y_train)

    model = Sequential()
    model.add(Dense(num_hidden, input_dim=input_dim, activation='relu'))
    model.add(Dense(num_class, activation='softmax'))
    model.compile(loss='binary_crossentropy',
                  optimizer='sgd',
                  metrics=['accuracy'])

    cbks = [
        callbacks.ReduceLROnPlateau(
            monitor='val_loss',
            factor=0.5,
            patience=4,
            verbose=1),
        callbacks.TensorBoard(
            log_dir=filepath)]

    model.fit(X_train, y_train, batch_size=batch_size,
              validation_data=(X_test, y_test), callbacks=cbks, epochs=2)

    assert os.path.isdir(filepath)
    shutil.rmtree(filepath)
    assert not tmpdir.listdir()
Пример #11
0
def model_1(lr=.001, rho=.9, epsilon=1.0e-6):
    dnn = Sequential()
    dnn.add(BatchNormalization(input_shape=(3, 101, 101)))
    dnn.add(Convolution2D(16, 2, 2, init='he_normal'))
    dnn.add(MaxPooling2D())
    dnn.add(LeakyReLU(alpha=.01))
    dnn.add(Convolution2D(16, 3, 3, init='he_normal'))
    dnn.add(MaxPooling2D())
    dnn.add(LeakyReLU(alpha=.01))
    dnn.add(Convolution2D(16, 3, 3, init='he_normal'))
    dnn.add(MaxPooling2D())
    dnn.add(LeakyReLU(alpha=.01))
    dnn.add(Convolution2D(16, 2, 2, init='he_normal'))
    dnn.add(MaxPooling2D())
    dnn.add(LeakyReLU(alpha=.01))
    dnn.add(Convolution2D(16, 2, 2, init='he_normal'))
    dnn.add(MaxPooling2D())
    dnn.add(LeakyReLU(alpha=.01))
    dnn.add(Flatten())
    dnn.add(Dense(100))
    dnn.add(Dense(2))
    dnn.add(Activation('softmax'))
    dnn.compile(loss='binary_crossentropy', optimizer=Adamax(lr=lr))

    return dnn
Пример #12
0
def test_EarlyStopping():
    np.random.seed(1337)
    (X_train, y_train), (X_test, y_test) = get_test_data(num_train=train_samples,
                                                         num_test=test_samples,
                                                         input_shape=(input_dim,),
                                                         classification=True,
                                                         num_classes=num_class)
    y_test = np_utils.to_categorical(y_test)
    y_train = np_utils.to_categorical(y_train)
    model = Sequential()
    model.add(Dense(num_hidden, input_dim=input_dim, activation='relu'))
    model.add(Dense(num_class, activation='softmax'))
    model.compile(loss='categorical_crossentropy',
                  optimizer='rmsprop',
                  metrics=['accuracy'])
    mode = 'max'
    monitor = 'val_acc'
    patience = 0
    cbks = [callbacks.EarlyStopping(patience=patience, monitor=monitor, mode=mode)]
    history = model.fit(X_train, y_train, batch_size=batch_size,
                        validation_data=(X_test, y_test), callbacks=cbks, epochs=20)

    mode = 'auto'
    monitor = 'val_acc'
    patience = 2
    cbks = [callbacks.EarlyStopping(patience=patience, monitor=monitor, mode=mode)]
    history = model.fit(X_train, y_train, batch_size=batch_size,
                        validation_data=(X_test, y_test), callbacks=cbks, epochs=20)
Пример #13
0
def test_LambdaCallback():
    np.random.seed(1337)
    (X_train, y_train), (X_test, y_test) = get_test_data(num_train=train_samples,
                                                         num_test=test_samples,
                                                         input_shape=(input_dim,),
                                                         classification=True,
                                                         num_classes=num_class)
    y_test = np_utils.to_categorical(y_test)
    y_train = np_utils.to_categorical(y_train)
    model = Sequential()
    model.add(Dense(num_hidden, input_dim=input_dim, activation='relu'))
    model.add(Dense(num_class, activation='softmax'))
    model.compile(loss='categorical_crossentropy',
                  optimizer='sgd',
                  metrics=['accuracy'])

    # Start an arbitrary process that should run during model training and be terminated after training has completed.
    def f():
        while True:
            pass

    p = multiprocessing.Process(target=f)
    p.start()
    cleanup_callback = callbacks.LambdaCallback(on_train_end=lambda logs: p.terminate())

    cbks = [cleanup_callback]
    model.fit(X_train, y_train, batch_size=batch_size,
              validation_data=(X_test, y_test), callbacks=cbks, epochs=5)
    p.join()
    assert not p.is_alive()
Пример #14
0
	def ann(self):
		#print self.company.X_train.shape[1]
		model = Sequential()
		model.add(Dense(input_dim=self.search_inputs.X_train.shape[1], output_dim=10, init="glorot_uniform"))
		model.add(Activation('tanh'))
		model.add(Dropout(0.1))
		model.add(Dense(input_dim=10, output_dim=10, init="uniform"))
		model.add(Activation('tanh'))
		model.add(Dropout(0.5))
		model.add(Dense(input_dim=10, output_dim=1, init="glorot_uniform"))
		model.add(Activation("linear"))

		sgd = SGD(lr=0.3, decay=1e-6, momentum=0.9, nesterov=True)
		model.compile(loss='mean_squared_error', optimizer='sgd')
		early_stopping = EarlyStopping(monitor='val_loss', patience=25)
		#epoch_score = model.evaluate(X_score, y_score, batch_size = 16) # this doesn't work
		# first model
		print "fitting first model"
		model.fit(self.search_inputs.X_train, self.search_inputs.y_train, nb_epoch=100, validation_split=.1, batch_size=100, verbose = 1, show_accuracy = True, shuffle = True, callbacks=[early_stopping])
		#score = model.evaluate(self.company.X_cv, self.company.y_cv, show_accuracy=True, batch_size=16)
		self.ann_preds = model.predict(self.search_inputs.X_test)
		#just in case (like w/ svr)
		for i in range(0,len(self.ann_preds) - 1):
			if self.ann_preds[i] < 1:
				self.ann_preds[i] = 1.00
			elif self.ann_preds[i] > 3:
				self.ann_preds[i] = 3.00

		self.search_inputs.fin_df['relevance'] = np.array(self.ann_preds) # easy swap in / out 
		final_file_ann = self.search_inputs.fin_df.to_csv(self.fin_file_name+'_ann.csv', float_format='%.5f', index=False)
Пример #15
0
def define_model(lr, momentum):
    # CONFIG
    model = Sequential()

    # Create Layers
    # CONVNET
    layers = []
    #layers.append(GaussianNoise(0.02))
    layers.append(Convolution2D(8, 9, 9, activation = "relu", input_shape=(1,100,100)))
    layers.append(MaxPooling2D(pool_size=(2,2)))
    layers.append(Convolution2D(16, 7, 7, activation = "relu"))
    layers.append(MaxPooling2D(pool_size=(2,2)))
    layers.append(Convolution2D(32, 5, 5, activation = "relu"))
    layers.append(MaxPooling2D(pool_size=(2,2)))
    layers.append(Convolution2D(64, 3, 3, activation = "relu"))
    layers.append(MaxPooling2D(pool_size=(2,2)))
    layers.append(Convolution2D(250, 3, 3, activation= "relu"))
    # MLP
    layers.append(Flatten())
    layers.append(Dense(125, activation="relu"))
    layers.append(Dense(2, activation="softmax"))

    # Adding Layers
    for layer in layers:
        model.add(layer)

    # COMPILE (learning rate, momentum, objective...)
    sgd = SGD(lr=lr, momentum=momentum)

    model.compile(loss="categorical_crossentropy", optimizer=sgd)

    return model
Пример #16
0
def test_merge_overlap():
    left = Sequential()
    left.add(Dense(nb_hidden, input_shape=(input_dim,)))
    left.add(Activation('relu'))

    model = Sequential()
    model.add(Merge([left, left], mode='sum'))
    model.add(Dense(nb_class))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=1, validation_data=(X_test, y_test))
    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=False, verbose=2, validation_data=(X_test, y_test))
    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=2, validation_split=0.1)
    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=False, verbose=1, validation_split=0.1)
    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0)
    model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1, shuffle=False)

    model.train_on_batch(X_train[:32], y_train[:32])

    loss = model.evaluate(X_train, y_train, verbose=0)
    assert(loss < 0.7)
    model.predict(X_test, verbose=0)
    model.predict_classes(X_test, verbose=0)
    model.predict_proba(X_test, verbose=0)
    model.get_config(verbose=0)

    fname = 'test_merge_overlap_temp.h5'
    model.save_weights(fname, overwrite=True)
    model.load_weights(fname)
    os.remove(fname)

    nloss = model.evaluate(X_train, y_train, verbose=0)
    assert(loss == nloss)
class QLearn:
    def __init__(self, actions, epsilon, alpha, gamma):
        
        # instead of a dictionary, we'll be using
        #   a neural network
        # self.q = {}
        self.epsilon = epsilon  # exploration constant
        self.alpha = alpha      # discount constant
        self.gamma = gamma      # discount factor
        self.actions = actions
        
        # Build the neural network
        self.network = Sequential()
        self.network.add(Dense(50, init='lecun_uniform', input_shape=(4,)))
        # self.network.add(Activation('sigmoid'))
        #self.network.add(Dropout(0.2))

        self.network.add(Dense(20, init='lecun_uniform'))
        # self.network.add(Activation('sigmoid'))
        # #self.network.add(Dropout(0.2))

        self.network.add(Dense(2, init='lecun_uniform'))
        # self.network.add(Activation('linear')) #linear output so we can have range of real-valued outputs

        # rms = SGD(lr=0.0001, decay=1e-6, momentum=0.5) # explodes to non
        rms = RMSprop()
        # rms = Adagrad()
        # rms = Adam()
        self.network.compile(loss='mse', optimizer=rms)
        # Get a summary of the network
        self.network.summary()
Пример #18
0
def train_rnn(character_corpus, seq_len, train_test_split_ratio):
    model = Sequential()
    model.add(Embedding(character_corpus.char_num(), 256))
    model.add(LSTM(256, 5120, activation='sigmoid', inner_activation='hard_sigmoid', return_sequences=True))
    model.add(Dropout(0.5))
    model.add(TimeDistributedDense(5120, character_corpus.char_num()))
    model.add(Activation('time_distributed_softmax'))

    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

    seq_X, seq_Y = character_corpus.make_sequences(seq_len)

    print "Sequences are made"

    train_seq_num = train_test_split_ratio*seq_X.shape[0]
    X_train = seq_X[:train_seq_num]
    Y_train = to_time_distributed_categorical(seq_Y[:train_seq_num], character_corpus.char_num())

    X_test = seq_X[train_seq_num:]
    Y_test = to_time_distributed_categorical(seq_Y[train_seq_num:], character_corpus.char_num())

    print "Begin train model"
    checkpointer = ModelCheckpoint(filepath="model.step", verbose=1, save_best_only=True)
    model.fit(X_train, Y_train, batch_size=256, nb_epoch=100, verbose=2, validation_data=(X_test, Y_test), callbacks=[checkpointer])

    print "Model is trained"

    score = model.evaluate(X_test, Y_test, batch_size=512)

    print "valid score = ", score

    return model
Пример #19
0
def test_merge_dot():
    if K._BACKEND == 'tensorflow':
        return

    left = Sequential()
    left.add(Dense(input_dim=input_dim, output_dim=nb_hidden))
    left.add(Activation('relu'))

    right = Sequential()
    right.add(Dense(input_dim=input_dim, output_dim=nb_hidden))
    right.add(Activation('relu'))

    model = Sequential()
    model.add(Merge([left, right], mode='dot', dot_axes=1))
    model.add(Dense(nb_class))
    model.add(Activation('softmax'))

    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

    left = Sequential()
    left.add(Dense(input_dim=input_dim, output_dim=nb_hidden))
    left.add(Activation('relu'))

    right = Sequential()
    right.add(Dense(input_dim=input_dim, output_dim=nb_hidden))
    right.add(Activation('relu'))

    model = Sequential()
    model.add(Merge([left, right], mode='dot', dot_axes=([1], [1])))
    model.add(Dense(nb_class))
    model.add(Activation('softmax'))

    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
Пример #20
0
def test_sequential_model_saving():
    model = Sequential()
    model.add(Dense(2, input_dim=3))
    model.add(Dense(3))
    model.compile(loss='mse', optimizer='rmsprop', metrics=['acc'])

    x = np.random.random((1, 3))
    y = np.random.random((1, 3))
    model.train_on_batch(x, y)

    out = model.predict(x)
    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)

    new_model = load_model(fname)

    out2 = new_model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

    # test that new updates are the same with both models
    x = np.random.random((1, 3))
    y = np.random.random((1, 3))
    model.train_on_batch(x, y)
    new_model.train_on_batch(x, y)
    out = model.predict(x)
    out2 = new_model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

    # test load_weights on model file
    model.load_weights(fname)
    os.remove(fname)
Пример #21
0
def test_sequential_model_saving_2():
    # test with funkier config
    model = Sequential()
    model.add(Dense(2, input_dim=3))
    model.add(RepeatVector(3))
    model.add(TimeDistributed(Dense(3)))
    model.compile(loss=objectives.MSE,
                  optimizer=optimizers.RMSprop(lr=0.0001),
                  metrics=[metrics.categorical_accuracy],
                  sample_weight_mode='temporal')
    x = np.random.random((1, 3))
    y = np.random.random((1, 3, 3))
    model.train_on_batch(x, y)

    out = model.predict(x)
    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)

    new_model = load_model(fname)
    os.remove(fname)

    out2 = new_model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

    # test that new updates are the same with both models
    x = np.random.random((1, 3))
    y = np.random.random((1, 3, 3))
    model.train_on_batch(x, y)
    new_model.train_on_batch(x, y)
    out = model.predict(x)
    out2 = new_model.predict(x)
    assert_allclose(out, out2, atol=1e-05)
Пример #22
0
def model(X_train, Y_train, X_test, Y_test):
    '''
    Model providing function:

    Create Keras model with double curly brackets dropped-in as needed.
    Return value has to be a valid python dictionary with two customary keys:
        - loss: Specify a numeric evaluation metric to be minimized
        - status: Just use STATUS_OK and see hyperopt documentation if not feasible
    The last one is optional, though recommended, namely:
        - model: specify the model just created so that we can later use it again.
    '''
    model = Sequential()
    model.add(Dense(512, input_shape=(784,)))
    model.add(Activation('relu'))
    model.add(Dropout({{uniform(0, 1)}}))
    model.add(Dense({{choice([256, 512, 1024])}}))
    model.add(Activation('relu'))
    model.add(Dropout({{uniform(0, 1)}}))
    model.add(Dense(10))
    model.add(Activation('softmax'))

    rms = RMSprop()
    model.compile(loss='categorical_crossentropy', optimizer=rms, metrics=['accuracy'])

    model.fit(X_train, Y_train,
              batch_size={{choice([64, 128])}},
              nb_epoch=1,
              verbose=2,
              validation_data=(X_test, Y_test))
    score, acc = model.evaluate(X_test, Y_test, verbose=0)
    print('Test accuracy:', acc)
    return {'loss': -acc, 'status': STATUS_OK, 'model': model}
Пример #23
0
  def __init__(self, nb_filters=32, nb_conv=3, nb_pool=2):
    model = Sequential()
    model.add(Convolution2D(nb_filters, nb_conv, nb_conv,
                        border_mode='valid',
                        input_shape=(1, 5, 30)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
    model.add(Convolution2D(nb_filters/2, nb_conv, nb_conv))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
    model.add(Dropout(0.2))

    model.add(Flatten())
    model.add(Dense(500))
    model.add(Dropout(0.2))
    model.add(Activation('relu'))
    model.add(Dense(500))
    model.add(Dropout(0.2))
    model.add(Activation('relu'))
    model.add(Dense(10))
    model.add(Activation('softmax'))

    model.compile(loss='categorical_crossentropy',
                  optimizer='adadelta',
                  metrics=['accuracy'])

    self.model = model
Пример #24
0
def simple_cnn_vgg_like(lr=1e-3, weights_path=None):
    img_rows, img_cols = 210, 70
    # standard VGG16 network architecture
    
    structure_path = "%s/cache/simple_cnn_vgg_like.json" % config.project.project_path
    if weights_path is not None and os.path.exists(weights_path) \
        and os.path.exists(structure_path):

        logger.debug("load weigth from fine-tuning weight %s" % weights_path)
        model = model_from_json(open(structure_path).read())
        model.load_weights(weights_path)
    else:
        model = Sequential()
        model.add(ZeroPadding2D((1, 1), input_shape=(1, img_rows, img_cols)))
        model.add(Convolution2D(64, 7, 7, activation='relu'))
        model.add(ZeroPadding2D((1, 1)))
        model.add(Convolution2D(64, 7, 7, activation='relu'))
        model.add(MaxPooling2D((2, 2), strides=(2, 2)))

        model.add(Flatten())
        model.add(Dense(4096, activation='relu'))
        model.add(Dropout(0.5))

        # replace more fc layer
        model.add(Dense(124, activation='softmax'))

        # load the weights
        logger.debug('Model loaded.')

    sgd = SGD(lr=lr, decay=1e-6, momentum=0.9, nesterov=True)
    model.compile(optimizer=sgd, loss='categorical_crossentropy',  metrics=['accuracy'])

    return model
Пример #25
0
def simple_cnn_for_test(lr=1e-3, weights_path=None):
    img_rows, img_cols = 210, 70
    if weights_path is not None and os.path.exists(weights_path):
        logging.debug("load weigth from fine-tuning weight %s" % weights_path)
        model = load_model(weights_path)
    else:
        model = Sequential()
        model.add(ZeroPadding2D((1, 1), input_shape=(1, img_rows, img_cols)))
        model.add(Convolution2D(6, 3, 3, activation='relu'))
        model.add(ZeroPadding2D((1, 1)))
        model.add(Convolution2D(6, 3, 3, activation='relu'))
        model.add(MaxPooling2D((2, 2), strides=(2, 2)))

        model.add(ZeroPadding2D((1, 1)))
        model.add(Convolution2D(6, 3, 3, activation='relu'))
        model.add(ZeroPadding2D((1, 1)))
        model.add(Convolution2D(6, 3, 3, activation='relu'))

        model.add(Flatten())
        # replace more fc layer
        model.add(Dense(124, activation='softmax'))

        # load the weights
        logging.debug('Model loaded.')

    sgd = SGD(lr=lr, decay=1e-6, momentum=0.9, nesterov=True)
    model.compile(optimizer=sgd, loss='categorical_crossentropy',  metrics=['accuracy'])
    return model
def baseline_model():
  model = Sequential()
  model.add(Dense(4, input_dim=4, init='normal', activation='relu'))
  model.add(Dense(3, init='normal', activation='sigmoid'))
  model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
  
  return model
def get_ts_model( trainX, trainY, look_back = 1, nb_epochs = 100 ):
    model = Sequential()
    # takes input array of shape (*, 1) where (2,1) - (row,col) array example looks like [23]
    # 																					 [43]
    model.add(LSTM(20, input_shape=(None , look_back) ))
    #model.add(LSTM(20,  batch_input_shape=(None, None, look_back), return_sequences= True ))
    #print(model.summary)
    model.add( Dense(1) )
    model.add(Dense(1))
    model.add(Dense(1))
    model.add(Dense(1))
    model.add(Dense(1))
    model.add(Dense(1))
    #model.add(LSTM(1, return_sequences= True))
    #model.add(LSTM(1))
    # outputs array of shape (*,1)
    #model.add(Dense(1))
    #model.compile(loss='mean_absolute_error', optimizer='SGD')  # mape
    #model.compile(loss='poisson', optimizer='adam')  # mape
    model.compile( loss =  'mean_squared_error', optimizer = 'adam' ) # values closer to zero are better.
    #model.compile(loss='mean_squared_error', optimizer='adagrad')
    # Values of MSE are used for comparative purposes of two or more statistical meythods. Heavily weight outliers,  i.e weighs large errors more heavily than the small ones.
    # "In cases where this is undesired, mean absolute error is used.
    # REF: Available loss functions  https://keras.io/objectives.
    print('Start : Training model')
    # default  configuration
    model.fit(trainX, trainY, nb_epoch=nb_epochs, batch_size=1, verbose=2)
    #model.fit(trainX, trainY, nb_epoch=100, batch_size=1, verbose=2)
    print('Ends : Training Model')
    return model
Пример #28
0
def create_model(x_train, y_train, x_test, y_test):
    """
    Create your model...
    """
    layer_1_size = {{quniform(12, 256, 4)}}
    l1_dropout = {{uniform(0.001, 0.7)}}
    params = {
        'l1_size': layer_1_size,
        'l1_dropout': l1_dropout
    }
    num_classes = 10
    model = Sequential()
    model.add(Dense(int(layer_1_size), activation='relu'))
    model.add(Dropout(l1_dropout))
    model.add(Dense(num_classes, activation='softmax'))
    model.compile(loss='categorical_crossentropy',
                  optimizer=RMSprop(),
                  metrics=['accuracy'])
    model.fit(x_train, y_train, batch_size=128, epochs=10, validation_data=(x_test, y_test))
    score, acc = model.evaluate(x_test, y_test, verbose=0)
    out = {
        'loss': -acc,
        'score': score,
        'status': STATUS_OK,
        'model_params': params,
    }
    # optionally store a dump of your model here so you can get it from the database later
    temp_name = tempfile.gettempdir()+'/'+next(tempfile._get_candidate_names()) + '.h5'
    model.save(temp_name)
    with open(temp_name, 'rb') as infile:
        model_bytes = infile.read()
    out['model_serial'] = model_bytes
    return out
Пример #29
0
    def _small_model(self):
        '''
        Alternative model architecture with fewer layers for computationally expensive
            training datasets
        '''
        print 'Compiling Small Net...'

        model = Sequential()
        model.add(ZeroPadding2D((1,1), input_shape=self.input_shape))
        model.add(Convolution2D(64, self.kernel_size, self.kernel_size,activation='relu',
                                input_shape=self.input_shape))
        model.add(ZeroPadding2D((1,1)))
        model.add(Convolution2D(64, self.kernel_size, self.kernel_size,
                                activation='relu'))
        model.add(MaxPooling2D((2,2), strides=(2,2)))

        model.add(ZeroPadding2D((1,1)))
        model.add(Convolution2D(128, self.kernel_size, self.kernel_size,
                                activation='relu'))
        model.add(ZeroPadding2D((1,1)))
        model.add(Convolution2D(128, self.kernel_size, self.kernel_size,
                                activation='relu'))
        model.add(MaxPooling2D((2,2), strides=(2,2)))

        model.add(Flatten())
        model.add(Dense(2048, activation='relu'))
        model.add(Dropout(0.5))
        model.add(Dense(2048, activation='relu'))
        model.add(Dropout(0.5))
        model.add(Dense(self.nb_classes, activation='softmax'))

        sgd = SGD(lr=self.lr, decay=0.01, momentum=0.9, nesterov=True)
        model.compile(optimizer = 'sgd', loss = 'categorical_crossentropy')
        return model
    def compile_model(self):
        '''
        compiles standard single model with 4 convolitional/max-pooling layers.
        '''
        print 'Compiling single model...'
        single = Sequential()

        single.add(Convolution2D(self.n_filters[0], self.k_dims[0], self.k_dims[0], border_mode='valid', W_regularizer=l1l2(l1=self.w_reg, l2=self.w_reg), input_shape=(self.n_chan,33,33)))
        single.add(Activation(self.activation))
        single.add(BatchNormalization(mode=0, axis=1))
        single.add(MaxPooling2D(pool_size=(2,2), strides=(1,1)))
        single.add(Dropout(0.5))
        single.add(Convolution2D(self.n_filters[1], self.k_dims[1], self.k_dims[1], activation=self.activation, border_mode='valid', W_regularizer=l1l2(l1=self.w_reg, l2=self.w_reg)))
        single.add(BatchNormalization(mode=0, axis=1))
        single.add(MaxPooling2D(pool_size=(2,2), strides=(1,1)))
        single.add(Dropout(0.5))
        single.add(Convolution2D(self.n_filters[2], self.k_dims[2], self.k_dims[2], activation=self.activation, border_mode='valid', W_regularizer=l1l2(l1=self.w_reg, l2=self.w_reg)))
        single.add(BatchNormalization(mode=0, axis=1))
        single.add(MaxPooling2D(pool_size=(2,2), strides=(1,1)))
        single.add(Dropout(0.5))
        single.add(Convolution2D(self.n_filters[3], self.k_dims[3], self.k_dims[3], activation=self.activation, border_mode='valid', W_regularizer=l1l2(l1=self.w_reg, l2=self.w_reg)))
        single.add(Dropout(0.25))

        single.add(Flatten())
        single.add(Dense(5))
        single.add(Activation('softmax'))

        sgd = SGD(lr=0.001, decay=0.01, momentum=0.9)
        single.compile(loss='categorical_crossentropy', optimizer='sgd')
        print 'Done.'
        return single
Пример #31
0
class LanguageModel(RNNModel):
    def __init__(self, *args, **kwargs):
        '''
        field is 'sentence' by default, but can be e.g. pos_sentence.
        '''
        self.field = kwargs.pop('field', 'sentence')
        super(LanguageModel, self).__init__(*args, **kwargs)
        self.class_to_code = {'VBZ': 0, 'VBP': 1}
        self.inflect_verb, _ = gen_inflect_from_vocab(self.vocab_file)

    def process_single_dependency(self, dep):
        dep['label'] = dep['verb_pos']
        tokens = dep[self.field].split()
        return tokens

    def create_train_and_test(self, examples):
        random.seed(1)
        random.shuffle(examples)

        first = 1
        self.X_train = []
        self.Y_train = []
        self.X_test = self.Y_test = []  # not used; just for compatibility
        self.deps_train = []
        n_train = int(len(examples) * self.prop_train)
        for _, ints, dep in examples[:n_train]:
            self.deps_train.append(dep)
            for i in range(first, len(ints) - 1):
                self.X_train.append(ints[:i])
                self.Y_train.append(ints[i])

        self.Y_train = np.asarray(self.Y_train)
        self.deps_test = [x[2] for x in examples[n_train:]]

    def create_model(self):
        self.log('Creating model')
        self.model = Sequential()
        self.model.add(
            Embedding(len(self.vocab_to_ints) + 1,
                      self.embedding_size,
                      input_length=self.maxlen))
        self.model.add(
            self.rnn_class(output_dim=self.rnn_output_size,
                           input_length=self.maxlen))
        self.model.add(Dense(len(self.vocab_to_ints) + 1))
        self.model.add(Activation('softmax'))

    def compile_model(self):
        self.log('Compiling model')
        self.model.compile(loss='sparse_categorical_crossentropy',
                           optimizer='adam')

    def results(self):
        recs = []
        columns = ['gram_loss', 'ungram_loss', 'correct'] + dependency_fields
        self.model.model._make_test_function()
        progbar = Progbar(len(self.deps_test))
        for i, dep in enumerate(self.deps_test):
            inp = np.zeros((1, self.maxlen))
            v = int(dep['verb_index']) - 1
            tokens = dep[self.field].split()[:v + 1]
            ints = [self.vocab_to_ints[x] for x in tokens]
            try:
                ungram = self.vocab_to_ints[self.inflect_verb[tokens[v]]]
            except KeyError:  # reinflected form not in vocabulary: ignore
                continue
            n = len(ints) - 1
            inp[0, -n:] = ints[:-1]
            gram_loss = self.model.test_on_batch(inp, np.array([ints[v]]))
            ungram_loss = self.model.test_on_batch(inp, np.array([ungram]))
            recs.append((gram_loss, ungram_loss, gram_loss < ungram_loss) +
                        tuple(dep[x] for x in dependency_fields))
            if i % 16 == 0:
                progbar.update(i)

        self.test_results = pd.DataFrame(recs, columns=columns)

    def train(self, n_epochs=10):
        if not hasattr(self, 'model'):
            self.create_model()
            self.compile_model()

        self.serialize_class_data()
        self.serialize_model()

        validation_split = 0.1
        split_at = int(len(self.X_train) * (1. - validation_split))
        x, val_x = self.X_train[:split_at], self.X_train[split_at:]
        y, val_y = self.Y_train[:split_at], self.Y_train[split_at:]
        training_loss_history = []
        validation_loss_history = []

        for epoch in range(n_epochs):
            print 'Epoch', epoch
            training_loss = []
            end = int(float(len(x)) / self.batch_size)
            progbar = Progbar(end)
            for i in range(0, len(x), self.batch_size):
                inp = sequence.pad_sequences(x[i:i + self.batch_size],
                                             maxlen=self.maxlen)
                out = y[i:i + self.batch_size]
                loss = self.model.train_on_batch(inp, out)
                training_loss.append(loss)
                j = int(float(i) / self.batch_size)
                if j % 16 == 0:
                    progbar.update(j)
            progbar.update(end)

            # test on validation set
            validation_loss = []
            print
            print 'Evaluating on validation set:'
            end = int(float(len(val_x)) / self.batch_size)
            progbar = Progbar(end)
            for i in range(0, len(val_x), self.batch_size):
                inp = sequence.pad_sequences(val_x[i:i + self.batch_size],
                                             maxlen=self.maxlen)
                out = val_y[i:i + self.batch_size]
                output = self.model.test_on_batch(inp, out)
                validation_loss.append(output)
                j = int(float(i) / self.batch_size)
                if j % 16 == 0:
                    progbar.update(j)
            progbar.update(end)

            training_loss_history.append(np.mean(training_loss))
            validation_loss_history.append(np.mean(validation_loss))
            filename = op.join(self.serialization_dir,
                               'weights_epoch%d.h5' % epoch)
            self.model.save_weights(filename, overwrite=True)
            print
            print('Mean training loss: %5.3f; mean validation loss: %5.3f\n' %
                  (training_loss_history[-1], validation_loss_history[-1]))
            if (len(validation_loss_history) > 1
                    and validation_loss_history[-1] >=
                    validation_loss_history[-2]):
                break

        self.training_history = (map(float, training_loss_history),
                                 map(float, validation_loss_history))

    def evaluate(self, howmany=1000):
        self.model.model._make_test_function()
        random.seed(0)
        shuffled = self.deps_test[:]
        random.shuffle(shuffled)
        shuffled = shuffled[:howmany]
        X_test = []
        Y_test = []

        for dep in shuffled:
            tokens = self.process_single_dependency(dep)
            ints = []
            for token in tokens:
                if token not in self.vocab_to_ints:
                    # zero is for pad
                    x = self.vocab_to_ints[token] = len(self.vocab_to_ints) + 1
                    self.ints_to_vocab[x] = token
                ints.append(self.vocab_to_ints[token])

            first = 1
            for i in range(first, len(ints) - 1):
                X_test.append(ints[:i])
                Y_test.append(ints[i])

        test_loss = []
        end = int(float(len(X_test) / self.batch_size))
        progbar = Progbar(end)
        for i in range(0, len(X_test), self.batch_size):
            inp = sequence.pad_sequences(X_test[i:i + self.batch_size],
                                         maxlen=self.maxlen)
            out = Y_test[i:i + self.batch_size]
            output = self.model.test_on_batch(inp, out)
            test_loss.append(output)
            j = int(float(i) / self.batch_size)
            if j % 16 == 0:
                progbar.update(j)
        progbar.update(end)

        return np.mean(test_loss)
Пример #32
0
X_train = np.reshape(X_train, (X_train.shape[0], 1, X_train.shape[1]))
X_test = np.reshape(X_test, (X_test.shape[0], 1, X_test.shape[1]))

model = Sequential()
model.add(SimpleRNN(units=128, input_shape=(1, step), activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(32, activation="relu"))
model.add(Dropout(0.05))
model.add(Dense(32, activation="relu"))
model.add(Dense(16, activation="relu"))
model.add(Dense(8, activation="relu"))
model.add(Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy',
              optimizer='rmsprop',
              metrics=['accuracy'])
model.summary()

#모델 학습
optimizer_history = model.fit(X_train,
                              y_train,
                              epochs=300,
                              batch_size=1500,
                              verbose=1,
                              validation_data=(X_test, y_test))

#모델 평가
results = model.evaluate(X_test, y_test)

print('loss: ', results[0])
Пример #33
0
model.add(Dropout(0.25))
# Слой преобразования данных из 2D представления в плоское
model.add(Flatten())
# Полносвязный слой для классификации
model.add(Dense(512, activation='relu'))
# Слой регуляризации Dropout
model.add(Dropout(0.5))
# Выходной полносвязный слой
model.add(Dense(nb_classes, activation='softmax'))

# Задаем параметры оптимизации
# lr = 0.01
lr = 0.02
sgd = SGD(lr=lr, decay=1e-6, momentum=0.9, nesterov=True)

model.compile(loss='categorical_crossentropy',
              optimizer=sgd,
              metrics=['accuracy'])
# Обучаем модель
model.fit(X_train,
          Y_train,
          batch_size=batch_size,
          nb_epoch=nb_epoch,
          validation_split=0.1,
          shuffle=True)

# Оцениваем качество обучения модели на тестовых данных
scores = model.evaluate(X_test, Y_test, verbose=0)

print("Точность работы на тестовых данных:")
print(round(scores[1] * 100, 3))
Пример #34
0
                name='dense8'))
model.add(BatchNormalization(epsilon=epsilon, momentum=momentum, name='bn8'))
model.add(Activation(binary_tanh, name='act8'))
# dense3
model.add(
    BinaryDense(classes,
                H=H,
                kernel_lr_multiplier=kernel_lr_multiplier,
                use_bias=use_bias,
                name='dense9'))
model.add(BatchNormalization(epsilon=epsilon, momentum=momentum, name='bn9'))

# In[11]:

opt = Adam(lr=lr_start)
model.compile(loss='squared_hinge', optimizer=opt, metrics=['acc'])
model.summary()

# In[14]:

epochs = 50
lr_scheduler = LearningRateScheduler(lambda e: lr_start * lr_decay**e)
history = model.fit(X_train,
                    Y_train,
                    batch_size=batch_size,
                    epochs=epochs,
                    verbose=1,
                    validation_data=(X_test, Y_test),
                    callbacks=[lr_scheduler])
score = model.evaluate(X_test, Y_test, verbose=0)
print('Test score:', score[0])
Пример #35
0
def main():
    # used to get the session/graph data from keras
    K.set_learning_phase(0)
    # get the data in a Pandas dataframe
    raw_data = pd.read_csv(FLAGS.csv_file)

    # convert to one hot vectors
    emotion_array = process_emotion(raw_data[['emotion']])
    # convert to a 48x48 float matrix
    pixel_array = process_pixels(raw_data[['pixels']])

    # split for test/train
    y_train, y_test = split_for_test(emotion_array)
    x_train_matrix, x_test_matrix = split_for_test(pixel_array)

    n_train = int(len(x_train_matrix))
    n_test = int(len(x_test_matrix))

    x_train_input = duplicate_input_layer(x_train_matrix, n_train)
    x_test_input = duplicate_input_layer(x_test_matrix, n_test)

    # vgg 16. include_top=False so the output is the 512 and use the learned weights
    vgg16 = VGG16(include_top=False, input_shape=(48, 48, 3), pooling='avg', weights='imagenet')

    # get vgg16 outputs
    x_train_feature_map = get_vgg16_output(vgg16, x_train_matrix, n_train)
    x_test_feature_map = get_vgg16_output(vgg16, x_test_matrix, n_test)

    # build and train model
    top_layer_model = Sequential()
    top_layer_model.add(Dense(256, input_shape=(512,), activation='relu'))
    top_layer_model.add(Dense(256, input_shape=(256,), activation='relu'))
    top_layer_model.add(Dropout(0.5))
    top_layer_model.add(Dense(128, input_shape=(256,)))
    top_layer_model.add(Dense(NUM_CLASSES, activation='softmax'))

    adamax = Adamax()

    top_layer_model.compile(loss='categorical_crossentropy',
                            optimizer=adamax, metrics=['accuracy'])

    # train
    top_layer_model.fit(x_train_feature_map, y_train,
                        validation_data=(x_train_feature_map, y_train),
                        nb_epoch=FLAGS.n_epochs, batch_size=FLAGS.batch_size)
    # Evaluate
    score = top_layer_model.evaluate(x_test_feature_map,
                                     y_test, batch_size=FLAGS.batch_size)

    print("After top_layer_model training (test set): {}".format(score))

    # Merge two models and create the final_model_final_final
    inputs = Input(shape=(48, 48, 3))
    vg_output = vgg16(inputs)
    print("vg_output: {}".format(vg_output.shape))
    # TODO: the 'pooling' argument of the VGG16 model is important for this to work otherwise you will have to  squash
    # output from (?, 1, 1, 512) to (?, 512)
    model_predictions = top_layer_model(vg_output)
    final_model = Model(input=inputs, output=model_predictions)
    final_model.compile(loss='categorical_crossentropy',
                        optimizer=adamax, metrics=['accuracy'])
    final_model_score = final_model.evaluate(x_train_input,
                                             y_train, batch_size=FLAGS.batch_size)
    print("Sanity check - final_model (train score): {}".format(final_model_score))

    final_model_score = final_model.evaluate(x_test_input,
                                             y_test, batch_size=FLAGS.batch_size)
    print("Sanity check - final_model (test score): {}".format(final_model_score))
    # config = final_model.get_config()
    # weights = final_model.get_weights()

    # probably don't need to create a new model
    # model_to_save = Model.from_config(config)
    # model_to_save.set_weights(weights)
    model_to_save = final_model

    print("Model input name: {}".format(model_to_save.input))
    print("Model output name: {}".format(model_to_save.output))

    # Save Model
    builder = saved_model_builder.SavedModelBuilder(FLAGS.export_path)
    signature = predict_signature_def(inputs={'images': model_to_save.input},
                                      outputs={'scores': model_to_save.output})
    with K.get_session() as sess:
        builder.add_meta_graph_and_variables(sess=sess,
                                             tags=[tag_constants.SERVING],
                                             signature_def_map={'predict': signature})
        builder.save()
Пример #36
0
def myTradingSystem(DATE, OPEN, HIGH, LOW, CLOSE, VOL, exposure, equity, settings):
    ''' This system uses trend following techniques to allocate capital into the desired equities'''

    try:
        print(settings['counter'])
    except:
        settings['counter'] = 0
        settings['LSTM_regressor'] = dict()
        settings['sc'] = dict()

    nMarkets = CLOSE.shape[1]
    pos = numpy.zeros(nMarkets)
    print(DATE[0])
    count_in_cash = 0
    for market in range(1,nMarkets):
        print("Processing market, index: {}".format(market))

        close = CLOSE[:, market]
        DATE_ = DATE
        data = pd.DataFrame()
        print(len(close) == len(DATE_))
        data['CLOSE'] = close
        data['observation_date'] = DATE_
        data['observation_date'] = pd.to_datetime(data['observation_date'].astype(str))
        data = data.merge(features, on=date, how="left")
        print(data.head())

        # retrain the lSTM model every 100 days
        if settings['counter']%100==0:
            training_set = numpy.reshape(CLOSE[:,market], (CLOSE[:,market].shape[0], 1))
            print("training_set", training_set.shape)
            training_set = (training_set-numpy.insert(training_set, 0, 0, axis=0)[:-1,])/numpy.insert(training_set, 0, 0, axis=0)[:-1,]
            training_set = training_set[1:,]
            print(training_set.shape)

            sc = MinMaxScaler(feature_range = (0, 1))
            training_set_scaled = sc.fit_transform(training_set)
            settings['sc'][str(market)] = sc 

            
            X_train = []
            y_train = []
            
            for i in range(30, training_set.shape[0]):
                X_train.append(training_set_scaled[i-30:i,0])
                y_train.append(training_set_scaled[i,0])
            
            X_train, y_train = numpy.array(X_train), numpy.array(y_train)
            print("len of X_train", len(X_train))
            X_train = numpy.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
            print(X_train.shape)
            

            # LSTM
            print("Re-training LSTM!")
            regressor = Sequential()

            regressor.add(LSTM(units = 25, return_sequences = True, input_shape = (X_train.shape[1], 1)))
            regressor.add(Dropout(0.1))

            regressor.add(LSTM(units = 20, return_sequences = True))
            regressor.add(Dropout(0.1))

            regressor.add(LSTM(units = 5))
            regressor.add(Dropout(0.1))

            regressor.add(Dense(units = 1))

            regressor.compile(optimizer = 'Adam', loss = 'mean_squared_error')

            regressor.fit(X_train, y_train, epochs = 1, batch_size = 32)
            
            settings['LSTM_regressor'][str(market)] = regressor
            
            print("Completed re-training!")
        else:
            print("Deploying existing LSTM!")
            sc = settings['sc'][str(market)]
            regressor = settings['LSTM_regressor'][str(market)]
        

        X_pred = []
        pred_set = numpy.reshape(CLOSE[:,market], (CLOSE[:,market].shape[0], 1))


        pred_set = (pred_set - numpy.insert(pred_set, 0, 0, axis=0)[:-1,])/numpy.insert(pred_set, 0, 0, axis=0)[:-1,]
        pred_set = pred_set[1:,]
        pred_set_scaled = sc.fit_transform(pred_set)
        X_pred.append(pred_set_scaled[-30:,0])
        X_pred = numpy.array(X_pred)
        X_pred = numpy.reshape(X_pred, (X_pred.shape[0], X_pred.shape[1], 1))
        predicted_stock_price = regressor.predict(X_pred)
        print(predicted_stock_price)
        predicted_stock_price = sc.inverse_transform(predicted_stock_price)

        err_term = 0.005 #%[0.005:0.001:0.008]#
        if predicted_stock_price[0][0] > 0:
            pos[market] = 1
            # print('LONG')


        elif predicted_stock_price[0][0] + err_term < 0:
            pos[market] = -1
            # print('SHORT') 

        else:
            pos[0] = pos[0] + 1 

        print('*' * 100)
    
    settings['counter'] = settings['counter'] + 1

    return pos, settings
Пример #37
0
def compile():
    single = Sequential()

    single.add(
        Conv2D(64, (7, 7),
               strides=(1, 1),
               padding='valid',
               activation='relu',
               kernel_regularizer=l1_l2(l1=0.01, l2=0.01),
               input_shape=(33, 33, 4)))

    single.add(BatchNormalization())
    single.add(Dropout(0.5))

    single.add(
        Conv2D(128, (5, 5),
               strides=(1, 1),
               padding='valid',
               kernel_regularizer=l1_l2(l1=0.01, l2=0.01),
               activation='relu'))
    single.add(BatchNormalization())
    single.add(Dropout(0.5))

    single.add(
        Conv2D(128, (5, 5),
               strides=(1, 1),
               padding='valid',
               kernel_regularizer=l1_l2(l1=0.01, l2=0.01),
               activation='relu'))
    single.add(BatchNormalization())
    single.add(Dropout(0.5))

    single.add(
        Conv2D(128, (3, 3),
               strides=(1, 1),
               padding='valid',
               kernel_regularizer=l1_l2(l1=0.01, l2=0.01),
               activation='relu'))
    single.add(BatchNormalization())
    single.add(Dropout(0.25))

    single.add(
        Conv2D(128, (3, 3),
               strides=(1, 1),
               padding='valid',
               kernel_regularizer=l1_l2(l1=0.01, l2=0.01),
               activation='relu'))
    single.add(Dropout(0.25))

    single.add(
        Conv2D(128, (3, 3),
               strides=(1, 1),
               padding='valid',
               kernel_regularizer=l1_l2(l1=0.01, l2=0.01),
               activation='relu'))
    single.add(BatchNormalization())
    single.add(Dropout(0.25))

    single.add(Flatten())
    single.add(Dense(5, activation='softmax'))

    sgd = SGD(lr=0.001, decay=0.01, momentum=0.9)
    single.compile(loss='categorical_crossentropy', optimizer='sgd')

    return single
Пример #38
0
test_accuracy = []
for i in range(0,50):
    model = Sequential()
    model.add(Conv2D(32, kernel_size=(3, 3),
                     activation='relu',
                     input_shape=input_shape))
    model.add(Conv2D(64, (3, 3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))
    model.add(Flatten())
    model.add(Dense(128, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(num_classes, activation='softmax'))

    model.compile(loss=keras.losses.categorical_crossentropy,#keras.losses.mean_squared_error,
                  optimizer=keras.optimizers.Adadelta(),
                  metrics=['accuracy'])

    model.fit(x_train, y_train,
              batch_size=batch_size,
              epochs=epochs,
              verbose=1,
              validation_data=(x_test, y_test))
    score = model.evaluate(x_test, y_test, verbose=0)
    print('Test loss:', score[0])
    print('Test accuracy:', score[1])
    test_loss.append(score[0])
    test_accuracy.append(score[1])
    
np.save('./test_loss_CNN_permute',test_loss)
np.save('./test_accuracy_CNN_permute',test_accuracy)
def create_model():
    model = Sequential()
    model.add(LSTM(units=4, input_shape=(1, look_back)))
    model.add(Dense(units=1))
    model.compile(loss='mean_squared_error', optimizer='adam')
    return model
model.add(Conv2D(1256, (3, 3), padding='same', activation='relu'))
model.add(Conv2D(1256, (1, 1), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='valid'))
#2x2x1256

model.add(AveragePooling2D(pool_size=(2, 2), strides=(1, 1), padding='valid'))
#1x1x1256

model.add(Flatten())

#model.add(Dense(4096, activation='relu', use_bias=True))
model.add(Dense(628, activation='relu', use_bias=True))
model.add(Dense(314, activation='relu', use_bias=True))
model.add(Dense(5, activation='softmax', use_bias=True))

print("Model created!!")
model.summary()
print("\nCompiling model...")
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

sgd = SGD(momentum=0.9, nesterov=True, lr=0.003)
#callbacks = [LearningRateScheduler(8, verbose=1)]
model.compile(optimizer='sgd',
              loss='categorical_crossentropy',
              metrics=['accuracy'])
print("Successfully compiled model!!")

model.save("WorkingModels/convnet224x224x3_untrained.h5")
print("Model saved!!")
Пример #41
0
plt.show()
print("a")

#学習モデルの作成
in_out_neurons = 1
hidden_neurons = 300

pat = 1
early_stopping = EarlyStopping(monitor='val_loss', patience=pat, verbose=1)
csv = CSVLogger('./Training_patience_' + str(pat) + '.csv')

model = Sequential()  
model.add(LSTM(hidden_neurons, batch_input_shape=(None, maxlen, in_out_neurons), return_sequences=False))  
model.add(Dense(in_out_neurons))  
model.add(Activation("linear"))  
model.compile(loss="mean_squared_error", optimizer="rmsprop")
fit = model.fit(X_train, Y_train, batch_size=512, epochs=10000, 
          validation_data=(X_validation, Y_validation), callbacks=[early_stopping, csv])

#予測
n_in = len(X[0][0])
original = x_lin[len(x_lin)-500:len(x_lin)-500+maxlen]
Z = original.reshape(1, 150, -1)
predicted = [None for i in range(maxlen)]

for i in range(length_of_sequences - maxlen + 1):
    z_ = Z[-1:]
    y_ = model.predict(z_)
    sequence_ = np.concatenate((z_.reshape(maxlen, n_in)[1:], y_),axis=0)
    sequence_ = sequence_.reshape(1, maxlen, n_in)
    Z = np.append(Z, sequence_, axis=0)
Пример #42
0
parser = argparse.ArgumentParser()
parser.add_argument("--datapath",help="path to the pickled data",default=save_path)
args = parser.parse_args()
data_path = args.datapath 

with open(data_path,"r") as f:
    (X,y) = pickle.load(f)
(X,y) = shuffle(X,y)
s=720
Xtr,ytr = X[:s],y[:s]
Xt,yt = X[s:],y[s:]

input_shape = X.shape[1:]

model = Sequential()
model.add(Conv2D(16,(7,7),strides=(3,3),activation="relu", input_shape=input_shape))
model.add(Dropout(0.1))
model.add(Conv2D(16,(3,3),strides=(1,1),activation="relu"))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(1,activation="sigmoid"))
model.compile(loss="binary_crossentropy",optimizer=Adam(),metrics=["accuracy"])



roc_curves,safeset_percents = k_fold_crossvalidation(X,y,10,model,epochs=14)
aucs = [area_under_curve(f,p) for (f,p) in roc_curves.values()]
print("Got average safeset % of {:.2f}. min :{:.2f}, max: {:.2f}, std: {:.2f}".format(np.mean(safeset_percents),min(safeset_percents),max(safeset_percents),np.std(safeset_percents)))
print("Got average ROC AUC of {:.2f}. min :{:.2f}, max: {:.2f}, std: {:.2f}".format(np.mean(aucs),min(aucs),max(aucs),np.std(aucs)))
plot_crossval_auc(roc_curves)
Пример #43
0
model.add(Dropout(0.25))
model.add(Conv2D(64, (3,3), activation='relu', padding='same'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.25))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes, activation='softmax'))

model.summary()
#훈련(얼리스타핑+체크포인트)
es = EarlyStopping(monitor='val_loss', patience=13, mode='auto')
modelpath = './miniprojectdata/checkpoint/cp-{epoch:02d}-{val_loss:.4f}.hdf5'
cp = ModelCheckpoint(filepath=modelpath, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])
hist = model.fit(x_train, y_train, epochs=35, batch_size=4, verbose=1, validation_split=0.2,callbacks=[es, cp])
#평가
loss, acc = model.evaluate(x_test, y_test, batch_size=2)
print("loss :", loss)
print("acc :", acc)
## pyplot 시각화
y_vloss = hist.history['val_loss']
y_loss = hist.history['loss']
y_vacc = hist.history['val_acc']
y_acc = hist.history['acc']
x_len1 = np.arange(len(y_loss))
x_len2 = np.arange(len(y_acc))
plt.figure(figsize=(6,6))
## 1 Loss 그래프
plt.subplot(2,1,1)
	model.add(Convolution2D(nb_filters*2, nb_conv, nb_conv))
	model.add(Activation('relu'))
	model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
	model.add(Dropout(0.25))

	c = 10
	Weight_Decay = c / float(X_train.shape[0])
	model.add(Flatten())
	model.add(Dense(128, W_regularizer=l2(Weight_Decay)))
	model.add(Activation('relu'))
	model.add(Dropout(0.5))
	model.add(Dense(nb_classes))
	model.add(Activation('softmax'))


	model.compile(loss='categorical_crossentropy', optimizer='adam')
	hist = model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=1, validation_data=(X_valid, Y_valid))
	Train_Result_Optimizer = hist.history
	Train_Loss = np.asarray(Train_Result_Optimizer.get('loss'))
	Train_Loss = np.array([Train_Loss]).T
	Valid_Loss = np.asarray(Train_Result_Optimizer.get('val_loss'))
	Valid_Loss = np.asarray([Valid_Loss]).T
	Train_Acc = np.asarray(Train_Result_Optimizer.get('acc'))
	Train_Acc = np.array([Train_Acc]).T
	Valid_Acc = np.asarray(Train_Result_Optimizer.get('val_acc'))
	Valid_Acc = np.asarray([Valid_Acc]).T


	Pool_Train_Loss = Train_Loss
	Pool_Valid_Loss = Valid_Loss
	Pool_Train_Acc = Train_Acc
Пример #45
0
model.add(Bidirectional(LSTM(lstm_num_hidden,return_sequences=True)))
#model.add(Bidirectional(GRU(lstm_num_hidden,return_sequences=True)))
#model.add(Bidirectional(GRU(lstm_num_hidden,return_sequences=True)))
model.add(AttentionLayer(hidden_layers=(75,)))
model.add(Dropout(0.5))
model.add(Dense(100,activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(2,activation='softmax'))
model.summary()

optimizer=SGD(learning_rate,decay=1e-6,momentum=0.9,nesterov=True,clipnorm=1.)
metrics=['accuracy','categorical_crossentropy']
loss='categorical_crossentropy'
checkpoint=ModelCheckpoint('bestmodels/cnn_dac_dbgru_da_te',monitor='val_acc',save_best_only=True,mode='max',verbose=1)

model.compile(optimizer=optimizer,loss=loss,metrics=metrics)

max_f1=0

for i in range(20):
	model.fit(trainX,trainY,epochs=1,batch_size=24)
	# model.save('epochmodels/cnn_dac_dbgru_da_te_'+str(i))
	#val_f1=F1(valY,model.predict(valX))
	#print 'Epoch',i,': Validation F1 score:',val_f1
	print 'Epoch',i
        print 'Validation'
        print_result(valY,model.predict(valX))
        print 'Test'
	print_result(testY,model.predict(testX))
	print ''
	continue
regressor.add(input_1)

regressor.add(LSTM(200, return_sequences = True))

regressor.add(LSTM(200, return_sequences = True))

regressor.add(LSTM(100, return_sequences = False))

# print(regressor.get_layer(index=0))

regressor.add(Dense(1))

learning_rate = optimizers.SGD(lr=0.001)

regressor.compile(optimizer = 'adam', loss = 'mae')

print()

regressor.fit(x_train, y_train, epochs = 200, batch_size = 52, verbose=2, shuffle=True)

# print(x_train[-52:])

yhat = regressor.predict([x_train[-52:], y_train[-52:]])

print(yhat)

test_set['Generated'] = yhat

test_set
Пример #47
0
#生成X和y矩阵
dataMat = np.array(data)

X = dataMat[:, 0:1]  # 变量x
y = dataMat[:, 1]  #变量y

print(dataMat.shape)
print(X.shape)
print(y.shape)

# 构建神经网络模型
model = Sequential()
model.add(Dense(input_dim=1, units=1))

# 选定loss函数和优化器
model.compile(loss='mse', optimizer='sgd', context=["gpu(0)"])
#model.compile(loss='mse', optimizer='sgd')
#如果是mxnet  指定 context=["gpu(0)"] 可以使用gpu
# 训练过程
print('Training -----------')
for step in range(1001):
    cost = model.train_on_batch(X, y)
    if step % 50 == 0:
        print("After %d trainings, the cost: %f" % (step, cost))

# 测试过程
print('\nTesting ------------')
cost = model.evaluate(X, y, batch_size=40)
print('test cost:', cost)
W, b = model.layers[0].get_weights()
print('Weights=', W, '\nbiases=', b)
Пример #48
0
    def build_model(self, last_activation, loss, lr, output_dim):
        '''
        used to initialize actor and critic models 
        generates different models based on the parameters passed
        '''

        kernel_initializer = keras.initializers.VarianceScaling(
            scale=1.0, mode='fan_in', distribution="normal", seed=None)
        bias_initializer = "zeros"

        model = Sequential()
        model.add(
            Conv2D(input_shape=self.state_dim,
                   data_format="channels_last",
                   filters=16,
                   kernel_size=(8, 8),
                   strides=(4, 4),
                   padding="same",
                   activation="relu",
                   kernel_initializer=kernel_initializer))

        model.add(
            Conv2D(data_format="channels_last",
                   filters=32,
                   kernel_size=(6, 6),
                   strides=(2, 2),
                   padding="same",
                   activation="relu",
                   kernel_initializer=kernel_initializer))

        model.add(
            Conv2D(data_format="channels_last",
                   filters=32,
                   kernel_size=(4, 4),
                   strides=(1, 1),
                   padding="same",
                   activation="relu",
                   kernel_initializer=kernel_initializer))

        model.add(
            Conv2D(data_format="channels_last",
                   filters=64,
                   kernel_size=(4, 4),
                   strides=(2, 2),
                   padding="same",
                   activation="relu",
                   kernel_initializer=kernel_initializer))

        model.add(
            Conv2D(data_format="channels_last",
                   filters=64,
                   kernel_size=(3, 3),
                   strides=(1, 1),
                   padding="same",
                   activation="relu",
                   kernel_initializer=kernel_initializer))

        model.add(Flatten(data_format="channels_last"))

        model.add(
            Dense(units=512,
                  activation="relu",
                  kernel_initializer=kernel_initializer,
                  bias_initializer=bias_initializer))

        model.add(Dense(output_dim, activation=last_activation))

        model.compile(optimizer=Adam(lr=lr), loss=loss)

        return model
Пример #49
0
class Model(BaseEstimator, RegressorMixin):
    """
    Model using Keras
    """
    def __init__(self, lr=0.1, epochs=300, tokens=None, n=0):
        """
        Init of the model layers

        :param lr: learning rate
        :type lr: float
        :param epochs: number of epochs
        :type epochs: int
        :param tokens: tokens used to train the RNN
        :type tokens: list of str
        :param n: input_length
        :type n: int
        """
        if tokens is None:
            tokens = ['\n', '&', 'C', '1', '2', '3', '4', 'O', 'N', '(', ')', '=', 'c', '[nH]', 'S', 'n', 's', 'o', '#', 'Cl', '[NH]']
        self.lr = lr
        self.epochs = epochs
        self.number = datetime.datetime.now()
        self.estimator = Sequential()
        self.estimator.add(Embedding(input_dim=len(tokens), output_dim=len(tokens), input_length=n, mask_zero=False))
        self.estimator.add(GRU(units=256, return_sequences=True, activation="tanh", input_shape=(81, 26)))
        self.estimator.add(Dropout(0.2))
        self.estimator.add(GRU(256, activation='tanh', return_sequences=True))
        self.estimator.add(Dropout(0.2))
        self.estimator.add(TimeDistributed(Dense(len(tokens), activation='softmax')))
        self.tensorboard = None
        self.early_stopping = None
        self.mcp_save = None

    def fit(self, x, y):
        """
        Compile and fit the model

        :param x: training data
        :type x: list of list of int
        :param y: target data
        :type y: list of list of int
        :return: None
        """
        optimizer = Adam(lr=self.lr)
        self.estimator.summary()

        save_directory = 'rnn_models/' + "test" + '/'
        log = save_directory + 'log'
        model_weights = save_directory + 'model_weights_' + str(self.number) + '.h5'
        # model_architecture = save_directory + 'model_architecture_' + str(number) + '.json'

        if not os.path.isdir(save_directory):
            os.mkdir(save_directory)
        if not os.path.isdir(log):
            os.mkdir(log)

        self.tensorboard = TensorBoard(
            log_dir=log + "/{}".format(str(datetime.datetime.now()) + '_model' + str(self.number)))
        self.early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=0, mode='min')
        self.mcp_save = ModelCheckpoint(model_weights, save_best_only=True, monitor='val_loss', mode='min')

        self.estimator.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
        self.estimator.fit(x, y, epochs=self.epochs, batch_size=512, validation_split=0.1,
                           callbacks=[self.tensorboard, self.early_stopping, self.mcp_save])

    def score(self, x, y, sample_weight=None):
        """
        Score the model

        :param x: training data
        :type x: list of list of int
        :param y: target data
        :type y: list of list of int
        :param sample_weight:
        :return: Score for the model
        """
        score = self.estimator.evaluate(x, y)
        print(score)
        return score[1]
# ANN

# Libraries
import keras
from keras.models import Sequential
from keras.layers import Dense

classifier=Sequential()

classifier.add(Dense(activation="relu", input_dim=5, units=3, kernel_initializer="uniform"))

classifier.add(Dense(activation="relu", units=3, kernel_initializer="uniform"))

classifier.add(Dense(activation="sigmoid", units=1, kernel_initializer="uniform"))

classifier.compile(optimizer='adam',loss='binary_crossentropy',metrics=['accuracy'])

classifier.fit(x=X,y=Y,batch_size=4,epochs=100)

y_pred=classifier.predict(X_test)

from pandas import DataFrame



for i in range(0,418):
   y_pred[i]=y_pred[i]>0.5
   
Z=pd.DataFrame(data=dataset_test.iloc[:,0].values,columns=['PassengerId'])

outputframe=pd.DataFrame(data=y_pred,columns=['Survived']) 
Пример #51
0
def fit_model(train_data, valid_data, vocab_size, train_portion, time_step = 30, batch_size = 30):
    train_data = train_data[0:int(train_data.shape[0]*train_portion)]
    train_x, train_y = train_data[0:-1], train_data[1:]
    
    trunc_step = time_step*batch_size
    
    trun_index = train_x.shape[0]//trunc_step*trunc_step
    train_x = train_x[0:trun_index]
    train_y = train_y[0:trun_index]
    
    #split subsequence and reshape
    train_x = np.reshape(train_x, (1, len(train_x)))
    train_y = np.reshape(train_y, (1, len(train_y)))
    
    train_x = np.split(train_x, batch_size, axis = 1)
    train_y = np.split(train_y, batch_size, axis = 1)
    
    for i in range(batch_size):
        train_x[i] = np.split(train_x[i], train_x[i].shape[1]//time_step, axis = 1)
        train_y[i] = np.split(train_y[i], train_y[i].shape[1]//time_step, axis = 1)
    
    train_x = np.concatenate(train_x, axis = 1)
    train_y = np.concatenate(train_y, axis = 1)
    
    train_x = train_x.reshape(train_x.shape[0]*train_x.shape[1], train_x.shape[2])
    #train_x = train_x.reshape(train_x.shape[0]*train_x.shape[1], train_x.shape[2], 1)
    #train_y = train_y.reshape(train_y.shape[0]*train_y.shape[1], train_y.shape[2], 1)
    train_y = train_y.reshape(train_y.shape[0]*train_y.shape[1], train_y.shape[2])
    train_y = np.array([train_y[i][len(train_y[i])-1] for i in range(train_y.shape[0])])
    train_y = train_y.reshape(len(train_y), 1)
    
    #validation data prepare
    valid_x, valid_y = valid_data[0:-1], valid_data[1:]
    trun_index = valid_x.shape[0]//trunc_step*trunc_step
    valid_x = valid_x[0:trun_index]
    valid_y = valid_y[0:trun_index]
    
    valid_x = np.reshape(valid_x, (1, len(valid_x)))
    valid_y = np.reshape(valid_y, (1, len(valid_y)))
    
    valid_x = np.split(valid_x, batch_size, axis = 1)
    valid_y = np.split(valid_y, batch_size, axis = 1)
    
    for i in range(batch_size):
        valid_x[i] = np.split(valid_x[i], valid_x[i].shape[1]//time_step, axis = 1)
        valid_y[i] = np.split(valid_y[i], valid_y[i].shape[1]//time_step, axis = 1)
    
    valid_x = np.concatenate(valid_x, axis = 1)
    valid_y = np.concatenate(valid_y, axis = 1)
    
    valid_x = valid_x.reshape(valid_x.shape[0]*valid_x.shape[1], valid_x.shape[2])
    #valid_x = valid_x.reshape(valid_x.shape[0]*valid_x.shape[1], valid_x.shape[2], 1)
    #valid_y = valid_y.reshape(valid_y.shape[0]*valid_y.shape[1], valid_y.shape[2], 1)
    valid_y = valid_y.reshape(valid_y.shape[0]*valid_y.shape[1], valid_y.shape[2])
    valid_y = np.array([valid_y[i][len(valid_y[i])-1] for i in range(valid_y.shape[0])])
    valid_y = valid_y.reshape(len(valid_y), 1)
    
    print(train_x.shape)
    print(valid_x.shape)
    print(train_y.shape)
    print(valid_y.shape)
    
    model = Sequential()
    model.add(Embedding(vocab_size, 120, batch_input_shape = (batch_size, train_x.shape[1])))
    model.add(CuDNNLSTM(256, stateful = True))
    #model.add(CuDNNLSTM(256, batch_input_shape = (batch_size, train_x.shape[1], 1), return_sequences = True, stateful = True))
    model.add(Dropout(0.2))
    model.add(Dense(vocab_size, activation = "softmax"))
    
    csv_logger = CSVLogger("ptb_words.csv", append=True, separator=',')
    checkpoint = ModelCheckpoint('ptb_words_best.h5', monitor = 'loss', verbose=1, save_best_only = True, mode='min')
    adam_opt = Adam(lr = 0.001)
    model.compile(loss='sparse_categorical_crossentropy',optimizer=adam_opt,metrics=['accuracy', perplexity])
    
    model.summary()
    
    for i in range(100):
        model.fit(train_x, train_y, validation_data=(valid_x, valid_y), batch_size=batch_size, epochs=1, verbose=1, shuffle=False, callbacks=[csv_logger, checkpoint])
        model.reset_states()
Пример #52
0
# 2. 모델
model = Sequential()
model.add(LSTM(64, input_shape=(4, 2), activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(64, activation='relu'))
model.add(Dense(1))

model.summary()

# 3. 컴파일(훈련준비),실행(훈련)
model.compile(optimizer='adam', loss='mse', metrics=['mse'])

hist = model.fit(x_train,
                 y_train,
                 epochs=30,
                 batch_size=2,
                 callbacks=[],
                 verbose=2,
                 validation_split=0.03)

plt.figure(figsize=(10, 6))  # -> 도화지의 크기? 출력되는 창의 크기인가 그래프의 크기인가

plt.subplot(2, 1, 1)  # 2행1열의 첫번쨰 그림을 그린다.
plt.title('keras80 loss plot')
plt.plot(hist.history['loss'], marker='.', c='red', label='loss')
plt.plot(hist.history['val_loss'], marker='.', c='blue', label='val_loss')
Пример #53
0
# Convert class vectors to binary class matrices
y_train = np_utils.to_categorical(y_train, nb_classes)
y_test = np_utils.to_categorical(y_test, nb_classes)

model = Sequential()
model.add(Dense(128, input_dim=784))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(10))
model.add(Activation('softmax'))

model.compile(optimizer="sgd", loss="categorical_crossentropy", metrics=["acc"])


def test_serialization():
    spark_model = SparkModel(model, frequency='epoch', mode='synchronous', num_workers=2)
    spark_model.save("test.h5")
    recov = load_spark_model("test.h5")


def test_spark_model_end_to_end(spark_context):
    rdd = to_simple_rdd(spark_context, x_train, y_train)

    # sync epoch
    spark_model = SparkModel(model, frequency='epoch', mode='synchronous', num_workers=2)
    spark_model.fit(rdd, epochs=epochs, batch_size=batch_size, verbose=2, validation_split=0.1)
    score = spark_model.master_network.evaluate(x_test, y_test, verbose=2)
Пример #54
0
def train_rnn(config, data, tokens, number):
    """
    Train one RNN, keep the best weights of the model and stop it when it doesnt learn anymore

    :param config: config to use
    :type config: dict
    :param data: data to use to train the RNN
    :type data: list of list of str
    :param tokens: list of tokens used to train the RNN
    :type tokens: list of str
    :param number: id of the model
    :type number: int
    :return: None
    """
    print("Model : " + str(number))
    x_train, y_train = convert_data_to_numbers(tokens, data)

    print("SMILES converted to numbers")

    maxlen = 81

    x = sequence.pad_sequences(x_train, maxlen=maxlen, dtype='int32',
                               padding='post', truncating='pre', value=0.)
    y = sequence.pad_sequences(y_train, maxlen=maxlen, dtype='int32',
                               padding='post', truncating='pre', value=0.)

    print("Loading y_train_one_hot")

    y_train_one_hot = np.array([to_categorical(y_i, num_classes=len(tokens)) for y_i in y])
    print(y_train_one_hot.shape)

    n = x.shape[1]

    model = Sequential()

    model.add(Embedding(input_dim=len(tokens), output_dim=len(tokens), input_length=n, mask_zero=False))
    model.add(GRU(units=256, return_sequences=True, activation="tanh", input_shape=(81, 26)))
    model.add(Dropout(0.2))
    model.add(GRU(256, activation='tanh', return_sequences=True))
    model.add(Dropout(0.2))
    model.add(TimeDistributed(Dense(len(tokens), activation='softmax')))
    optimizer = Adam(lr=config['learning_rate'])
    model.summary()

    save_directory = 'rnn_models/' + config['configuration_name'] + '/'
    log = save_directory + 'log'
    model_weights = save_directory + 'model_weights_' + str(number) + '.h5'
    model_architecture = save_directory + 'model_architecture_' + str(number) + '.json'

    if not os.path.isdir(save_directory):
        os.mkdir(save_directory)
    if not os.path.isdir(log):
        os.mkdir(log)

    tensorboard = TensorBoard(log_dir=log + "/{}".format(str(datetime.datetime.now()) + '_model' + str(number)))
    early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=0, mode='min')
    mcp_save = ModelCheckpoint(model_weights, save_best_only=True, monitor='val_loss', mode='min')

    model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])

    model.fit(x, y_train_one_hot, epochs=config['epochs'], batch_size=512,
              validation_split=0.1, callbacks=[tensorboard, early_stopping, mcp_save])

    model_json = model.to_json()
    with open(model_architecture, "w") as json_file:
        json_file.write(model_json)
    print("Model saved")

    with open(save_directory + "config.json", 'w') as conf:
        json.dump(config, conf)
print('x train shape', x_train.shape)
print('y train shape', y_train.shape)
#Small dataset for illustration
x_train_small = x_train[:1000]
y_train_small = y_train[:1000]

#   DEFINE THE NEURAL NETWORK WITH KERAS
#  hidden layers = 3 and output
model_1 = Sequential([
    Dense(units=128, input_dim=TOTAL_Inputs, activation='relu', name='hidden_1'),
    Dense(units=64, activation='relu', name='hidden_2'),    #no need to input dimensions, keras can get it anyway
    Dense(16, activation='relu', name='hidden_3'),  #16 = units(neurons), its more often in docs. so good practice
    Dense(10,activation='softmax', name='output') #output
    ])

model_1.compile(optimizer='adam',loss='sparse_categorical_crossentropy',metrics=['accuracy'])
display(model_1.summary())

model_2 = Sequential()   # adding dropout - its randomly taking out a neuron
model_2.add(Dropout(0.2, seed=42, input_shape=(TOTAL_Inputs,)))
model_2.add(Dense(128, activation='relu', name='Model_2_hidden_1'))
model_2.add(Dense(64, activation='relu', name='Model_2_hidden_2'))
model_2.add(Dense(16, activation='relu', name='Model_2_hidden_3'))
model_2.add(Dense(10, activation='softmax', name='Model_2_output'))
model_2.compile(optimizer='adam',loss='sparse_categorical_crossentropy',metrics=['accuracy'])

model_3 = Sequential()
model_3.add(Dropout(0.2, seed=42, input_shape=(TOTAL_Inputs,)))
model_3.add(Dense(128, activation='relu', name='Model_3_hidden_1'))
model_3.add(Dropout(0.25, seed=42))
model_3.add(Dense(64, activation='relu', name='Model_3_hidden_2'))
Пример #56
0
def get_model(data_type):
    # Model parameters and other things based upon the type of data
    op_dims = 4
    op_activ = 'softmax'
    # krnl_init = keras.initializers.RandomNormal(mean = 0.0, stddev = 0.05, seed = 11211)
    krnl_init = keras.initializers.GlorotUniform(seed = 11211)
    bias_init = keras.initializers.Zeros()
    if (data_type == 'encoded'):
        # NOT IN USE!!!!
        ip_dims = 6
        l1_units = 200
        l2_units = 200
        activ = 'relu'
        lrn_rt = 1.0e-06
    
        #https://stackoverflow.com/questions/59737875/keras-change-learning-rate
        opt = keras.optimizers.Adam(lr = lrn_rt)
    elif (data_type == 'sparse'):
        ip_dims = INPUT_SIZE
        l1_units = 400
        l2_units = 90
        activ = 'relu'
        lrn_rt = 1.0e-04
    
        # Adagrad
        #opt = keras.optimizers.Adagrad(
        #    learning_rate=lrn_rt, initial_accumulator_value=0.1, epsilon=1e-07,
        #    name='Adagrad')

        # Adadelta optimizer
        #opt = keras.optimizers.Adadelta(
        #    lr = lrn_rt, rho = 0.95, epsilon = 1e-07)

        # RMSprop optimizer
        #https://stackoverflow.com/questions/59737875/keras-change-learning-rate
        # opt = keras.optimizers.RMSprop(
        #    learning_rate = lrn_rt, rho = 0.9, momentum = 0.0, epsilon = 1e-07, 
        #    centered = False, name = 'RMSprop')
    

        # Adam w/wo amsgrad
        opt = keras.optimizers.Adam(lr = lrn_rt, amsgrad=True)


        # Nadam
        # opt = keras.optimizers.Nadam(
        #    lr=lrn_rt, beta_1=0.9, beta_2=0.999, epsilon=1e-07, name = 'nadam')
        
        #opt = keras.optimizers.SGD(
        #       learning_rate=lrn_rt, momentum=0.001, nesterov=True, name='SGD'
        #    )


        # Loss
        # loss = keras.losses.KLDivergence()

        loss = keras.losses.MeanSquaredError(reduction="auto", name="mean_squared_error") 

    else:
        import sys
        sys.exit(0)

    # creates a generic neural network architecture
    model = Sequential()
    
    model.add(Dense(units = l1_units,
                    input_dim = ip_dims, 
                    activation = activ,
                    kernel_initializer = krnl_init, 
                    bias_initializer = bias_init))
    model.add(Dense(units = l1_units,
                    input_dim = ip_dims, 
                    activation = activ,
                    kernel_initializer = krnl_init, 
                    bias_initializer = bias_init))
    model.add(Dense(units = l2_units, 
                    activation= activ,
                    kernel_initializer = krnl_init,  
                    bias_initializer = bias_init))
    model.add(Dense(units = l2_units, 
                    activation= activ,
                    kernel_initializer = krnl_init,  
                    bias_initializer = bias_init))
    
    # output layer
    model.add(Dense(units = op_dims, 
                    activation = op_activ,
                    kernel_initializer = krnl_init,  
                    bias_initializer = bias_init))
    
    # compile the model using traditional Machine Learning losses and optimizers
    model.compile(loss=loss, optimizer=opt, metrics=['accuracy'])

    return model
parser = argparse.ArgumentParser()
parser.add_argument("--model",
                    help="filename for output file",
                    default="six_bin.alt.leaky.sigmoid",
                    required=False)
args = parser.parse_args()

num_input_dimensions = 9600
num_neurons_in_layer1 = 1500
num_neurons_in_layer2 = 500
num_output_dimensions = 12
model = Sequential()

model.add(
    Dense(num_neurons_in_layer1,
          input_dim=num_input_dimensions,
          activation='relu'))
model.add(LeakyReLU(alpha=.01))
model.add(Dense(num_neurons_in_layer2, activation='relu'))
model.add(LeakyReLU(alpha=.01))
model.add(Dense(num_output_dimensions, activation='sigmoid'))

# 3) Compile Model

metrics_to_output = ['accuracy']
model.compile(loss='mean_absolute_error',
              optimizer='adam',
              metrics=metrics_to_output)
model.save(args.model + ".h5")
x_validation=x_validation.reshape(x_validation.shape[0],*im_shape)

#creating the model
cnn_model=Sequential([

    Conv2D(filters=32,kernel_size=3, activation="relu",input_shape=im_shape),
    MaxPooling2D(pool_size=(2,2)),
    Dropout(0.2),
    Flatten(),
    Dense(32,activation="relu"),
    Dense(10,activation="softmax")
])
#compiling
cnn_model.compile(
    loss="sparse_categorical_crossentropy",
    optimizer=adam(lr=0.001),
    metrics=["accuracy"]
)

# training the model
cnn_model.fit(

    x_train,y_train,batch_size=batch_size,epochs=3 , 
    validation_data=(x_validation,y_validation) 
)

score=cnn_model.evaluate(x_test,y_test)

print("test loss: {}".format(score[0]))
print("test accuracy: {}".format(score[1]))
Пример #59
0
def build_LSTM_model(trainData, trainBatches, testData, testBatches,
                     windowSize, class_count, numCalls, batch_size):
    # Specify number of units
    # https://stackoverflow.com/questions/37901047/what-is-num-units-in-tensorflow-basiclstmcell#39440218
    num_units = 128

    embedding_size = 256

    # https://keras.io/callbacks/#earlystopping
    early_stop = cb.EarlyStopping(monitor='sparse_categorical_accuracy',
                                  min_delta=0.0001,
                                  patience=3)

    model = Sequential()

    # We need to add an embedding layer because LSTM (at this moment) that the API call indices (numbers)
    # are of some mathematical significance. E.g., system call 2 is "closer" to system calls 3 and 4.
    # But system call numbers have nothing to do with their semantic meaning and relation to other
    # system calls. So we transform it using an embedding layer so the LSTM can figure these relationships
    # out for itself.
    # https://blog.keras.io/a-ten-minute-introduction-to-sequence-to-sequence-learning-in-keras.html

    # https://stackoverflow.com/questions/40695452/stateful-lstm-with-embedding-layer-shapes-dont-match
    api_count = numCalls + 1  # +1 because 0 is our padding number
    model.add(
        Embedding(input_dim=api_count, output_dim=256,
                  input_length=windowSize))

    # https://keras.io/layers/recurrent/#lstm
    #   model.add(LSTM(num_units,input_shape=(windowSize, api_count),return_sequences=False))
    #TODO - GPU stuffs
    model.add(
        CuDNNLSTM(num_units,
                  input_shape=(windowSize, api_count),
                  return_sequences=False))

    # NOTE:  If I want to add more layers
    # https://stackoverflow.com/questions/40331510/how-to-stack-multiple-lstm-in-keras

    # https://keras.io/layers/core/#dense
    model.add(Dense(128))
    # https://keras.io/activations/
    model.add(Activation('relu'))

    # https://keras.io/layers/core/#dropout
    model.add(Dropout(0.5))

    model.add(Dense(class_count, name='logits'))
    model.add(Activation('softmax'))

    # Which optimizer to use
    # https://keras.io/optimizers/
    opt = optimizers.RMSprop(lr=0.01, decay=0.001)

    # https://keras.io/models/model/#compile
    model.compile(
        loss='sparse_categorical_crossentropy',
        optimizer=opt,
        # Metrics to print
        # We use sparse_categorical_accuracy as opposed to categorical_accuracy
        # because: https://stackoverflow.com/questions/44477489/keras-difference-between-categorical-accuracy-and-sparse-categorical-accuracy
        # I.e., since we don't use hot-encoding, we use sparse_categorical_accuracy
        metrics=['sparse_categorical_accuracy'])

    # https://keras.io/models/model/#fit_generator
    hist = model.fit_generator(
        # Data to train
        trainData,
        # Use multiprocessing because python Threading isn't really
        # threading: https://docs.python.org/3/glossary.html#term-global-interpreter-lock
        use_multiprocessing=True,
        # Number of steps per epoch (this is how we train our large
        # number of samples dataset without running out of memory)
        steps_per_epoch=trainBatches,
        #TODO
        # Number of epochs
        epochs=100,
        # Validation data (will not be trained on)
        validation_data=testData,
        validation_steps=testBatches,
        # Do not shuffle batches.
        shuffle=False,
        # List of callbacks to be called while training.
        callbacks=[early_stop])

    return model, hist
Пример #60
0
model.add(Lambda(lambda x: x / 127.5 - 1.0, input_shape=INPUT_SHAPE))
model.add(Conv2D(24, 5, 5, activation='elu', subsample=(2, 2)))
model.add(Conv2D(36, 5, 5, activation='elu', subsample=(2, 2)))
model.add(Conv2D(48, 5, 5, activation='elu', subsample=(2, 2)))
model.add(Conv2D(64, 3, 3, activation='elu'))
model.add(Conv2D(64, 3, 3, activation='elu'))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(100, activation='elu'))
model.add(Dense(50, activation='elu'))
model.add(Dense(10, activation='elu'))
model.add(Dense(1))
model.summary()

checkpoint = ModelCheckpoint('model-{epoch:03d}.h5',
                             monitor='val_loss',
                             verbose=0,
                             save_best_only=True,
                             mode='auto')

model.compile(loss='mean_squared_error', optimizer=Adam(lr=1.0e-4))

model.fit_generator(batch_generator(X_train),
                    validation_data=batch_generator(X_validation),
                    nb_val_samples=len(X_validation),
                    samples_per_epoch=len(X_train),
                    nb_epoch=10,
                    verbose=1)

model.save('./model1.h5')