#---compile model (this step is missing in Max's code)
model.compile(loss='categorical_crossentropy', optimizer='adam')
model.summary()

#---Build RDD from trainin examples
#---(i.e. in training each worker will train on part of the data)
rdd = to_simple_rdd(sc, X_train, y_train)

#---Initialize SparkModel from Keras model and Spark context
#---there are two optimizers needed:
sgd = SGD(lr=0.1)  #<---the master optimizer
adagrad = elephas_optimizers.Adagrad()  #<---the elephas opimizer
spark_model = SparkModel(sc,
                         model,
                         optimizer=adagrad,
                         frequency='epoch',
                         mode='asynchronous',
                         num_workers=args.N_workers,
                         master_optimizer=sgd)

#---Train Spark model
spark_model.train(rdd,
                  nb_epoch=args.nb_epoch,
                  batch_size=args.batch_size,
                  verbose=1,
                  validation_split=0.25)

#---Evaluate Spark model by evaluating the underlying Keras master model
pred = spark_model.predict(X_test)
print np.shape(pred)
print np.shape(y_test)
Esempio n. 2
0
    model.add(Dropout(0.2))
    model.add(Dense(10))
    model.add(Activation('softmax'))

    # Compile model
    sgd = SGD(lr=0.1)
    model.compile(loss='categorical_crossentropy', optimizer=sgd)
    # Build RDD from numpy features and labels
    rdd = to_simple_rdd(sc, x_train, y_train)

    # Initialize SparkModel from Keras model and Spark context
    print(model.to_yaml())
    adagrad = elephas_optimizers.Adagrad()
    spark_model = SparkModel(sc,
                             model,
                             optimizer=adagrad,
                             frequency='epoch',
                             mode='asynchronous',
                             num_workers=3)

    # Train Spark model
    try:
        spark_model.train(rdd,
                          nb_epoch=nb_epoch,
                          batch_size=batch_size,
                          verbose=2,
                          validation_split=0.1)

        # Evaluate Spark model by evaluating the underlying model
        score = spark_model.master_network.evaluate(x_test,
                                                    y_test,
                                                    show_accuracy=True,
Esempio n. 3
0
                .map(lambda x: (np.array(x[0]).reshape(data_size,1), np.array(x[1])))

print("Num Partitions: ", train_rdd.getNumPartitions())


# ============ MODEL SETUP ===========
from keras.optimizers import SGD

wavenet_model = create_wavenet(stack_layers, n_output_channels, n_filter_list, num_stacks, skip=False)
adam_opt = keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False)
wavenet_model.compile(optimizer=SGD(), loss='categorical_crossentropy')
print(wavenet_model.summary())


# ============ ELEPHAS TRAIN ===========
spark_model = SparkModel(wavenet_model, mode='hogwild', num_workers=128)
spark_model.fit(train_rdd, epochs=64, batch_size=64, verbose=1, validation_split=0.1)

print("Finished Training :)")

# =========== SAVE FITTED MDOEL ===========
# Save model and weights out to local
filename_out = model_save_out + "_weights.h5"
spark_model.save(filename_out)

wavenet_json = wavenet_model.to_json()
with open(model_save_out + ".json", "w") as save_model:
    save_model.write(wavenet_json)


x_test = np.array(train_rdd.map(lambda x: x[0]).take(1))
# Convert class vectors to binary class matrices
y_train = to_categorical(y_train, nb_classes)
y_test = to_categorical(y_test, nb_classes)

model = Sequential()
model.add(Dense(128, input_dim=784))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(10))
model.add(Activation('softmax'))

sgd = SGD(lr=0.1)
model.compile(sgd, 'categorical_crossentropy', ['acc'])

# Build RDD from numpy features and labels
rdd = to_simple_rdd(sc, x_train, y_train)

# Initialize SparkModel from tensorflow.keras model and Spark context
spark_model = SparkModel(model, mode='synchronous')

# Train Spark model
spark_model.fit(rdd, epochs=epochs, batch_size=batch_size, verbose=2, validation_split=0.1)

# Evaluate Spark model by evaluating the underlying model
score = spark_model.evaluate(x_test, y_test, verbose=2)
print('Test accuracy:', score[1])

# Convert class vectors to binary class matrices
y_train = np_utils.to_categorical(y_train, nb_classes)
y_test = np_utils.to_categorical(y_test, nb_classes)

#Model loading
json_model = open('mnist_test.json', 'r')
loaded_model_json = json_model.read()
json_model.close()
loaded_model = model_from_json(loaded_model_json)

loaded_model.load_weights('mnist_test.h5')

#Make sure model was loaded in correctly (should be 9.80% accuracy)
sgd = SGD(lr=0.1)
loaded_model.compile(sgd, 'categorical_crossentropy', ['acc'])
score = loaded_model.evaluate(x_test, y_test, verbose=0)
print("%s: %.2f%%" % (loaded_model.metrics_names[1], score[1] * 100))

# Create Spark context
conf = SparkConf().setAppName('Mnist_Spark_MLP').setMaster('local[8]')
sc = SparkContext(conf=conf)

# Build RDD from numpy features and labels
rdd = to_simple_rdd(sc, x_train, y_train)

# Initialize SparkModel from Keras model and Spark context
spark_model = SparkModel(loaded_model, frequency='epoch', mode='asynchronous')

# Evaluate Spark model by evaluating the underlying model
score = spark_model.master_network.evaluate(x_test, y_test, verbose=2)
print('Test accuracy:', score[1])
from keras.utils import np_utils
from elephas.spark_model import SparkModel
from elephas.utils.rdd_utils import to_simple_rdd
from elephas import optimizers as elephas_optimizers

model = Sequential()
model.add(LSTM(5, input_shape=(train_X.shape[1], train_X.shape[2])))
model.add(Dense(1))
model.compile(loss='mae', optimizer='adam')

adam = elephas_optimizers.Adam()

spark_model = SparkModel(sc,
                         model,
                         optimizer=adam,
                         frequency='epoch',
                         num_workers=2)
spark_model.train(rdd,
                  nb_epoch=50,
                  batch_size=4,
                  verbose=2,
                  validation_split=0.1)

# ## Multifeature Sentiment Analysis

df1 = pd.read_csv('/Users/rohitsuvarna/Downloads/features.csv', header=None)
df1.columns = [
    'DateTime', 'Negative', 'Neutral', 'Positive', 'Compound', 'Price'
]
df1.head()
Esempio n. 7
0
    .getOrCreate())

sc = spark.sparkContext
"""### Training Model"""

from elephas.spark_model import SparkModel
from elephas.utils.rdd_utils import to_simple_rdd
# Compile the model.
model_14.compile(optimizer='adam',
                 loss='binary_crossentropy',
                 metrics=['accuracy'])
# Build RDD from features and labels.
rdd = to_simple_rdd(sc, x_train, y_train)
# Initialize SparkModel from Keras model and Spark context.
spark_model = SparkModel(model_14,
                         frequency='epoch',
                         mode='asynchronous',
                         num_workers=3)
# Train the Spark model.
spark_model.fit(rdd, epochs=10, batch_size=32, verbose=1, validation_split=0.1)

score = spark_model.master_network.evaluate(x_test, y_test, verbose=1)
print('Test accuracy:', score)
"""### Predcit and evaluate Model"""
"""### Save Model"""

import json
#lets assume 'model' is main model
model_json = model_14.to_json()
with open("model_in_json.json", "w") as json_file:
    json.dump(model_json, json_file)
Esempio n. 8
0
#custom function to process iamage and get label
def load_data(path,sun397path=data_path):
    target_class=path[3:-25]
    target_class=output_dict[target_class]
    target_class=to_categorical(target_class,397)
    full_path=sun397path+path
    img=image.load_img(full_path, target_size=(256, 256))
    img_array = image.img_to_array(img)
    img_array=img_array/255
    return (img_array,target_class)

#data processing with spark
conf = SparkConf().setMaster('local[2]').setAppName('cnn_local')
sc = SparkContext(conf = conf)
sc.setLogLevel("WARN")
RDD_train=sc.textFile('./Training_01.txt')
RDD_test==sc.textFile('./Testing_01.txt')
train_data=RDD_train.map(lambda line:load_data(line) )
test_data=RDD_test.map(lambda line:load_data(line) )
#Model Training
model=get_model(32)
spark_model = SparkModel(model, frequency='epoch', mode='synchronous')
spark_model.fit(train_data, epochs=5, batch_size=32, verbose=1, validation_split=0.1)
#Model Evaluating
x_test=np.array(test_data.map(lambda tuple:tuple[0]).collect())
y_test=np.array(test_data.map(lambda tuple:tuple[1]).collect())
accuracy=spark_model._master_network.evaluate(x_test,y_test)[1]
print('Accuracy is {}'.format(accuracy))


model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))

model.compile(loss='categorical_crossentropy', optimizer='adadelta')

## spark
conf = SparkConf().setAppName(APP_NAME).setMaster(MASTER_IP)
sc = SparkContext(conf=conf)

# Build RDD from numpy features and labels
rdd = to_simple_rdd(sc, X_train, Y_train)

# Initialize SparkModel from Keras model and Spark context
spark_model = SparkModel(sc, model)

# Train Spark model
spark_model.train(rdd,
                  nb_epoch=nb_epoch,
                  batch_size=batch_size,
                  verbose=0,
                  validation_split=0.1,
                  num_workers=24)

# Evaluate Spark model by evaluating the underlying model
score = spark_model.get_network().evaluate(X_test,
                                           Y_test,
                                           show_accuracy=True,
                                           verbose=2)
print('Test accuracy:', score[1])
Esempio n. 10
0
def test_serialization():
    spark_model = SparkModel(model, frequency='epoch', mode='synchronous', num_workers=2)
    spark_model.save("test.h5")
    recov = load_spark_model("test.h5")
Esempio n. 11
0
keras_loss = 'categorical_crossentropy'
keras_optimizer = 'sgd'


# Compile model
model.compile(loss=keras_loss, optimizer=keras_optimizer, metrics=["accuracy"])

# Build RDD from numpy features and labels
rdd = to_simple_rdd(sc, X_train, Y_train)

# Initialize SparkModel from Keras model and Spark context
print(model.to_yaml())
adagrad = elephas_optimizers.Adagrad(lr=0.01)
spark_model = SparkModel(sc,
                         model,
                         keras_losss=keras_loss,
                         keras_optimizer=keras_optimizer,
                         optimizer=adagrad,
                         frequency='epoch',
                         mode='asynchronous',
                         num_workers=4)

# Train Spark model
spark_model.train(rdd, nb_epoch=nb_epoch, batch_size=batch_size, verbose=2, validation_split=0.1)

# Evaluate Spark model by evaluating the underlying model
loss, acc = spark_model.master_network.evaluate(X_test, Y_test, verbose=2)

print("Test accuracy:", acc)
Esempio n. 12
0
#checkpointer = ModelCheckpoint(filepath=MODEL_ROOT+MODEL+".h5", verbose=1, save_best_only=False)
#early_stopping = EarlyStopping(monitor='val_acc', patience=5)
#print 'Start training...'
#model.fit( X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=verbose, callbacks=[checkpointer],validation_split=validation_split, shuffle=shuffle,show_accuracy=show_accuracy)

# Create Spark Context
conf = SparkConf().setAppName(MODEL)
sc = SparkContext(conf=conf)

# Build RDD from numpy features and labels
rdd = to_simple_rdd(sc, X_train, Y_train)

# Initialize SparkModel from Keras model and Spark Context

rmsprop = elephas_optimizers.RMSprop()

spark_model = SparkModel(sc,\
                        model,\
                        optimizer=rmsprop,\
                        frequency='epoch',\
                        mode='asynchronous',\
                        num_workers=3)

spark_model.train(rdd,\
                    nb_epoch=nb_epoch,\
                    batch_size=batch_size,\
                    verbose=2,\
                    validation_split=validation_split)

spark_model.get_network().save_weights(MODEL_FILE_NAME)