#---compile model (this step is missing in Max's code)
model.compile(loss='categorical_crossentropy', optimizer='adam')
model.summary()
#---Build RDD from trainin examples
#---(i.e. in training each worker will train on part of the data)
rdd = to_simple_rdd(sc, X_train, y_train)
#---Initialize SparkModel from Keras model and Spark context
#---there are two optimizers needed:
sgd = SGD(lr=0.1) #<---the master optimizer
adagrad = elephas_optimizers.Adagrad() #<---the elephas opimizer
spark_model = SparkModel(sc,
model,
optimizer=adagrad,
frequency='epoch',
mode='asynchronous',
num_workers=args.N_workers,
master_optimizer=sgd)
#---Train Spark model
spark_model.train(rdd,
nb_epoch=args.nb_epoch,
batch_size=args.batch_size,
verbose=1,
validation_split=0.25)
#---Evaluate Spark model by evaluating the underlying Keras master model
pred = spark_model.predict(X_test)
print np.shape(pred)
print np.shape(y_test)
model.add(Dropout(0.2))
model.add(Dense(10))
model.add(Activation('softmax'))
# Compile model
sgd = SGD(lr=0.1)
model.compile(loss='categorical_crossentropy', optimizer=sgd)
# Build RDD from numpy features and labels
rdd = to_simple_rdd(sc, x_train, y_train)
# Initialize SparkModel from Keras model and Spark context
print(model.to_yaml())
adagrad = elephas_optimizers.Adagrad()
spark_model = SparkModel(sc,
model,
optimizer=adagrad,
frequency='epoch',
mode='asynchronous',
num_workers=3)
# Train Spark model
try:
spark_model.train(rdd,
nb_epoch=nb_epoch,
batch_size=batch_size,
verbose=2,
validation_split=0.1)
# Evaluate Spark model by evaluating the underlying model
score = spark_model.master_network.evaluate(x_test,
y_test,
show_accuracy=True,
.map(lambda x: (np.array(x[0]).reshape(data_size,1), np.array(x[1])))
print("Num Partitions: ", train_rdd.getNumPartitions())
# ============ MODEL SETUP ===========
from keras.optimizers import SGD
wavenet_model = create_wavenet(stack_layers, n_output_channels, n_filter_list, num_stacks, skip=False)
adam_opt = keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False)
wavenet_model.compile(optimizer=SGD(), loss='categorical_crossentropy')
print(wavenet_model.summary())
# ============ ELEPHAS TRAIN ===========
spark_model = SparkModel(wavenet_model, mode='hogwild', num_workers=128)
spark_model.fit(train_rdd, epochs=64, batch_size=64, verbose=1, validation_split=0.1)
print("Finished Training :)")
# =========== SAVE FITTED MDOEL ===========
# Save model and weights out to local
filename_out = model_save_out + "_weights.h5"
spark_model.save(filename_out)
wavenet_json = wavenet_model.to_json()
with open(model_save_out + ".json", "w") as save_model:
save_model.write(wavenet_json)
x_test = np.array(train_rdd.map(lambda x: x[0]).take(1))
# Convert class vectors to binary class matrices
y_train = to_categorical(y_train, nb_classes)
y_test = to_categorical(y_test, nb_classes)
model = Sequential()
model.add(Dense(128, input_dim=784))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(10))
model.add(Activation('softmax'))
sgd = SGD(lr=0.1)
model.compile(sgd, 'categorical_crossentropy', ['acc'])
# Build RDD from numpy features and labels
rdd = to_simple_rdd(sc, x_train, y_train)
# Initialize SparkModel from tensorflow.keras model and Spark context
spark_model = SparkModel(model, mode='synchronous')
# Train Spark model
spark_model.fit(rdd, epochs=epochs, batch_size=batch_size, verbose=2, validation_split=0.1)
# Evaluate Spark model by evaluating the underlying model
score = spark_model.evaluate(x_test, y_test, verbose=2)
print('Test accuracy:', score[1])
# Convert class vectors to binary class matrices
y_train = np_utils.to_categorical(y_train, nb_classes)
y_test = np_utils.to_categorical(y_test, nb_classes)
#Model loading
json_model = open('mnist_test.json', 'r')
loaded_model_json = json_model.read()
json_model.close()
loaded_model = model_from_json(loaded_model_json)
loaded_model.load_weights('mnist_test.h5')
#Make sure model was loaded in correctly (should be 9.80% accuracy)
sgd = SGD(lr=0.1)
loaded_model.compile(sgd, 'categorical_crossentropy', ['acc'])
score = loaded_model.evaluate(x_test, y_test, verbose=0)
print("%s: %.2f%%" % (loaded_model.metrics_names[1], score[1] * 100))
# Create Spark context
conf = SparkConf().setAppName('Mnist_Spark_MLP').setMaster('local[8]')
sc = SparkContext(conf=conf)
# Build RDD from numpy features and labels
rdd = to_simple_rdd(sc, x_train, y_train)
# Initialize SparkModel from Keras model and Spark context
spark_model = SparkModel(loaded_model, frequency='epoch', mode='asynchronous')
# Evaluate Spark model by evaluating the underlying model
score = spark_model.master_network.evaluate(x_test, y_test, verbose=2)
print('Test accuracy:', score[1])
from keras.utils import np_utils
from elephas.spark_model import SparkModel
from elephas.utils.rdd_utils import to_simple_rdd
from elephas import optimizers as elephas_optimizers
model = Sequential()
model.add(LSTM(5, input_shape=(train_X.shape[1], train_X.shape[2])))
model.add(Dense(1))
model.compile(loss='mae', optimizer='adam')
adam = elephas_optimizers.Adam()
spark_model = SparkModel(sc,
model,
optimizer=adam,
frequency='epoch',
num_workers=2)
spark_model.train(rdd,
nb_epoch=50,
batch_size=4,
verbose=2,
validation_split=0.1)
# ## Multifeature Sentiment Analysis
df1 = pd.read_csv('/Users/rohitsuvarna/Downloads/features.csv', header=None)
df1.columns = [
'DateTime', 'Negative', 'Neutral', 'Positive', 'Compound', 'Price'
]
df1.head()
.getOrCreate())
sc = spark.sparkContext
"""### Training Model"""
from elephas.spark_model import SparkModel
from elephas.utils.rdd_utils import to_simple_rdd
# Compile the model.
model_14.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
# Build RDD from features and labels.
rdd = to_simple_rdd(sc, x_train, y_train)
# Initialize SparkModel from Keras model and Spark context.
spark_model = SparkModel(model_14,
frequency='epoch',
mode='asynchronous',
num_workers=3)
# Train the Spark model.
spark_model.fit(rdd, epochs=10, batch_size=32, verbose=1, validation_split=0.1)
score = spark_model.master_network.evaluate(x_test, y_test, verbose=1)
print('Test accuracy:', score)
"""### Predcit and evaluate Model"""
"""### Save Model"""
import json
#lets assume 'model' is main model
model_json = model_14.to_json()
with open("model_in_json.json", "w") as json_file:
json.dump(model_json, json_file)
#custom function to process iamage and get label
def load_data(path,sun397path=data_path):
target_class=path[3:-25]
target_class=output_dict[target_class]
target_class=to_categorical(target_class,397)
full_path=sun397path+path
img=image.load_img(full_path, target_size=(256, 256))
img_array = image.img_to_array(img)
img_array=img_array/255
return (img_array,target_class)
#data processing with spark
conf = SparkConf().setMaster('local[2]').setAppName('cnn_local')
sc = SparkContext(conf = conf)
sc.setLogLevel("WARN")
RDD_train=sc.textFile('./Training_01.txt')
RDD_test==sc.textFile('./Testing_01.txt')
train_data=RDD_train.map(lambda line:load_data(line) )
test_data=RDD_test.map(lambda line:load_data(line) )
#Model Training
model=get_model(32)
spark_model = SparkModel(model, frequency='epoch', mode='synchronous')
spark_model.fit(train_data, epochs=5, batch_size=32, verbose=1, validation_split=0.1)
#Model Evaluating
x_test=np.array(test_data.map(lambda tuple:tuple[0]).collect())
y_test=np.array(test_data.map(lambda tuple:tuple[1]).collect())
accuracy=spark_model._master_network.evaluate(x_test,y_test)[1]
print('Accuracy is {}'.format(accuracy))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adadelta')
## spark
conf = SparkConf().setAppName(APP_NAME).setMaster(MASTER_IP)
sc = SparkContext(conf=conf)
# Build RDD from numpy features and labels
rdd = to_simple_rdd(sc, X_train, Y_train)
# Initialize SparkModel from Keras model and Spark context
spark_model = SparkModel(sc, model)
# Train Spark model
spark_model.train(rdd,
nb_epoch=nb_epoch,
batch_size=batch_size,
verbose=0,
validation_split=0.1,
num_workers=24)
# Evaluate Spark model by evaluating the underlying model
score = spark_model.get_network().evaluate(X_test,
Y_test,
show_accuracy=True,
verbose=2)
print('Test accuracy:', score[1])
def test_serialization():
spark_model = SparkModel(model, frequency='epoch', mode='synchronous', num_workers=2)
spark_model.save("test.h5")
recov = load_spark_model("test.h5")
keras_loss = 'categorical_crossentropy'
keras_optimizer = 'sgd'
# Compile model
model.compile(loss=keras_loss, optimizer=keras_optimizer, metrics=["accuracy"])
# Build RDD from numpy features and labels
rdd = to_simple_rdd(sc, X_train, Y_train)
# Initialize SparkModel from Keras model and Spark context
print(model.to_yaml())
adagrad = elephas_optimizers.Adagrad(lr=0.01)
spark_model = SparkModel(sc,
model,
keras_losss=keras_loss,
keras_optimizer=keras_optimizer,
optimizer=adagrad,
frequency='epoch',
mode='asynchronous',
num_workers=4)
# Train Spark model
spark_model.train(rdd, nb_epoch=nb_epoch, batch_size=batch_size, verbose=2, validation_split=0.1)
# Evaluate Spark model by evaluating the underlying model
loss, acc = spark_model.master_network.evaluate(X_test, Y_test, verbose=2)
print("Test accuracy:", acc)
#checkpointer = ModelCheckpoint(filepath=MODEL_ROOT+MODEL+".h5", verbose=1, save_best_only=False)
#early_stopping = EarlyStopping(monitor='val_acc', patience=5)
#print 'Start training...'
#model.fit( X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=verbose, callbacks=[checkpointer],validation_split=validation_split, shuffle=shuffle,show_accuracy=show_accuracy)
# Create Spark Context
conf = SparkConf().setAppName(MODEL)
sc = SparkContext(conf=conf)
# Build RDD from numpy features and labels
rdd = to_simple_rdd(sc, X_train, Y_train)
# Initialize SparkModel from Keras model and Spark Context
rmsprop = elephas_optimizers.RMSprop()
spark_model = SparkModel(sc,\
model,\
optimizer=rmsprop,\
frequency='epoch',\
mode='asynchronous',\
num_workers=3)
spark_model.train(rdd,\
nb_epoch=nb_epoch,\
batch_size=batch_size,\
verbose=2,\
validation_split=validation_split)
spark_model.get_network().save_weights(MODEL_FILE_NAME)
