#---compile model (this step is missing in Max's code) model.compile(loss='categorical_crossentropy', optimizer='adam') model.summary() #---Build RDD from trainin examples #---(i.e. in training each worker will train on part of the data) rdd = to_simple_rdd(sc, X_train, y_train) #---Initialize SparkModel from Keras model and Spark context #---there are two optimizers needed: sgd = SGD(lr=0.1) #<---the master optimizer adagrad = elephas_optimizers.Adagrad() #<---the elephas opimizer spark_model = SparkModel(sc, model, optimizer=adagrad, frequency='epoch', mode='asynchronous', num_workers=args.N_workers, master_optimizer=sgd) #---Train Spark model spark_model.train(rdd, nb_epoch=args.nb_epoch, batch_size=args.batch_size, verbose=1, validation_split=0.25) #---Evaluate Spark model by evaluating the underlying Keras master model pred = spark_model.predict(X_test) print np.shape(pred) print np.shape(y_test)
model.add(Dropout(0.2)) model.add(Dense(10)) model.add(Activation('softmax')) # Compile model sgd = SGD(lr=0.1) model.compile(loss='categorical_crossentropy', optimizer=sgd) # Build RDD from numpy features and labels rdd = to_simple_rdd(sc, x_train, y_train) # Initialize SparkModel from Keras model and Spark context print(model.to_yaml()) adagrad = elephas_optimizers.Adagrad() spark_model = SparkModel(sc, model, optimizer=adagrad, frequency='epoch', mode='asynchronous', num_workers=3) # Train Spark model try: spark_model.train(rdd, nb_epoch=nb_epoch, batch_size=batch_size, verbose=2, validation_split=0.1) # Evaluate Spark model by evaluating the underlying model score = spark_model.master_network.evaluate(x_test, y_test, show_accuracy=True,
.map(lambda x: (np.array(x[0]).reshape(data_size,1), np.array(x[1]))) print("Num Partitions: ", train_rdd.getNumPartitions()) # ============ MODEL SETUP =========== from keras.optimizers import SGD wavenet_model = create_wavenet(stack_layers, n_output_channels, n_filter_list, num_stacks, skip=False) adam_opt = keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False) wavenet_model.compile(optimizer=SGD(), loss='categorical_crossentropy') print(wavenet_model.summary()) # ============ ELEPHAS TRAIN =========== spark_model = SparkModel(wavenet_model, mode='hogwild', num_workers=128) spark_model.fit(train_rdd, epochs=64, batch_size=64, verbose=1, validation_split=0.1) print("Finished Training :)") # =========== SAVE FITTED MDOEL =========== # Save model and weights out to local filename_out = model_save_out + "_weights.h5" spark_model.save(filename_out) wavenet_json = wavenet_model.to_json() with open(model_save_out + ".json", "w") as save_model: save_model.write(wavenet_json) x_test = np.array(train_rdd.map(lambda x: x[0]).take(1))
# Convert class vectors to binary class matrices y_train = to_categorical(y_train, nb_classes) y_test = to_categorical(y_test, nb_classes) model = Sequential() model.add(Dense(128, input_dim=784)) model.add(Activation('relu')) model.add(Dropout(0.2)) model.add(Dense(128)) model.add(Activation('relu')) model.add(Dropout(0.2)) model.add(Dense(10)) model.add(Activation('softmax')) sgd = SGD(lr=0.1) model.compile(sgd, 'categorical_crossentropy', ['acc']) # Build RDD from numpy features and labels rdd = to_simple_rdd(sc, x_train, y_train) # Initialize SparkModel from tensorflow.keras model and Spark context spark_model = SparkModel(model, mode='synchronous') # Train Spark model spark_model.fit(rdd, epochs=epochs, batch_size=batch_size, verbose=2, validation_split=0.1) # Evaluate Spark model by evaluating the underlying model score = spark_model.evaluate(x_test, y_test, verbose=2) print('Test accuracy:', score[1])
# Convert class vectors to binary class matrices y_train = np_utils.to_categorical(y_train, nb_classes) y_test = np_utils.to_categorical(y_test, nb_classes) #Model loading json_model = open('mnist_test.json', 'r') loaded_model_json = json_model.read() json_model.close() loaded_model = model_from_json(loaded_model_json) loaded_model.load_weights('mnist_test.h5') #Make sure model was loaded in correctly (should be 9.80% accuracy) sgd = SGD(lr=0.1) loaded_model.compile(sgd, 'categorical_crossentropy', ['acc']) score = loaded_model.evaluate(x_test, y_test, verbose=0) print("%s: %.2f%%" % (loaded_model.metrics_names[1], score[1] * 100)) # Create Spark context conf = SparkConf().setAppName('Mnist_Spark_MLP').setMaster('local[8]') sc = SparkContext(conf=conf) # Build RDD from numpy features and labels rdd = to_simple_rdd(sc, x_train, y_train) # Initialize SparkModel from Keras model and Spark context spark_model = SparkModel(loaded_model, frequency='epoch', mode='asynchronous') # Evaluate Spark model by evaluating the underlying model score = spark_model.master_network.evaluate(x_test, y_test, verbose=2) print('Test accuracy:', score[1])
from keras.utils import np_utils from elephas.spark_model import SparkModel from elephas.utils.rdd_utils import to_simple_rdd from elephas import optimizers as elephas_optimizers model = Sequential() model.add(LSTM(5, input_shape=(train_X.shape[1], train_X.shape[2]))) model.add(Dense(1)) model.compile(loss='mae', optimizer='adam') adam = elephas_optimizers.Adam() spark_model = SparkModel(sc, model, optimizer=adam, frequency='epoch', num_workers=2) spark_model.train(rdd, nb_epoch=50, batch_size=4, verbose=2, validation_split=0.1) # ## Multifeature Sentiment Analysis df1 = pd.read_csv('/Users/rohitsuvarna/Downloads/features.csv', header=None) df1.columns = [ 'DateTime', 'Negative', 'Neutral', 'Positive', 'Compound', 'Price' ] df1.head()
.getOrCreate()) sc = spark.sparkContext """### Training Model""" from elephas.spark_model import SparkModel from elephas.utils.rdd_utils import to_simple_rdd # Compile the model. model_14.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) # Build RDD from features and labels. rdd = to_simple_rdd(sc, x_train, y_train) # Initialize SparkModel from Keras model and Spark context. spark_model = SparkModel(model_14, frequency='epoch', mode='asynchronous', num_workers=3) # Train the Spark model. spark_model.fit(rdd, epochs=10, batch_size=32, verbose=1, validation_split=0.1) score = spark_model.master_network.evaluate(x_test, y_test, verbose=1) print('Test accuracy:', score) """### Predcit and evaluate Model""" """### Save Model""" import json #lets assume 'model' is main model model_json = model_14.to_json() with open("model_in_json.json", "w") as json_file: json.dump(model_json, json_file)
#custom function to process iamage and get label def load_data(path,sun397path=data_path): target_class=path[3:-25] target_class=output_dict[target_class] target_class=to_categorical(target_class,397) full_path=sun397path+path img=image.load_img(full_path, target_size=(256, 256)) img_array = image.img_to_array(img) img_array=img_array/255 return (img_array,target_class) #data processing with spark conf = SparkConf().setMaster('local[2]').setAppName('cnn_local') sc = SparkContext(conf = conf) sc.setLogLevel("WARN") RDD_train=sc.textFile('./Training_01.txt') RDD_test==sc.textFile('./Testing_01.txt') train_data=RDD_train.map(lambda line:load_data(line) ) test_data=RDD_test.map(lambda line:load_data(line) ) #Model Training model=get_model(32) spark_model = SparkModel(model, frequency='epoch', mode='synchronous') spark_model.fit(train_data, epochs=5, batch_size=32, verbose=1, validation_split=0.1) #Model Evaluating x_test=np.array(test_data.map(lambda tuple:tuple[0]).collect()) y_test=np.array(test_data.map(lambda tuple:tuple[1]).collect()) accuracy=spark_model._master_network.evaluate(x_test,y_test)[1] print('Accuracy is {}'.format(accuracy))
model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(nb_classes)) model.add(Activation('softmax')) model.compile(loss='categorical_crossentropy', optimizer='adadelta') ## spark conf = SparkConf().setAppName(APP_NAME).setMaster(MASTER_IP) sc = SparkContext(conf=conf) # Build RDD from numpy features and labels rdd = to_simple_rdd(sc, X_train, Y_train) # Initialize SparkModel from Keras model and Spark context spark_model = SparkModel(sc, model) # Train Spark model spark_model.train(rdd, nb_epoch=nb_epoch, batch_size=batch_size, verbose=0, validation_split=0.1, num_workers=24) # Evaluate Spark model by evaluating the underlying model score = spark_model.get_network().evaluate(X_test, Y_test, show_accuracy=True, verbose=2) print('Test accuracy:', score[1])
def test_serialization(): spark_model = SparkModel(model, frequency='epoch', mode='synchronous', num_workers=2) spark_model.save("test.h5") recov = load_spark_model("test.h5")
keras_loss = 'categorical_crossentropy' keras_optimizer = 'sgd' # Compile model model.compile(loss=keras_loss, optimizer=keras_optimizer, metrics=["accuracy"]) # Build RDD from numpy features and labels rdd = to_simple_rdd(sc, X_train, Y_train) # Initialize SparkModel from Keras model and Spark context print(model.to_yaml()) adagrad = elephas_optimizers.Adagrad(lr=0.01) spark_model = SparkModel(sc, model, keras_losss=keras_loss, keras_optimizer=keras_optimizer, optimizer=adagrad, frequency='epoch', mode='asynchronous', num_workers=4) # Train Spark model spark_model.train(rdd, nb_epoch=nb_epoch, batch_size=batch_size, verbose=2, validation_split=0.1) # Evaluate Spark model by evaluating the underlying model loss, acc = spark_model.master_network.evaluate(X_test, Y_test, verbose=2) print("Test accuracy:", acc)
#checkpointer = ModelCheckpoint(filepath=MODEL_ROOT+MODEL+".h5", verbose=1, save_best_only=False) #early_stopping = EarlyStopping(monitor='val_acc', patience=5) #print 'Start training...' #model.fit( X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=verbose, callbacks=[checkpointer],validation_split=validation_split, shuffle=shuffle,show_accuracy=show_accuracy) # Create Spark Context conf = SparkConf().setAppName(MODEL) sc = SparkContext(conf=conf) # Build RDD from numpy features and labels rdd = to_simple_rdd(sc, X_train, Y_train) # Initialize SparkModel from Keras model and Spark Context rmsprop = elephas_optimizers.RMSprop() spark_model = SparkModel(sc,\ model,\ optimizer=rmsprop,\ frequency='epoch',\ mode='asynchronous',\ num_workers=3) spark_model.train(rdd,\ nb_epoch=nb_epoch,\ batch_size=batch_size,\ verbose=2,\ validation_split=validation_split) spark_model.get_network().save_weights(MODEL_FILE_NAME)