def main():
gps_files = glob.glob('../data/prototype/**/gps_points.csv')
trip_files = glob.glob('../data/prototype/**/gps_trips.csv')
file_results = process_file(trip_file = trip_files[0], gps_file = gps_files[0])
seq_results = build_seq(input_df = file_results['df'], unique_trips = file_results['unique_trips'])
X = seq_results['x']
y = seq_results['y']
print('Bulding training data from files..')
for i in range(1, len(gps_files)):
file_results = process_file(trip_file = trip_files[i], gps_file = gps_files[i])
seq_results = build_seq(input_df = file_results['df'], unique_trips = file_results['unique_trips'])
X = np.vstack((X, seq_results['x']))
y = np.vstack((y, seq_results['y']))
x_train, x_val, y_train, y_val = train_test_split(X, y, random_state=1, train_size=0.8)
rdd = to_simple_rdd(sc, x_train, y_train)
model = build_model()
spark_model = SparkModel(model, frequency='epoch', mode='asynchronous')
spark_model.fit(rdd, epochs=5, batch_size=32, verbose=0, validation_split=0.1)
# model.fit(x_train, y_train, epochs=5, validation_data=(x_val, y_val))
y_pred = spark_model.predict(x_val)
acc = sum(np.argmax(y_pred, axis=1) == np.argmax(y_val, axis=1)) / y_pred.shape[0]
print("Validation Accuracy: {number:.{digits}f}%".format(number=(acc*100), digits=2))
def test_training_classification(spark_context, mode, parameter_server_mode,
mnist_data, classification_model):
# Define basic parameters
batch_size = 64
epochs = 10
# Load data
x_train, y_train, x_test, y_test = mnist_data
x_train = x_train[:1000]
y_train = y_train[:1000]
sgd = SGD(lr=0.1)
classification_model.compile(sgd, 'categorical_crossentropy', ['acc'])
# Build RDD from numpy features and labels
rdd = to_simple_rdd(spark_context, x_train, y_train)
# Initialize SparkModel from keras model and Spark context
spark_model = SparkModel(classification_model,
frequency='epoch',
mode=mode,
parameter_server_mode=parameter_server_mode,
port=4000 + random.randint(0, 500))
# Train Spark model
spark_model.fit(rdd,
epochs=epochs,
batch_size=batch_size,
verbose=0,
validation_split=0.1)
# run inference on trained spark model
predictions = spark_model.predict(x_test)
# run evaluation on trained spark model
evals = spark_model.evaluate(x_test, y_test)
# assert we can supply rdd and get same prediction results when supplying numpy array
test_rdd = spark_context.parallelize(x_test)
assert [np.argmax(x) for x in predictions
] == [np.argmax(x) for x in spark_model.predict(test_rdd)]
# assert we get the same prediction result with calling predict on keras model directly
assert [np.argmax(x) for x in predictions] == [
np.argmax(x) for x in spark_model.master_network.predict(x_test)
]
# assert we get the same evaluation results when calling evaluate on keras model directly
assert isclose(evals[0],
spark_model.master_network.evaluate(x_test, y_test)[0],
abs_tol=0.01)
assert isclose(evals[1],
spark_model.master_network.evaluate(x_test, y_test)[1],
abs_tol=0.01)
def test_sync_mode(spark_context):
# Define basic parameters
batch_size = 64
nb_classes = 10
epochs = 10
# Load data
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(60000, 784)
x_test = x_test.reshape(10000, 784)
x_train = x_train.astype("float32")
x_test = x_test.astype("float32")
x_train /= 255
x_test /= 255
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
# Convert class vectors to binary class matrices
y_train = np_utils.to_categorical(y_train, nb_classes)
y_test = np_utils.to_categorical(y_test, nb_classes)
model = Sequential()
model.add(Dense(128, input_dim=784))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(10))
model.add(Activation('softmax'))
sgd = SGD(lr=0.1)
model.compile(sgd, 'categorical_crossentropy', ['acc'])
# Build RDD from numpy features and labels
rdd = to_simple_rdd(spark_context, x_train, y_train)
# Initialize SparkModel from Keras model and Spark context
spark_model = SparkModel(model, mode='synchronous')
# Train Spark model
spark_model.fit(rdd,
epochs=epochs,
batch_size=batch_size,
verbose=2,
validation_split=0.1)
# Evaluate Spark model by evaluating the underlying model
score = spark_model.master_network.evaluate(x_test, y_test, verbose=2)
assert score[1] >= 0.70
def test_spark_model_end_to_end(spark_context):
rdd = to_simple_rdd(spark_context, x_train, y_train)
# sync epoch
spark_model = SparkModel(model, frequency='epoch', mode='synchronous', num_workers=2)
spark_model.fit(rdd, epochs=epochs, batch_size=batch_size, verbose=2, validation_split=0.1)
score = spark_model.master_network.evaluate(x_test, y_test, verbose=2)
print('Test accuracy:', score[1])
# sync batch
spark_model = SparkModel(model, frequency='batch', mode='synchronous', num_workers=2)
spark_model.fit(rdd, epochs=epochs, batch_size=batch_size, verbose=2, validation_split=0.1)
score = spark_model.master_network.evaluate(x_test, y_test, verbose=2)
print('Test accuracy:', score[1])
# async epoch
spark_model = SparkModel(model, frequency='epoch', mode='asynchronous')
spark_model.fit(rdd, epochs=epochs, batch_size=batch_size, verbose=2, validation_split=0.1)
score = spark_model.master_network.evaluate(x_test, y_test, verbose=2)
print('Test accuracy:', score[1])
# hogwild epoch
spark_model = SparkModel(model, frequency='epoch', mode='hogwild')
spark_model.fit(rdd, epochs=epochs, batch_size=batch_size, verbose=2, validation_split=0.1)
score = spark_model.master_network.evaluate(x_test, y_test, verbose=2)
print('Test accuracy:', score[1])
def test_training_regression(spark_context, mode, parameter_server_mode,
boston_housing_dataset, regression_model):
x_train, y_train, x_test, y_test = boston_housing_dataset
rdd = to_simple_rdd(spark_context, x_train, y_train)
# Define basic parameters
batch_size = 64
epochs = 10
sgd = SGD(lr=0.0000001)
regression_model.compile(sgd, 'mse', ['mae'])
spark_model = SparkModel(regression_model,
frequency='epoch',
mode=mode,
parameter_server_mode=parameter_server_mode,
port=4000 + random.randint(0, 500))
# Train Spark model
spark_model.fit(rdd,
epochs=epochs,
batch_size=batch_size,
verbose=0,
validation_split=0.1)
# run inference on trained spark model
predictions = spark_model.predict(x_test)
# run evaluation on trained spark model
evals = spark_model.evaluate(x_test, y_test)
# assert we can supply rdd and get same prediction results when supplying numpy array
test_rdd = spark_context.parallelize(x_test)
assert all(
np.isclose(x, y, 0.01)
for x, y in zip(predictions, spark_model.predict(test_rdd)))
# assert we get the same prediction result with calling predict on keras model directly
assert all(
np.isclose(x, y, 0.01) for x, y in zip(
predictions, spark_model.master_network.predict(x_test)))
# assert we get the same evaluation results when calling evaluate on keras model directly
assert isclose(evals[0],
spark_model.master_network.evaluate(x_test, y_test)[0],
abs_tol=0.01)
assert isclose(evals[1],
spark_model.master_network.evaluate(x_test, y_test)[1],
abs_tol=0.01)
def train_elephas_model(x, y):
model = models.Sequential()
# Input Layer
sgd = optimizers.Adam(lr=0.01)
model.add(Dense(256, activation="relu", input_shape=(x.shape[1],)))
model.add(Dropout(0.05))
model.add(Dense(256, activation="relu", input_shape=(x.shape[1],)))
model.add(Dropout(0.05))
# output layer
model.add(Dense(1))
model.compile(optimizer=sgd, loss="mse", metrics=["mse"])
model.summary()
rdd = to_simple_rdd(sc, x, y)
spark_model = SparkModel(model, frequency='epoch', mode='asynchronous')
# spark_model.fit(rdd, epochs=10, batch_size=64, verbose=1, validation_split=0.2)
spark_model.fit(rdd, epochs=25, batch_size=64, verbose=1, validation_split=0.2)
return spark_model
def dist_training(n_iter):
sbcnn = SBCNN_Model(field_size, bands, frames, num_channels, num_labels)
sgd = SGD(lr=0.001, momentum=0.0, decay=0.0, nesterov=False)
sbcnn.compile(loss='categorical_crossentropy',
metrics=['accuracy'],
optimizer=sgd)
train_arr, train_labels_arr, test_arr, test_labels_arr = get_data()
rdd = to_simple_rdd(sc, train_arr, train_labels_arr)
spark_model = SparkModel(sbcnn, frequency='epoch', mode='asynchronous')
spark_model.fit(rdd,
epochs=n_iter,
batch_size=32,
verbose=0,
validation_split=0.1)
score = spark_model.master_network.evaluate(test_arr,
test_labels_arr,
verbose=2)
print('Test accuracy:', score[1])
def test_training_custom_activation(mode, spark_context):
def custom_activation(x):
return sigmoid(x) + 1
model = Sequential()
model.add(Dense(1, input_dim=1, activation=custom_activation))
model.add(Dense(1, activation='sigmoid'))
sgd = SGD(lr=0.1)
model.compile(sgd, 'binary_crossentropy', ['acc'])
x_train = np.random.rand(1000)
y_train = np.zeros(1000)
x_test = np.random.rand(100)
y_test = np.zeros(100)
y_train[:500] = 1
rdd = to_simple_rdd(spark_context, x_train, y_train)
spark_model = SparkModel(model, frequency='epoch', mode=mode,
custom_objects={'custom_activation': custom_activation})
spark_model.fit(rdd, epochs=1, batch_size=16, verbose=0, validation_split=0.1)
assert spark_model.predict(x_test)
assert spark_model.evaluate(x_test, y_test)
train_rdd = to_simple_rdd(sc, train_input, decoder_train_target)
if args.ensemble:
model = DistributedEnsembleSeq2Seq(model_config)
else:
model = DistributedSeq2Seq(model_config)
spark_model = SparkModel(model.model,
frequency='epoch',
mode='synchronous',
batch_size=args.batch_size,
custom_objects={'EncoderSlice': EncoderSlice, 'DecoderSlice': DecoderSlice})
spark_model.fit(train_rdd,
batch_size=model_config.batch_size,
epochs=model_config.epochs,
validation_split=0.0,
verbose=1)
model.evaluate(encoder_test_input, raw_test_target)
else:
training_generator = WMTSequence(encoder_train_input, decoder_train_input, decoder_train_target, model_config)
validation_generator = WMTSequence(encoder_dev_input, decoder_dev_input, decoder_dev_target, model_config)
if args.ensemble:
model = EnsembleSeq2Seq(model_config)
else:
model = Seq2Seq(model_config)
if args.load_checkpoint:
class SparseGate(ModelFrame):
def __init__(self, x_train, y_train, x_test, y_test, inputs,
spark_context):
ModelFrame.__init__(self, x_train, y_train, x_test, y_test,
spark_context)
self.gateModel = None
self.inputs = inputs
def gating_network(self):
c1 = Conv2D(32, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay),
input_shape=self.x_train.shape[1:],
name='gate1')(self.inputs)
c2 = Activation('elu', name='gate2')(c1)
c3 = BatchNormalization(name='gate3')(c2)
c4 = Conv2D(32, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay),
name='gate4')(c3)
c5 = Activation('elu', name='gate5')(c4)
c6 = BatchNormalization(name='gate6')(c5)
c7 = MaxPooling2D(pool_size=(2, 2), name='gate7')(c6)
c8 = Dropout(0.2, name='gate26')(c7)
c9 = Conv2D(32 * 2, (3, 3),
name='gate8',
padding='same',
kernel_regularizer=regularizers.l2(weight_decay))(c8)
c10 = Activation('elu', name='gate9')(c9)
c11 = BatchNormalization(name='gate25')(c10)
c12 = Conv2D(32 * 2, (3, 3),
name='gate10',
padding='same',
kernel_regularizer=regularizers.l2(weight_decay))(c11)
c13 = Activation('elu', name='gate11')(c12)
c14 = BatchNormalization(name='gate12')(c13)
c15 = MaxPooling2D(pool_size=(2, 2), name='gate13')(c14)
c16 = Dropout(0.3, name='gate14')(c15)
c25 = Flatten(name='gate23')(c16)
c26 = Dense(5, name='gate24', activation='elu')(c25)
model = Model(inputs=self.inputs, outputs=c26)
return model
def create_gate_model(self, expert_models):
gate_network = self.gating_network()
merged = Lambda(lambda x: K.tf.transpose(
sum(
K.tf.transpose(x[i]) * x[0][:, i - 1] for i in range(
1, len(x)))))([gate_network.layers[-1].output] +
[m.layers[-1].output for m in expert_models])
b = Activation('softmax', name='gatex')(merged)
model = Model(inputs=self.inputs, outputs=b)
model.compile(loss='categorical_crossentropy',
optimizer=Adam(),
metrics=['accuracy'])
return model
def train_gate(self, datagen, weights_file):
model = self.gateModel
model.compile(loss='categorical_crossentropy',
optimizer=Adam(),
metrics=['accuracy'])
print(model.summary())
self.gateModel = SparkModel(model,
frequency='epoch',
mode='asynchronous')
score = self.gateModel.master_network.evaluate(self.x_test,
self.y_test,
verbose=2,
batch_size=50)
self.gateModel.fit(self.rdd, epochs=1, batch_size=50, verbose=1)
self.gateModel = self.gateModel.master_network
self.gateModel.save_weights(weights_file + '.hdf5')
file = '../lib/output.txt'
if os.path.exists(file):
append_write = 'a'
else:
append_write = 'w'
#score = self.gateModel.evaluate(self.x_test, self.y_test, verbose=2, batch_size=50)
print("------------------------------")
print("Score is:" + str(score[1]))
print("-------------------------------")
text_file = open(file, append_write)
text_file.write("Score: %s" % score[1])
text_file.close()
def load_gate_weights(self,
model_old,
weights_file='../lib/weights/moe_full.hdf5'):
model_old.load_weights(weights_file)
for l in self.gateModel.layers:
for b in model_old.layers:
if (l.name == b.name):
l.set_weights(b.get_weights())
print("loaded gate layer " + str(l.name))
model.add(Dense(1))
metrics = ['MeanSquaredError', 'MeanAbsoluteError']
model.compile(loss='mean_squared_error', optimizer='adam', metrics=metrics)
print(model.summary())
rdd = train_data.rdd.map(lambda x:
(x[0].toArray().reshape(1, len(x[0])), x[1]))
spark_model = SparkModel(model,
frequency='epoch',
mode='synchronous',
metrics=metrics)
start = time()
spark_model.fit(rdd,
epochs=1,
batch_size=64,
verbose=0,
validation_split=0.1)
fit_dt = time() - start
print(f"Fit took: {fit_dt}")
x_test = test_data.toPandas()['features']
x_test = np.asarray(test_data.rdd.map(lambda x: x[0].toArray()).collect())
x_test = x_test.reshape((x_test.shape[0], 1, x_test.shape[1]))
y_test = test_data.toPandas()["Weighted_Price"].to_numpy()
y_test = y_test.reshape((len(y_test), 1, 1))
print(f"X shape: {x_test.shape}, Y shape: {y_test.shape}")
score = spark_model.master_network.evaluate(x_test, y_test, verbose=2)
print(f"Test score: {score}")
print("Num Partitions: ", train_rdd.getNumPartitions())
# ============ MODEL SETUP ===========
from keras.optimizers import SGD
wavenet_model = create_wavenet(stack_layers, n_output_channels, n_filter_list, num_stacks, skip=False)
adam_opt = keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False)
wavenet_model.compile(optimizer=SGD(), loss='categorical_crossentropy')
print(wavenet_model.summary())
# ============ ELEPHAS TRAIN ===========
spark_model = SparkModel(wavenet_model, mode='hogwild', num_workers=128)
spark_model.fit(train_rdd, epochs=64, batch_size=64, verbose=1, validation_split=0.1)
print("Finished Training :)")
# =========== SAVE FITTED MDOEL ===========
# Save model and weights out to local
filename_out = model_save_out + "_weights.h5"
spark_model.save(filename_out)
wavenet_json = wavenet_model.to_json()
with open(model_save_out + ".json", "w") as save_model:
save_model.write(wavenet_json)
x_test = np.array(train_rdd.map(lambda x: x[0]).take(1))
y_test = np.array(train_rdd.map(lambda x: x[1]).take(1))
class HAN(object):
"""
HAN model is implemented here.
"""
def __init__(self,
text,
labels,
pretrained_embedded_vector_path,
max_features,
max_senten_len,
max_senten_num,
embedding_size,
num_categories=None,
validation_split=0.2,
verbose=0):
"""Initialize the HAN module
Keyword arguments:
text -- list of the articles for training.
labels -- labels corresponding the given `text`.
pretrained_embedded_vector_path -- path of any pretrained vector
max_features -- max features embeddeding matrix can have. To more checkout https://keras.io/layers/embeddings/
max_senten_len -- maximum sentence length. It is recommended not to use the maximum one but the one that covers 0.95 quatile of the data.
max_senten_num -- maximum number of sentences. It is recommended not to use the maximum one but the one that covers 0.95 quatile of the data.
embedding_size -- size of the embedding vector
num_categories -- total number of categories.
validation_split -- train-test split.
verbose -- how much you want to see.
"""
try:
self.verbose = verbose
self.max_features = max_features
self.max_senten_len = max_senten_len
self.max_senten_num = max_senten_num
self.embed_size = embedding_size
self.validation_split = validation_split
self.embedded_dir = pretrained_embedded_vector_path
self.text = pd.Series(text)
self.categories = pd.Series(labels)
self.classes = self.categories.unique().tolist()
# Initialize default hyperparameters
# You can change it using `set_hyperparameters` function
self.hyperparameters = {
'l2_regulizer': None,
'dropout_regulizer': None,
'rnn': LSTM,
'rnn_units': 150,
'dense_units': 200,
'activation': 'softmax',
'optimizer': 'adam',
'metrics': ['acc'],
'loss': 'categorical_crossentropy'
}
if num_categories is not None:
assert (num_categories == len(self.classes))
assert (self.text.shape[0] == self.categories.shape[0])
self.data, self.labels = self.preprocessing()
self.x_train, self.y_train, self.x_val, self.y_val = self.split_dataset(
)
self.embedding_index = self.add_glove_model()
self.set_model()
except AssertionError:
print('Input and label data must be of same size')
# Implement this after you have seen all the different kinds of errors
# try:
# conf = SparkConf().setAppName('HANMusicClassifier').setMaster('')
# self.sc = SparkContext(conf=conf)
# except Error:
conf = SparkConf().setAppName('HANMusicClassifier')
self.sc = SparkContext(conf=conf)
def set_hyperparameters(self, tweaked_instances):
"""Set hyperparameters of HAN model.
Keywords arguemnts:
tweaked_instances -- dictionary of all those keys you want to change
"""
for key, value in tweaked_instances.items():
if key in self.hyperparameters:
self.hyperparameters[key] = value
else:
raise KeyError(key + ' does not exist in hyperparameters')
self.set_model()
def show_hyperparameters(self):
"""To check the values of all the current hyperparameters
"""
print('Hyperparameter\tCorresponding Value')
for key, value in self.hyperparameters.items():
print(key, '\t\t', value)
def clean_string(self, string):
"""
Tokenization/string cleaning for dataset
Every dataset is lower cased except
"""
string = re.sub(r"\\", "", string)
string = re.sub(r"\'", "", string)
string = re.sub(r"\"", "", string)
return string.strip().lower()
def add_dataset(self, text, labels):
try:
self.text = pd.concat([self.text, pd.Series(text)])
self.categories = pd.concat([self.categories, pd.Series(labels)])
assert (len(self.classes) == self.categories.unique().tolist())
except AssertionError:
print("New class cannot be added in this manner")
def preprocessing(self):
"""Preprocessing of the text to make it more resonant for training
"""
paras = []
labels = []
texts = []
for idx in range(self.text.shape[0]):
text = self.clean_string(self.text[idx])
texts.append(text)
sentences = tokenize.sent_tokenize(text)
paras.append(sentences)
tokenizer = Tokenizer(num_words=self.max_features, oov_token=True)
tokenizer.fit_on_texts(texts)
data = np.zeros((len(texts), self.max_senten_num, self.max_senten_len),
dtype='int32')
for i, sentences in enumerate(paras):
for j, sent in enumerate(sentences):
if j < self.max_senten_num:
wordTokens = text_to_word_sequence(sent)
k = 0
for _, word in enumerate(wordTokens):
if k < self.max_senten_len and word in tokenizer.word_index and tokenizer.word_index[
word] < self.max_features:
data[i, j, k] = tokenizer.word_index[word]
k = k + 1
self.word_index = tokenizer.word_index
if self.verbose == 1:
print('Total %s unique tokens.' % len(self.word_index))
labels = pd.get_dummies(self.categories)
if self.verbose == 1:
print('Shape of data tensor:', data.shape)
print('Shape of labels tensor:', labels.shape)
assert (len(self.classes) == labels.shape[1])
assert (data.shape[0] == labels.shape[0])
return data, labels
def split_dataset(self):
indices = np.arange(self.data.shape[0])
np.random.shuffle(indices)
self.data = self.data[indices]
self.labels = self.labels.iloc[indices]
nb_validation_samples = int(self.validation_split * self.data.shape[0])
x_train = self.data[:-nb_validation_samples]
y_train = self.labels[:-nb_validation_samples]
x_val = self.data[-nb_validation_samples:]
y_val = self.labels[-nb_validation_samples:]
if self.verbose == 1:
print(
'Number of positive and negative reviews in traing and validation set'
)
print(y_train.columns.tolist())
print(y_train.sum(axis=0).tolist())
print(y_val.sum(axis=0).tolist())
return x_train, y_train, x_val, y_val
def get_model(self):
"""
Returns the HAN model so that it can be used as a part of pipeline
"""
return self.model
def add_glove_model(self):
"""
Read and save Pretrained Embedding model
"""
embeddings_index = {}
try:
f = open(self.embedded_dir)
for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
assert (coefs.shape[0] == self.embed_size)
embeddings_index[word] = coefs
f.close()
except OSError:
print('Embedded file does not found')
exit()
except AssertionError:
print(
"Embedding vector size does not match with given embedded size"
)
return embeddings_index
def get_embedding_matrix(self):
"""
Returns Embedding matrix
"""
embedding_matrix = np.random.random(
(len(self.word_index) + 1, self.embed_size))
absent_words = 0
for word, i in self.word_index.items():
embedding_vector = self.embedding_index.get(word)
if embedding_vector is not None:
# words not found in embedding index will be all-zeros.
embedding_matrix[i] = embedding_vector
else:
absent_words += 1
if self.verbose == 1:
print('Total absent words are', absent_words, 'which is',
"%0.2f" % (absent_words * 100 / len(self.word_index)),
'% of total words')
return embedding_matrix
def get_embedding_layer(self):
"""
Returns Embedding layer
"""
embedding_matrix = self.get_embedding_matrix()
return Embedding(len(self.word_index) + 1,
self.embed_size,
weights=[embedding_matrix],
input_length=self.max_senten_len,
trainable=False)
def set_model(self):
"""
Set the HAN model according to the given hyperparameters
"""
if self.hyperparameters['l2_regulizer'] is None:
kernel_regularizer = None
else:
kernel_regularizer = regularizers.l2(
self.hyperparameters['l2_regulizer'])
if self.hyperparameters['dropout_regulizer'] is None:
dropout_regularizer = 1
else:
dropout_regularizer = self.hyperparameters['dropout_regulizer']
word_input = Input(shape=(self.max_senten_len, ), dtype='float32')
word_sequences = self.get_embedding_layer()(word_input)
word_lstm = Bidirectional(self.hyperparameters['rnn'](
self.hyperparameters['rnn_units'],
return_sequences=True,
kernel_regularizer=kernel_regularizer))(word_sequences)
word_dense = TimeDistributed(
Dense(self.hyperparameters['dense_units'],
kernel_regularizer=kernel_regularizer))(word_lstm)
word_att = AttentionWithContext()(word_dense)
wordEncoder = Model(word_input, word_att)
sent_input = Input(shape=(self.max_senten_num, self.max_senten_len),
dtype='float32')
sent_encoder = TimeDistributed(wordEncoder)(sent_input)
sent_lstm = Bidirectional(self.hyperparameters['rnn'](
self.hyperparameters['rnn_units'],
return_sequences=True,
kernel_regularizer=kernel_regularizer))(sent_encoder)
sent_dense = TimeDistributed(
Dense(self.hyperparameters['dense_units'],
kernel_regularizer=kernel_regularizer))(sent_lstm)
sent_att = Dropout(dropout_regularizer)(
AttentionWithContext()(sent_dense))
preds = Dense(len(self.classes))(sent_att)
self.model = Model(sent_input, preds)
self.model.compile(loss=self.hyperparameters['loss'],
optimizer=self.hyperparameters['optimizer'],
metrics=self.hyperparameters['metrics'])
self.spark_model = SparkModel(self.model,
frequency='epoch',
mode='asynchronous')
# Currently cannot plot learning curve
def train_model(self,
rdd,
epochs,
batch_size,
verbose=1,
validation_split=0.1):
"""Training the model
rdd -- The actual data
epochs -- Total number of epochs
batch_size -- size of a batch
verbose -- Whether or not we want verbose feedback
validation_split -- What percentage of the data from the rdd is actually used as a validation set
"""
self.spark_model.fit(self,
rdd,
epochs=epochs,
batch_size=batch_size,
verbose=verbose,
validation_split=validation_split)
def predict(self, rdd):
self.spark_model.predict(rdd)
def plot_results(self):
"""
Plotting learning curve of last trained model.
"""
# summarize history for accuracy
plt.subplot(211)
plt.plot(self.history.history['acc'])
plt.plot(self.history.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
# summarize history for loss
plt.subplot(212)
plt.plot(self.history.history['val_loss'])
plt.plot(self.history.history['loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
time.sleep(10)
plt.close()
from elephas.spark_model import SparkModel
from elephas.utils.rdd_utils import to_simple_rdd
# Compile the model.
model_9.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
# Build RDD from features and labels.
rdd = to_simple_rdd(sc, x_train, y_train)
# Initialize SparkModel from Keras model and Spark context.
spark_model = SparkModel(model_9,
frequency='epoch',
mode='asynchronous',
num_workers=3)
# Train the Spark model.
spark_model.fit(rdd, epochs=10, batch_size=32, verbose=1, validation_split=0.1)
score = spark_model.master_network.evaluate(x_test, y_test, verbose=1)
print('Test accuracy:', score)
"""### Predcit and evaluate Model"""
"""### Save Model"""
import json
#lets assume 'model' is main model
model_json = model_9.to_json()
with open("model_in_json.json", "w") as json_file:
json.dump(model_json, json_file)
spark_model.save('resnet.h5')
# Load saved Model and using keras to predict
# print('Test accuracy:', score[1])
# Create Spark context
conf = SparkConf().setAppName('Mnist_Spark_MLP')
# .setMaster('local[8]')
sc = SparkContext(conf=conf)
# Build RDD from numpy features and labels
# lp_rdd = to_labeled_point(sc, x_train, y_train, categorical=True)
rdd = to_simple_rdd(sc, x_train, y_train)
# Train Spark model
spark_model = SparkModel(model, frequency='epoch', mode='asynchronous')
spark_model.fit(rdd,
epochs=epochs,
batch_size=batch_size,
verbose=2,
validation_split=0.1)
# Evaluate Spark model by evaluating the underlying model
score = spark_model.master_network.evaluate(x_test, y_test, verbose=2)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
model_file = 'save/mlp.h5'
import os
if not os.path.exists("save/"):
os.mkdir("save/")
model.save(model_file)
labels = []
features = []
for message in consumer:
#print(message.value)
labels.append(message.value["label"])
features.append(message.value["features"]["values"])
labeledpoints = np.array(labels, features)
model = Sequential()
model.add(Dense(2, input_dim=11))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(10))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer=SGD())
lp_rdd = to_simple_rdd(sc, features, labels, categorical=True)
spark_model = SparkModel(model, frequency='epoch', mode='asynchronous')
spark_model.fit(lp_rdd,
epochs=20,
batch_size=32,
verbose=0,
validation_split=0.1)
spark_model.save("model.h5")
x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1))
x_train.shape
model = Sequential()
model.add(LSTM(205, return_sequences=False, input_shape=(x_train.shape[1], 1)))
model.add(Dense(1, activation='relu'))
model.compile(optimizer='adam', loss='mean_squared_error')
LOGGER.info('Model compiled')
rdd = to_simple_rdd(sc, x_train, y_train)
spark_model = SparkModel(model,
frequency='batch',
mode='hogwild',
num_workers=2)
LOGGER.info('Spark model created')
spark_model.fit(rdd, epochs=50, batch_size=500, validation_split=0.01)
LOGGER.info('Spark model trained')
LOGGER.info(model.summary())
test_data = scaled_data[training_data_len - 30:, :]
x_test = []
y_test = dataset[training_data_len:, :]
y_test = dataset[training_data_len:, :]
for i in range(30, len(test_data)):
x_test.append(test_data[i - 30:i, 0])
x_test = np.array(x_test)
x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1))
predictions = spark_model.predict(x_test)
train_rdd = train_rdd.map(lambda x: (x[0], x[1]))
test_rdd = test_rdd.map(lambda x: (x[0], x[1]))
def convert_labels(labels):
allowed = np.arange(10)
one_hot = np.zeros(10)
for l in labels:
if l in allowed:
one_hot[l] = 1
return one_hot
train_rdd = train_rdd.map(lambda x: (np.array(x[0]), convert_labels(x[1])))
test_rdd = test_rdd.map(lambda x: (np.array(x[0]), convert_labels(x[1])))
# =========
# TRAINING
# =========
keras_model = create_model()
spark_model = SparkModel(keras_model, frequency='epoch', mode='asynchronous')
history = spark_model.fit(train_rdd, epochs=10, batch_size=32, verbose=2)
# =========
# TESTING
# =========
x_test = np.array(test_rdd.map(lambda x: x[0]).collect())
y_test = np.array(test_rdd.map(lambda x: x[1]).collect())
score = spark_model.master_network.evaluate(x_test, y_test, verbose=2)
print('Test accuracy:', score[1])
#function to convert label to one-hot encofing
def convert_one_hot(labels):
one_hot_encoding = np.zeros(397)
one_hot_encoding[labels] = 1
return one_hot_encoding
#train test split
train_df, test_df = df.randomSplit([0.8, 0.2])
#transform spark dataframe to rdd format
train_data = train_df.rdd.map(
lambda row: (to_array(row['image']), convert_one_hot(row['Label'])))
test_data = test_df.rdd.map(
lambda row: (to_array(row['image']), convert_one_hot(row['Label'])))
#Training part
model = get_model(32)
spark_model = SparkModel(model, frequency='batch', mode='synchronous')
spark_model.fit(train_data,
epochs=5,
batch_size=32,
verbose=1,
validation_split=0.1)
#Model_Evaluating
x_test = np.array(test_data.map(lambda tuple: tuple[0]).collect())
y_test = np.array(test_data.map(lambda tuple: tuple[1]).collect())
accuracy = spark_model._master_network.evaluate(x_test, y_test)[1]
print('Accuracy is {}'.format(accuracy))
