def classify_text(self, text):
"""
function to perform sentiment analysis text_classification
:param text: text sent in/from written query to be analyzed
"""
sentimentInfo = self.models.get("text_classification")
vocab = sentimentInfo["vocabulary"]
# Clean up text
text = lemmatize_text(text_clean_up([text]))
# Encode text
text = encode_text(vocab, text)
text = sequence.pad_sequences(text, sentimentInfo["max_text_length"])
model = sentimentInfo["model"]
prediction = tf.keras.backend.argmax(model.predict(text))
return sentimentInfo["classes"][tf.keras.backend.get_value(prediction)[0]]
def text_classification_query(self,
instruction,
drop=None,
preprocess=True,
label_column=None,
test_size=0.2,
random_state=49,
learning_rate=1e-2,
epochs=20,
monitor="val_loss",
batch_size=32,
max_text_length=200,
max_features=20000,
generate_plots=True,
save_model=False,
save_path=os.getcwd()):
"""
function to apply text_classification algorithm for sentiment analysis
:param many params: used to hyperparametrize the function.
:return a dictionary object with all of the information for the algorithm.
"""
if test_size < 0:
raise Exception("Test size must be a float between 0 and 1")
if test_size >= 1:
raise Exception(
"Test size must be a float between 0 and 1 (a test size greater than or equal to 1 results in no training "
"data)")
if epochs < 1:
raise Exception(
"Epoch number is less than 1 (model will not be trained)")
if batch_size < 1:
raise Exception("Batch size must be equal to or greater than 1")
if max_text_length < 1:
raise Exception("Max text length must be equal to or greater than 1")
if save_model:
if not os.path.exists(save_path):
raise Exception("Save path does not exists")
if test_size == 0:
testing = False
else:
testing = True
data = DataReader(self.dataset)
data = data.data_generator()
if preprocess:
data.fillna(0, inplace=True)
if drop is not None:
data.drop(drop, axis=1, inplace=True)
if label_column is None:
label = "label"
else:
label = label_column
X, Y, target = get_target_values(data, instruction, label)
Y = np.array(Y)
classes = np.unique(Y)
logger("->", "Target Column Found: {}".format(target))
vocab = {}
if preprocess:
logger("Preprocessing data")
X = lemmatize_text(text_clean_up(X.array))
vocab = X
X = encode_text(X, X)
X = np.array(X)
model = get_keras_text_class(max_features, len(classes), learning_rate)
logger("Building Keras LSTM model dynamically")
X_train, X_test, y_train, y_test = train_test_split(
X, Y, test_size=test_size, random_state=random_state)
X_train = sequence.pad_sequences(X_train, maxlen=max_text_length)
X_test = sequence.pad_sequences(X_test, maxlen=max_text_length)
y_vals = np.unique(np.append(y_train, y_test))
label_mappings = {}
for i in range(len(y_vals)):
label_mappings[y_vals[i]] = i
map_func = np.vectorize(lambda x: label_mappings[x])
y_train = map_func(y_train)
y_test = map_func(y_test)
logger("Training initial model")
# early stopping callback
es = EarlyStopping(monitor=monitor, mode='auto', verbose=0, patience=5)
history = model.fit(X_train,
y_train,
validation_data=(X_test, y_test),
batch_size=batch_size,
epochs=epochs,
callbacks=[es],
verbose=0)
logger(
"->", "Final training loss: {}".format(
history.history["loss"][len(history.history["loss"]) - 1]))
if testing:
logger(
"->", "Final validation loss: {}".format(
history.history["val_loss"][len(history.history["val_loss"]) -
1]))
logger(
"->", "Final validation accuracy: {}".format(
history.history["val_accuracy"][
len(history.history["val_accuracy"]) - 1]))
losses = {
'training_loss': history.history['loss'],
'val_loss': history.history['val_loss']
}
accuracy = {
'training_accuracy': history.history['accuracy'],
'validation_accuracy': history.history['val_accuracy']
}
else:
logger("->",
"Final validation loss: {}".format("0, No validation done"))
losses = {'training_loss': history.history['loss']}
accuracy = {'training_accuracy': history.history['accuracy']}
plots = {}
if generate_plots:
# generates appropriate classification plots by feeding all
# information
logger("Generating plots")
plots = generate_classification_plots(history, X, Y, model, X_test,
y_test)
if save_model:
save(model, save_model, save_path=save_path)
logger(
"Storing information in client object under key 'text_classification'")
# storing values the model dictionary
self.models["text_classification"] = {
"model": model,
"classes": classes,
"plots": plots,
"target": Y,
"vocabulary": vocab,
"interpreter": label_mappings,
"max_text_length": max_text_length,
'test_data': {
'X': X_test,
'y': y_test
},
'losses': losses,
'accuracy': accuracy
}
clearLog()
return self.models["text_classification"]
