def train_model(self):
w2v_model = word2vec.Word2Vec(name=self.model_name)
search_query = es.filter_by_query_string(self.model_settings["es_query_filter"])
sentences = list()
self.total_events = es.count_documents(search_query=search_query)
training_data_size_pct = settings.config.getint("machine_learning", "training_data_size_pct")
training_data_size = self.total_events / 100 * training_data_size_pct
logging.print_analysis_intro(event_type="training " + self.model_name, total_events=self.total_events)
total_training_events = int(min(training_data_size, self.total_events))
logging.init_ticker(total_steps=total_training_events, desc=self.model_name + " - preparing word2vec training set")
for doc in es.scan(search_query=search_query):
if len(sentences) < total_training_events:
logging.tick()
fields = es.extract_fields_from_document(doc)
if set(self.model_settings["sentence_format"]).issubset(fields.keys()):
new_sentences = helpers.utils.flatten_fields_into_sentences(fields=fields, sentence_format=self.model_settings["sentence_format"])
for sentence in new_sentences:
sentences.append(tuple(sentence))
# Remove all duplicates from sentences for training - REMOVED FOR TESTING
# sentences = list(sentences)
else:
# We have collected sufficient training data
break
# Now, train the model
if len(sentences) > 0:
w2v_model.train_model(sentences)
else:
logging.logger.warning("no sentences to train model on. Are you sure the sentence configuration is correctly defined?")
def evaluate_model(self):
self.extract_extra_model_settings()
# Train the model
if self.model_settings["train_model"]:
self.train_model()
return
w2v_model = word2vec.Word2Vec(name=self.model_name)
search_query = es.filter_by_query_string(self.model_settings["es_query_filter"])
if not w2v_model.is_trained():
logging.logger.warning("model was not trained! Skipping analysis.")
else:
# Check if we need to run the test data instead of real data
if w2v_model.use_test_data:
logging.print_generic_intro("using test data instead of live data to evaluate model " + self.model_name)
self.evaluate_test_sentences(w2v_model=w2v_model)
return
self.total_events = es.count_documents(search_query=search_query)
logging.print_analysis_intro(event_type="evaluating " + self.model_name, total_events=self.total_events)
logging.init_ticker(total_steps=self.total_events, desc=self.model_name + " - evaluating word2vec model")
raw_docs = list()
eval_sentences = list()
for doc in es.scan(search_query=search_query):
logging.tick()
fields = es.extract_fields_from_document(doc)
try:
new_sentences = helpers.utils.flatten_fields_into_sentences(fields=fields, sentence_format=self.model_settings["sentence_format"])
eval_sentences.extend(new_sentences)
except KeyError:
logging.logger.debug("skipping event which does not contain the target and aggregator fields we are processing. - [" + self.model_name + "]")
continue
for _ in new_sentences:
raw_docs.append(doc)
# Evaluate batch of events against the model
if logging.current_step == self.total_events or len(eval_sentences) >= settings.config.getint("machine_learning", "word2vec_batch_eval_size"):
logging.logger.info("evaluating batch of " + str(len(eval_sentences)) + " sentences")
outliers = self.evaluate_batch_for_outliers(w2v_model=w2v_model, eval_sentences=eval_sentences, raw_docs=raw_docs)
if len(outliers) > 0:
unique_summaries = len(set(o.outlier_dict["summary"] for o in outliers))
logging.logger.info("total outliers in batch processed: " + str(len(outliers)) + " [" + str(unique_summaries) + " unique summaries]")
# Reset data structures for next batch
raw_docs = list()
eval_sentences = list()