def main():
# time it
start = time.time()
# make rmn
rmn = RigidRMN(); rmn.load_rmn(name=RMN_NAME, save_path=RMN_PATH)
# declare analyzing function
analyze_func = partial(
analyze_session,
sample_n=sample_n,
doc_path=DOC_GEN_PATH,
rmn=rmn)
# gather data
data = [analyze_func(s) for s in sessions]
# Save
pickle_object(data, os.path.join(SAVE_PATH, TOPIC_TAG % 'all'))
end = time.time()
elapsed = end - start
# report
print("SUCCESS, took", round(elapsed / 60, 2), "minutes")
def analyze_session(session, sample_n, doc_path, rmn):
# read in session
df = load_generic_documents(sessions=[session], read_path=doc_path)
# analyze
analyzer = RMN_Analyzer(rmn, df)
print("Analyzing Session %s ..." % format(session, '03d'))
analyzer.predict_topics()
data = analyzer.analyze(sample_n)
print("Data Gathered for Session %s. " % format(session, '03d'))
# add session number
data.update({SESS: session})
# Save
pickle_object(data, os.path.join(SAVE_PATH, TOPIC_TAG % format(session, '03d')))
return data
def save_rmn(self, name, save_path):
"""
Save the model's weights, architecture and attributes
"""
# assemble attribute dictionary
attribute_dict = {
N_TOP_KEY: self.num_topics,
EMBED_KEY: self.embedding_matrix,
TOKEN_KEY: self.tokenizer_dict,
META_KEY: self.metadata_dict,
DIM_KEY: self.meta_embedding_dim
}
# make directory for model
model_path = os.path.join(save_path, RMN_TAG % name)
os.mkdir(model_path)
# save model weights
self.model.save_weights(os.path.join(model_path, MODEL))
# save model attributes
pickle_object(attribute_dict, os.path.join(model_path, ATTR))
def save_eval(self, name, save_path):
""""""
attribute_dict = {
'subjects': self.subjects,
'evl_started': self.evl_started,
'saved_index': self.saved_index
}
# make path for eval
eval_path = os.path.join(save_path, EVAL_TAG % name)
os.mkdir(eval_path)
# pickel df
pickle_object(self.df, os.path.join(eval_path, 'dataframe'))
# pickel found_keywords
pickle_object(self.to_embed_dict(self.found_keywords),
os.path.join(eval_path, 'found_keywords'))
# pickle attributes
pickle_object(attribute_dict, os.path.join(eval_path, 'attributes'))