def run_action(action, fawkes_config_file, app_config_file, query_term,
query_response_file_format):
if action == FawkesActions.FETCH:
import fawkes.fetch.fetch as fetch
fetch.fetch_reviews(fawkes_config_file)
elif action == FawkesActions.PARSE:
import fawkes.parse.parse as parse
parse.parse_reviews(fawkes_config_file)
elif action == FawkesActions.RUN_ALGO:
import fawkes.algorithms.algo as algo
algo.run_algo(fawkes_config_file)
elif action == FawkesActions.GENERATE_EMAIL:
import fawkes.email_summary.email_summary_detailed as email_summary_detailed
email_summary_detailed.generate_email_summary_detailed(
fawkes_config_file)
elif action == FawkesActions.SEND_EMAIL:
import fawkes.email_summary.send_email as send_email
send_email.send_email(fawkes_config_file)
elif action == FawkesActions.PUSH_ELASTICSEARCH:
import fawkes.datastore.elasticsearch as elasticsearch
elasticsearch.push_data_to_elasticsearch(fawkes_config_file)
elif action == FawkesActions.QUERY_ELASTICSEARCH:
import fawkes.datastore.elasticsearch as elasticsearch
elasticsearch.query_from_elasticsearch(
fawkes_config_file,
query_term=query_term,
format=query_response_file_format)
elif action == FawkesActions.PUSH_SLACK:
import fawkes.slackbot.slackbot as slackbot
slackbot.send_reviews_to_slack(fawkes_config_file)
elif action == FawkesActions.GENERATE_TEXT_MATCH_KEYWORDS:
import fawkes.algorithms.categorisation.text_match.trainer as text_match_trainer
text_match_trainer.generate_keyword_weights(fawkes_config_file)
elif action == FawkesActions.TRAIN_LSTM_MODEL:
import fawkes.algorithms.categorisation.lstm.trainer as lstm_trainer
lstm_trainer.train_lstm_model(fawkes_config_file)
elif action == FawkesActions.SUMMARIZE:
import fawkes.algorithms.summarization.summarization as summarize
summarize.generate_summary(app_config_file)
elif action == FawkesActions.QUERY_SIMILAR_REVIEWS:
import fawkes.algorithms.similarity.similarity as similarity
similarity.get_similar_reviews_for_app(app_config_file, query_term, 20)
else:
raise Exception("Invalid action!")
def test_sanity(self):
"""
Test for sanity that parsing and algorithms are working
"""
# First we parse the sample data.
parse.parse_reviews()
parsed_output = utils.open_json(
"data/parsed_data/sample-mint/parsed-user-feedback.json")
expected_parsed_output = [{
"message":
"I just heard about this budgeting app. So I gave it a try. I am impressed thus far. However I still cant add all of my financial institutions so my budget is kind of skewed. But other that I can say Im more aware of my spending",
"timestamp": "2020/03/15 14:13:17",
"rating": 5,
"app_name": "sample-mint",
"channel_name": "appstore",
"channel_type": "ios",
"hash_id": "de848685d11742dbea77e1e5ad7b892088ada9c9",
"derived_insight": {
"sentiment": None,
"category": "uncategorized",
"extra_properties": {}
}
}]
self.assertEqual(parsed_output, expected_parsed_output)
# Before running the algorithms, we generate the keyword weights.
text_match_trainer.generate_keyword_weights()
# We run the algorithms on that data
algo.run_algo()
processed_output = utils.open_json(
"data/processed_data/sample-mint/processed-user-feedback.json")
expected_processed_output = [{
"message":
"I just heard about this budgeting app. So I gave it a try. I am impressed thus far. However I still cant add all of my financial institutions so my budget is kind of skewed. But other that I can say Im more aware of my spending",
"timestamp": "2020/03/15 14:13:17",
"rating": 5,
"app_name": "sample-mint",
"channel_name": "appstore",
"channel_type": "ios",
"hash_id": "6dde3aa82726c0a9e3777623854d839184767571",
"derived_insight": {
"sentiment": {
"neg": 0.0,
"neu": 0.928,
"pos": 0.072,
"compound": 0.4767
},
"category": "Application",
"extra_properties": {
"category_scores": {
"User Experience": 0,
"sign-in/sign-up": 0,
"Notification": 0,
"Application": 1,
"ads": 0
},
"bug_feature": "feature"
}
}
}]
self.assertEqual(processed_output, expected_processed_output)
app_config_file = args.app_config
query_term = args.query
query_response_file_format = args.format
# Initialise the logger
init_logger()
if action == FawkesActions.FETCH:
import fawkes.fetch.fetch as fetch
fetch.fetch_reviews(fawkes_config_file)
elif action == FawkesActions.PARSE:
import fawkes.parse.parse as parse
parse.parse_reviews(fawkes_config_file)
elif action == FawkesActions.RUN_ALGO:
import fawkes.algorithms.algo as algo
algo.run_algo(fawkes_config_file)
elif action == FawkesActions.GENERATE_EMAIL:
import fawkes.email_summary.email_summary_detailed as email_summary_detailed
email_summary_detailed.generate_email_summary_detailed(
fawkes_config_file)
elif action == FawkesActions.SEND_EMAIL:
import fawkes.email_summary.send_email as send_email
send_email.send_email(fawkes_config_file)
elif action == FawkesActions.PUSH_ELASTICSEARCH:
import fawkes.datastore.elasticsearch as elasticsearch
elasticsearch.push_data_to_elasticsearch(fawkes_config_file)
elif action == FawkesActions.QUERY_ELASTICSEARCH:
import fawkes.datastore.elasticsearch as elasticsearch
elasticsearch.query_from_elasticsearch(
fawkes_config_file,
query_term=query_term,
def test_sanity(self):
"""
Test for sanity that parsing and algorithms are working
"""
# First we parse the sample data.
parse.parse_reviews()
parsed_output = utils.open_json(
"data/parsed_data/sample-mint/parsed-user-feedback.json")
expected_parsed_output = [{
"message":
"I just heard about this budgeting app. So I gave it a try. I am impressed thus far. However I still cant add all of my financial institutions so my budget is kind of skewed. But other that I can say Im more aware of my spending",
"timestamp": "2020/03/15 22:06:17",
"rating": 5.0,
"user_id": None,
"app_name": "sample-mint",
"channel_name": "appstore",
"channel_type": "ios",
"hash_id": "a5461e62ee4eccbab92900ba01d49d9ed0642dcc",
"derived_insight": {
"sentiment": None,
"category": "uncategorized",
"review_message_encoding": None,
"extra_properties": {}
},
"raw_review": {
"updated":
"2020-03-15 14:13:17",
"rating":
5,
"version":
"7.1.0",
"content":
"I just heard about this budgeting app. So I gave it a try. I am impressed thus far. However I still can\u00e2\u20ac\u2122t add all of my financial institutions so my budget is kind of skewed. But other that I can say I\u00e2\u20ac\u2122m more aware of my spending"
}
}]
self.assertEqual(parsed_output, expected_parsed_output)
# Before running the algorithms, we generate the keyword weights.
text_match_trainer.generate_keyword_weights()
# We run the algorithms on that data
algo.run_algo()
processed_output = utils.open_json(
"data/processed_data/sample-mint/processed-user-feedback.json")
expected_processed_output = [{
"message":
"I just heard about this budgeting app. So I gave it a try. I am impressed thus far. However I still cant add all of my financial institutions so my budget is kind of skewed. But other that I can say Im more aware of my spending",
"timestamp": "2020/03/15 22:06:17",
"rating": 5.0,
"user_id": None,
"app_name": "sample-mint",
"channel_name": "appstore",
"channel_type": "ios",
"hash_id": "a5461e62ee4eccbab92900ba01d49d9ed0642dcc",
"derived_insight": {
"sentiment": {
"neg": 0.0,
"neu": 0.928,
"pos": 0.072,
"compound": 0.4767
},
"category": "Application",
"review_message_encoding": None,
"extra_properties": {
"category_scores": {
"User Experience": 0,
"sign-in/sign-up": 0,
"Notification": 0,
"Application": 1,
"ads": 0
},
"bug_feature": "feature"
}
},
"raw_review": {
"updated":
"2020-03-15 14:13:17",
"rating":
5,
"version":
"7.1.0",
"content":
"I just heard about this budgeting app. So I gave it a try. I am impressed thus far. However I still can\u00e2\u20ac\u2122t add all of my financial institutions so my budget is kind of skewed. But other that I can say I\u00e2\u20ac\u2122m more aware of my spending"
}
}]
self.assertEqual(processed_output, expected_processed_output)
parser = argparse.ArgumentParser()
# Defining all the arguments
define_arguments(parser)
# Extracting all the alruments
args = parser.parse_args()
# Depending on the args, we execute the commands.
action = args.action
app_config_file = args.config
if action == FawkesActions.FETCH:
fetch.fetch_reviews(app_config_file)
elif action == FawkesActions.PARSE:
parse.parse_reviews(app_config_file)
elif action == FawkesActions.RUN_ALGO:
algo.run_algo(app_config_file)
elif action == FawkesActions.GENERATE_EMAIL:
email_summary_detailed.generate_email_summary_detailed(app_config_file)
elif action == FawkesActions.SEND_EMAIL:
send_email.send_email(app_config_file)
elif action == FawkesActions.PUSH_ELASTICSEARCH:
elasticsearch.push_data_to_elasticsearch(app_config_file)
elif action == FawkesActions.PUSH_SLACK:
slackbot.send_reviews_to_slack(app_config_file)
elif action == FawkesActions.GENERATE_TEXT_MATCH_KEYWORDS:
text_match_trainer.generate_keyword_weights(app_config_file)
elif action == FawkesActions.TRAIN_LSTM_MODEL:
lstm_trainer.train_lstm_model(app_config_file)
else:
raise Exception("Invalid action!")