def run_action(action, fawkes_config_file, app_config_file, query_term, query_response_file_format): if action == FawkesActions.FETCH: import fawkes.fetch.fetch as fetch fetch.fetch_reviews(fawkes_config_file) elif action == FawkesActions.PARSE: import fawkes.parse.parse as parse parse.parse_reviews(fawkes_config_file) elif action == FawkesActions.RUN_ALGO: import fawkes.algorithms.algo as algo algo.run_algo(fawkes_config_file) elif action == FawkesActions.GENERATE_EMAIL: import fawkes.email_summary.email_summary_detailed as email_summary_detailed email_summary_detailed.generate_email_summary_detailed( fawkes_config_file) elif action == FawkesActions.SEND_EMAIL: import fawkes.email_summary.send_email as send_email send_email.send_email(fawkes_config_file) elif action == FawkesActions.PUSH_ELASTICSEARCH: import fawkes.datastore.elasticsearch as elasticsearch elasticsearch.push_data_to_elasticsearch(fawkes_config_file) elif action == FawkesActions.QUERY_ELASTICSEARCH: import fawkes.datastore.elasticsearch as elasticsearch elasticsearch.query_from_elasticsearch( fawkes_config_file, query_term=query_term, format=query_response_file_format) elif action == FawkesActions.PUSH_SLACK: import fawkes.slackbot.slackbot as slackbot slackbot.send_reviews_to_slack(fawkes_config_file) elif action == FawkesActions.GENERATE_TEXT_MATCH_KEYWORDS: import fawkes.algorithms.categorisation.text_match.trainer as text_match_trainer text_match_trainer.generate_keyword_weights(fawkes_config_file) elif action == FawkesActions.TRAIN_LSTM_MODEL: import fawkes.algorithms.categorisation.lstm.trainer as lstm_trainer lstm_trainer.train_lstm_model(fawkes_config_file) elif action == FawkesActions.SUMMARIZE: import fawkes.algorithms.summarization.summarization as summarize summarize.generate_summary(app_config_file) elif action == FawkesActions.QUERY_SIMILAR_REVIEWS: import fawkes.algorithms.similarity.similarity as similarity similarity.get_similar_reviews_for_app(app_config_file, query_term, 20) else: raise Exception("Invalid action!")
def test_sanity(self): """ Test for sanity that parsing and algorithms are working """ # First we parse the sample data. parse.parse_reviews() parsed_output = utils.open_json( "data/parsed_data/sample-mint/parsed-user-feedback.json") expected_parsed_output = [{ "message": "I just heard about this budgeting app. So I gave it a try. I am impressed thus far. However I still cant add all of my financial institutions so my budget is kind of skewed. But other that I can say Im more aware of my spending", "timestamp": "2020/03/15 14:13:17", "rating": 5, "app_name": "sample-mint", "channel_name": "appstore", "channel_type": "ios", "hash_id": "de848685d11742dbea77e1e5ad7b892088ada9c9", "derived_insight": { "sentiment": None, "category": "uncategorized", "extra_properties": {} } }] self.assertEqual(parsed_output, expected_parsed_output) # Before running the algorithms, we generate the keyword weights. text_match_trainer.generate_keyword_weights() # We run the algorithms on that data algo.run_algo() processed_output = utils.open_json( "data/processed_data/sample-mint/processed-user-feedback.json") expected_processed_output = [{ "message": "I just heard about this budgeting app. So I gave it a try. I am impressed thus far. However I still cant add all of my financial institutions so my budget is kind of skewed. But other that I can say Im more aware of my spending", "timestamp": "2020/03/15 14:13:17", "rating": 5, "app_name": "sample-mint", "channel_name": "appstore", "channel_type": "ios", "hash_id": "6dde3aa82726c0a9e3777623854d839184767571", "derived_insight": { "sentiment": { "neg": 0.0, "neu": 0.928, "pos": 0.072, "compound": 0.4767 }, "category": "Application", "extra_properties": { "category_scores": { "User Experience": 0, "sign-in/sign-up": 0, "Notification": 0, "Application": 1, "ads": 0 }, "bug_feature": "feature" } } }] self.assertEqual(processed_output, expected_processed_output)
app_config_file = args.app_config query_term = args.query query_response_file_format = args.format # Initialise the logger init_logger() if action == FawkesActions.FETCH: import fawkes.fetch.fetch as fetch fetch.fetch_reviews(fawkes_config_file) elif action == FawkesActions.PARSE: import fawkes.parse.parse as parse parse.parse_reviews(fawkes_config_file) elif action == FawkesActions.RUN_ALGO: import fawkes.algorithms.algo as algo algo.run_algo(fawkes_config_file) elif action == FawkesActions.GENERATE_EMAIL: import fawkes.email_summary.email_summary_detailed as email_summary_detailed email_summary_detailed.generate_email_summary_detailed( fawkes_config_file) elif action == FawkesActions.SEND_EMAIL: import fawkes.email_summary.send_email as send_email send_email.send_email(fawkes_config_file) elif action == FawkesActions.PUSH_ELASTICSEARCH: import fawkes.datastore.elasticsearch as elasticsearch elasticsearch.push_data_to_elasticsearch(fawkes_config_file) elif action == FawkesActions.QUERY_ELASTICSEARCH: import fawkes.datastore.elasticsearch as elasticsearch elasticsearch.query_from_elasticsearch( fawkes_config_file, query_term=query_term,
def test_sanity(self): """ Test for sanity that parsing and algorithms are working """ # First we parse the sample data. parse.parse_reviews() parsed_output = utils.open_json( "data/parsed_data/sample-mint/parsed-user-feedback.json") expected_parsed_output = [{ "message": "I just heard about this budgeting app. So I gave it a try. I am impressed thus far. However I still cant add all of my financial institutions so my budget is kind of skewed. But other that I can say Im more aware of my spending", "timestamp": "2020/03/15 22:06:17", "rating": 5.0, "user_id": None, "app_name": "sample-mint", "channel_name": "appstore", "channel_type": "ios", "hash_id": "a5461e62ee4eccbab92900ba01d49d9ed0642dcc", "derived_insight": { "sentiment": None, "category": "uncategorized", "review_message_encoding": None, "extra_properties": {} }, "raw_review": { "updated": "2020-03-15 14:13:17", "rating": 5, "version": "7.1.0", "content": "I just heard about this budgeting app. So I gave it a try. I am impressed thus far. However I still can\u00e2\u20ac\u2122t add all of my financial institutions so my budget is kind of skewed. But other that I can say I\u00e2\u20ac\u2122m more aware of my spending" } }] self.assertEqual(parsed_output, expected_parsed_output) # Before running the algorithms, we generate the keyword weights. text_match_trainer.generate_keyword_weights() # We run the algorithms on that data algo.run_algo() processed_output = utils.open_json( "data/processed_data/sample-mint/processed-user-feedback.json") expected_processed_output = [{ "message": "I just heard about this budgeting app. So I gave it a try. I am impressed thus far. However I still cant add all of my financial institutions so my budget is kind of skewed. But other that I can say Im more aware of my spending", "timestamp": "2020/03/15 22:06:17", "rating": 5.0, "user_id": None, "app_name": "sample-mint", "channel_name": "appstore", "channel_type": "ios", "hash_id": "a5461e62ee4eccbab92900ba01d49d9ed0642dcc", "derived_insight": { "sentiment": { "neg": 0.0, "neu": 0.928, "pos": 0.072, "compound": 0.4767 }, "category": "Application", "review_message_encoding": None, "extra_properties": { "category_scores": { "User Experience": 0, "sign-in/sign-up": 0, "Notification": 0, "Application": 1, "ads": 0 }, "bug_feature": "feature" } }, "raw_review": { "updated": "2020-03-15 14:13:17", "rating": 5, "version": "7.1.0", "content": "I just heard about this budgeting app. So I gave it a try. I am impressed thus far. However I still can\u00e2\u20ac\u2122t add all of my financial institutions so my budget is kind of skewed. But other that I can say I\u00e2\u20ac\u2122m more aware of my spending" } }] self.assertEqual(processed_output, expected_processed_output)
parser = argparse.ArgumentParser() # Defining all the arguments define_arguments(parser) # Extracting all the alruments args = parser.parse_args() # Depending on the args, we execute the commands. action = args.action app_config_file = args.config if action == FawkesActions.FETCH: fetch.fetch_reviews(app_config_file) elif action == FawkesActions.PARSE: parse.parse_reviews(app_config_file) elif action == FawkesActions.RUN_ALGO: algo.run_algo(app_config_file) elif action == FawkesActions.GENERATE_EMAIL: email_summary_detailed.generate_email_summary_detailed(app_config_file) elif action == FawkesActions.SEND_EMAIL: send_email.send_email(app_config_file) elif action == FawkesActions.PUSH_ELASTICSEARCH: elasticsearch.push_data_to_elasticsearch(app_config_file) elif action == FawkesActions.PUSH_SLACK: slackbot.send_reviews_to_slack(app_config_file) elif action == FawkesActions.GENERATE_TEXT_MATCH_KEYWORDS: text_match_trainer.generate_keyword_weights(app_config_file) elif action == FawkesActions.TRAIN_LSTM_MODEL: lstm_trainer.train_lstm_model(app_config_file) else: raise Exception("Invalid action!")