class Main: """ Main class, makes necessary function calls to necessary classes """ def __init__(self): self.__db_manager = DBManager() self.__helper = GeneralHelpers() self.__plot_manager = PlotManager() self.__import_manager = ImportManager() self.__feature_manager = FeatureManager() self.years = ("2012", "2013", "2014", "2015") def retrieve_tweets(self, file_path_of_ids): """ Runs Import Manager to retrieve and import tweets :param file_path_of_ids: String, file path of tweets to import :return: void """ self.__import_manager.run(file_path_of_ids) def extract_features_and_generate_arff(self, n=3, analyzer='char', year='2012'): """ Makes necessary function calls to extract features for given year and to generate arff file :param n: int, ngram count :param analyzer: string, word or char :param year: string, 2012, 2013, 2014, 2015 or ALL :return: string, path of generated arff file """ # Getting tweets with year print("Getting tweets for year "+ year) tweets_for_given_year = self.__db_manager.get_tweets_for_year(year) print("Generating document and classes of tweets.") document, classes = self.__feature_manager.create_document_and_classes_for_tweets(tweets_for_given_year, True) print("Fitting the data, finding ngrams and frequencies.") ngrams, arff_data, vectorizer, X = self.__feature_manager.fit_data(document, classes, n, analyzer) print("Formatting the data for arff lib format.") formatted_arff_data = self.__feature_manager.format_data_for_arff(ngrams, arff_data) print("Generating file.") # Experiment name, 1grams, 2grams, 3grams.. or words experiment_name = str(n)+'Gram' if analyzer == 'char' else 'Word' # File name, TTNet_3grams_2012 file_name = MODEL_NAME + '_' + experiment_name + '_' + year # File name randomized TTNet_3grams_2012_asfas12.arff file_name = self.__helper.generate_random_file_name(file_name, ARFF_FILE_EXTENSION) # Arff file path ...../DataSet-ARFF/3Gram/TTNet/TTNet_3grams_2012_asfas12.arff arff_file_path = PROJECT_ROOT_DIRECTORY + DATASET_ARFF_DIR_NAME + experiment_name + '/' + MODEL_NAME + '/' # Generating the file with data self.__helper.generate_arff_file(arff_file_path, file_name, formatted_arff_data) print("Arff file generated at path:"+arff_file_path+file_name) def run_experiment_with_scikit_learn(self, n=1, analyzer='word'): """ Makes necessary method calls to run the experiment on scikit learn. :param n: int, count n in n-gram :param analyzer: string, either 'word' or 'char' :return: void """ # Retrieving all tweets from database print("Retrieving all tweets from database.") tweets_for_all_years = {} # Iterating over all years for year in self.years: # Retrieving tweets for the year tweets_for_year = self.__db_manager.get_tweets_for_year(year) tweets_for_all_years[year] = tweets_for_year # Creating a big list of tweets print("Creating a big list of tweets.") all_tweets = [] # Appending all tweets together for year, tweets in tweets_for_all_years.iteritems(): all_tweets += tweets # Generating document print("Generating document and classes by preprocessing") # Preprocessing and generation of document document, classes = self.__feature_manager.create_document_and_classes_for_tweets(all_tweets, True) # Getting years' tweets counts print("Getting years' tweets counts.") years_tweets_counts = {} for year in self.years: years_tweets_counts[year] = len(tweets_for_all_years[year]) all_processes = [] self.all_experiments_results = [] pool = Pool(cpu_count()-1 or 1) copy_reg.pickle(types.MethodType, self._reduce_method) print("Running experiments.") t0 = time.time() for i in range(0, N_EXPERIMENTS): print("Experiment:"+str(i)) experiment_manager = ExperimentManager(i, years_tweets_counts, n, analyzer) r = pool.apply_async(experiment_manager.run_experiment, args=(document, classes,), callback=self._accumulate_experiments_scores) all_processes.append(r) for a_process in all_processes: a_process.wait() t1 = time.time() print("Elapsed time:", t1- t0, " seconds") pool.close() pool.join() print("Cumulating all the experiments' scores.") final_results_from_all_experiments = self.__helper.cumulate_years_scores(self.all_experiments_results) return final_results_from_all_experiments def _reduce_method(self, m): """ :param m: :return: """ if m.im_self is None: return getattr, (m.im_class, m.im_func.func_name) else: return getattr, (m.im_self, m.im_func.func_name) def _accumulate_experiments_scores(self, an_experiments_result): """ Accumulates experiments' scores :return: void """ an_experiments_result = self.__helper.calculate_relative_scores(an_experiments_result) self.all_experiments_results.append(an_experiments_result) def plot_experiment_results(self, root_dir): """ Plots experiment's results from log files :param root_dir: string :return: void """ lines_scores = self.__helper.get_accuracy_scores_for_experiment_years_from_root_dir(root_dir) self.__plot_manager.plot_experiments_results(lines_scores) def plot_all_experiment_results_with_scikit_learn(self, all_line_scores_of_all_experiments): """ Plots all line scores of all experiments :param all_line_scores_of_all_experiments: dict :return: void """ self.__plot_manager.plot_experiments_results_with_scikit_learn(all_line_scores_of_all_experiments) def plot_years_scores(self, root_dir): """ Makes necessary function calls to plot years scores :param dir: string :return: void """ self.__plot_manager.plot_years_scores_from_root_directory(root_dir) def plot_2012_vs_rest(self, root_dir): """ Makes necessary function calls to plot 2012 vs REST scores :param root_dir: string :return: void """ self.__plot_manager.plot_2012_vs_rest(root_dir) def plot_top_feature_frequencies_in_years(self): """ Makes necessary function calls to plot top features frequencies' in years :return: void """ years_features_counts = {} for year in self.years: years_features_counts[year] = self.find_frequency_dictionary_for_year(year) self.__plot_manager.plot_top_feature_frequencies_in_years(years_features_counts) def find_frequency_dictionary_for_year(self, year): """ Finds frequencies of each feature for given year :param year: string :return: dict """ # For this particular method, find_roots=True, n=1, analyzer=word because we're working with top info gain words tweets_for_the_year = self.__db_manager.get_tweets_for_year(year) document, classes = self.__feature_manager.create_document_and_classes_for_tweets(tweets_for_the_year, find_roots=True) ngrams, arff_data, vectorizer, X = self.__feature_manager.fit_data(document, classes, n=1, analyzer='word') terms = vectorizer.get_feature_names() freqs = X.sum(axis=0).A1 result = sorted(zip(freqs, terms), reverse=True) freqs = [elm[0] for elm in result] terms = [elm[1] for elm in result] final_result = dict(zip(terms, freqs)) return final_result def plot_years_intersection_scores(self): """ Makes necessary function callst to plot a matrix which shows years' vocabularies similarities :return: void """ years_features_counts = {} for year in self.years: years_features_counts[year] = self.find_frequency_dictionary_for_year(year) self.__plot_manager.plot_years_intersection_scores(years_features_counts) def import_new_tweets_from_csv(self, root_path): """ :param root_path: :return: """ self.__import_manager.import_new_tweets_from_csv(root_path)