def __init__(self, db): #TODO: no kwargs in module constructors AbstractController.__init__(self, db) self._connection_type = unicode( self._config_parser.get(self.__class__.__name__, "connection_type")) self._max_objects_without_saving = self._config_parser.eval( self.__class__.__name__, "max_objects_without_saving") self._min_number_of_posts_per_author = self._config_parser.eval( self.__class__.__name__, "min_number_of_posts_per_author") self._num_of_random_authors_for_graph = self._config_parser.eval( self.__class__.__name__, "num_of_random_authors_for_graph") self._author_connections_edges = [] # will save the connection and the reversed in order to make this connection type as un directed. self._existing_connections_dict = {}
def __init__(self, db): AbstractController.__init__(self, db) self._config_parser = getConfig() self._path = self._config_parser.get(self.__class__.__name__, "path") self._backup_path = self._config_parser.get(self.__class__.__name__, "backup_path") self._bad_actors_file_name = self._config_parser.get( self.__class__.__name__, "bad_actors_file_name") self._potential_good_actors_file_name = self._config_parser.get( self.__class__.__name__, "potential_good_actors_file_name") self._csv_header = self._config_parser.eval(self.__class__.__name__, "csv_header") self._vico_importer_bad_actors = [] self._csv_importer = PostCSVExporter()
def __init__(self, db): AbstractController.__init__(self, db) self.necessary_columns = self._config_parser.eval( self.__class__.__name__, 'columns') self._snopes_strainer = SoupStrainer("li", {"class": "breadcrumb-item"}) self._header = { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3', 'Accept-Encoding': 'none', 'Accept-Language': 'en-US,en;q=0.8', 'Connection': 'keep-alive' }
def __init__(self, db): AbstractController.__init__(self, db) self._script_name = 'autotopic_model_creation.R' self._autotopics_dir = self._config_parser.get(self.__class__.__name__, "autotopics_dir") self._db_name = self._config_parser.get("DB", "DB_name_prefix") + \ self._config_parser.get("DEFAULT", "social_network_name") + \ self._config_parser.get("DB", "DB_name_suffix") self._min_topic = self._config_parser.get(self.__class__.__name__, "min_topics") self._max_topic = self._config_parser.get(self.__class__.__name__, "max_topics") self.single_post_per_author = self._config_parser.get(self.__class__.__name__, "single_post_per_author") self._thresh_low = self._config_parser.get(self.__class__.__name__, "thresh_low") self._frequent_keywords_to_remove = self._config_parser.get(self.__class__.__name__, "frequent_keywords_to_remove") self._directory = self._autotopics_dir + "/tmp1/"
def __init__(self, db): AbstractController.__init__(self, db) self._working_app_number = self._config_parser.eval(self.__class__.__name__, "working_app_number") self._maximal_get_friend_ids_requests_in_window = self._config_parser.eval(self.__class__.__name__, "maximal_get_friend_ids_requests_in_window") self._maximal_get_follower_ids_requests_in_window = self._config_parser.eval(self.__class__.__name__, "maximal_get_follower_ids_requests_in_window") self._maximal_get_user_requests_in_window = self._config_parser.eval(self.__class__.__name__, "maximal_get_user_requests_in_window") self._maximal_user_ids_allowed_in_single_get_user_request = self._config_parser.eval(self.__class__.__name__, "maximal_user_ids_allowed_in_single_get_user_request") self._num_of_twitter_status_id_requests_without_checking = self._config_parser.eval(self.__class__.__name__, "num_of_twitter_status_id_requests_without_checking") self._num_of_twitter_timeline_requests_without_checking = self._config_parser.eval(self.__class__.__name__, "num_of_twitter_timeline_requests_without_checking") self._max_tweet_ids_allowed_in_single_get_tweets_by_tweet_ids_request = self._config_parser.eval( self.__class__.__name__, "max_tweet_ids_allowed_in_single_get_tweets_by_tweet_ids_request") self._max_num_of_tweet_ids_requests_without_checking = self._config_parser.eval(self.__class__.__name__, "max_num_of_tweet_ids_requests_without_checking") self._num_of_get_friend_ids_requests = 0 self._num_of_get_follower_ids_requests = 0 self._num_of_get_timeline_statuses = 0 self._num_of_twitter_status_id_requests = 0 self._num_of_twitter_timeline_requests = 0 self._num_of_get_tweet_ids_requests = 0 self._total_author_connections = [] print("Creating TwitterApiRequester") self._twitter_api_requester = TwitterApiRequester() # self._find_source_twitter_id() logging.info("Setup DB...") print("Setup DB...") self._db = DB() self._db.setUp()
def __init__(self, db): AbstractController.__init__(self, db) self._set_type_affiliation = 'set_affiliation' self._targeted_class_dict = self._config_parser.eval( self.__class__.__name__, "targeted_class_dict") self._targeted_class_field_names = self._config_parser.eval( self.__class__.__name__, "targeted_class_field_names") self._classifier_type_names = self._config_parser.eval( self.__class__.__name__, "classifier_type_names") self._num_of_features = self._config_parser.eval( self.__class__.__name__, "num_of_features") self._indentifier_field_name = self._config_parser.get( self.__class__.__name__, "indentifier_field_name") self._replace_missing_values = self._config_parser.get( self.__class__.__name__, "replace_missing_values") self._selected_features = self._config_parser.eval( self.__class__.__name__, "selected_features") self._feature_names_to_remove = self._config_parser.eval( self.__class__.__name__, "feature_names_to_remove") self._order_of_results_dictionary = self._config_parser.eval( self.__class__.__name__, "order_of_results_dictionary") self._results_file_name = self._config_parser.get( self.__class__.__name__, "results_file_name") self._results_table_file_name = self._config_parser.get( self.__class__.__name__, "results_table_file_name") self._path = self._config_parser.get(self.__class__.__name__, "path") self._column_names_for_results_table = self._config_parser.eval( self.__class__.__name__, "column_names_for_results_table") self._full_path_model_directory = self._config_parser.get( self.__class__.__name__, "full_path_model_directory") self._is_divide_to_training_and_test_sets_by_field_name_then_train_and_evaluate = self._config_parser.eval( self.__class__.__name__, "is_divide_to_training_and_test_sets_by_field_name_then_train_and_evaluate" ) self._is_divide_to_training_and_test_sets_by_k_fold_cross_validation_then_train_and_evaluate = self._config_parser.eval( self.__class__.__name__, "is_divide_to_training_and_test_sets_by_k_fold_cross_validation_then_train_and_evaluate" ) self._k_for_fold = self._config_parser.eval(self.__class__.__name__, "k_for_fold") self._training_label = "training" self._test_label = "test"
def __init__(self, db, targeted_class_name): AbstractController.__init__(self, db) self._default_cols = [ 'author_guid', 'attribute_name', 'attribute_value', 'author_type' ] self.optional_classes = ['bad_actor', 'good_actor'] self._fill_empty = self._config_parser.eval(self.__class__.__name__, "fill_empty") self._to_replace_to_numerals = self._config_parser.eval( self.__class__.__name__, "to_replace_to_numerals") self._to_replace_authors_guid_author_name_to_authors_guid = False self._target_class_name = targeted_class_name self._author_guid = self._config_parser.eval(self.__class__.__name__, "id_field") self._remove_features_by_prefix = self._config_parser.eval( self.__class__.__name__, "remove_features_by_prefix") self._select_features_by_prefix = self._config_parser.eval( self.__class__.__name__, "select_features_by_prefix")
def __init__(self, db): AbstractController.__init__(self, db) self._db_path = self._config_parser.get(self.__class__.__name__, "DB_path").replace('\\', '/') self._db_name = self._config_parser.get(self.__class__.__name__, "DB_name_prefix") + \ self._config_parser.get("DEFAULT", "social_network_name") + \ self._config_parser.get(self.__class__.__name__, "DB_name_suffix") self._single_post_per_author = self._config_parser.get( self.__class__.__name__, "single_post_per_author") self._lda_model_path = self._config_parser.get(self.__class__.__name__, "lda_model_path") self._k = self._config_parser.get(self.__class__.__name__, "k") self._script_path = self._config_parser.get(self.__class__.__name__, "script_path") self._script_name = 'autotopic_wrapper.R'
def __init__(self, db, **kwargs): AbstractController.__init__(self, db) self._input_path = self._config_parser.eval(self.__class__.__name__, "input_path") self._topic_statistics_file = self._config_parser.eval( self.__class__.__name__, "topic_statistics_file") self._post_id_topic_id_dict_file = self._config_parser.eval( self.__class__.__name__, "post_id_topic_id_dict_file") self._classifier_name = self._config_parser.eval( self.__class__.__name__, "classifier_name") #self._dataset_name = self._config_parser.eval(self.__class__.__name__, "dataset_name") self._selected_features_file = self._config_parser.eval( self.__class__.__name__, "selected_features_file") self._features_extracted_by_tuple = self._config_parser.eval( self.__class__.__name__, "features_extracted_by_tuple") self._aggregated_features = self._config_parser.eval( self.__class__.__name__, "aggregated_features") self._prefix = self.__class__.__name__
def __init__(self, db): AbstractController.__init__(self, db) self._remove_stopwords = self._config_parser.eval( self.__class__.__name__, "remove_stopwords") self._apply_stemming = self._config_parser.eval( self.__class__.__name__, "apply_stemming") self._stopwords_file = self._config_parser.get(self.__class__.__name__, "stopwords_file") self._stemming_lang = self._config_parser.get(self.__class__.__name__, "stemming_language") self._is_preprocess_posts = self._config_parser.eval( self.__class__.__name__, "is_preprocess_posts") self._is_preprocess_authors = self._config_parser.eval( self.__class__.__name__, "is_preprocess_authors") self._max_objects_without_saving = self._config_parser.eval( self.__class__.__name__, "max_objects_without_saving") pattern_regex = "[A-Za-z']+".decode('utf-8') self.compiled_regex = re.compile(pattern_regex)
def __init__(self, db): AbstractController.__init__(self, db) self._graph_types = self._config_parser.eval(self.__class__.__name__, "graph_types") self._number_iterations = self._config_parser.eval( self.__class__.__name__, "number_iterations") negative_link_method = self._config_parser.eval( self.__class__.__name__, "negative_link_method") percent_of_links_to_sample = self._config_parser.eval( self.__class__.__name__, "percent_of_links_to_sample") measure_names = self._config_parser.eval(self.__class__.__name__, "measure_names") self._combinations = list( itertools.product(negative_link_method, percent_of_links_to_sample, measure_names)) self._author_features = []
def __init__(self, db): AbstractController.__init__(self, db) self._tsv_input_files_path = self._config_parser.get( self.__class__.__name__, "tsv_input_files") self._tsv_authors_file_name = self._config_parser.get( self.__class__.__name__, "tsv_authors_file_name") self._tsv_posts_file_name = self._config_parser.get( self.__class__.__name__, "tsv_posts_file_name") self._tsv_author_connections_file_name = self._config_parser.get( self.__class__.__name__, "tsv_author_connections_file_name") self._actions = self._config_parser.eval(self.__class__.__name__, "actions") self._authors_header = self._config_parser.eval( self.__class__.__name__, "authors_header") self._posts_header = self._config_parser.eval(self.__class__.__name__, "posts_header") self._author_connections_header = self._config_parser.eval( self.__class__.__name__, "author_connections_header") self._url_index = 1
def __init__(self, db, **kwargs): AbstractController.__init__(self, db) self._features_list = self._config_parser.eval(self.__class__.__name__, "features_list") self._prefix = self.__class__.__name__ self._num_of_terms_in_topic = self._config_parser.eval( self.__class__.__name__, "num_of_terms_in_topic") self.num_topics = self._config_parser.eval(self.__class__.__name__, "number_of_topics") self.stopword_file = self._config_parser.get(self.__class__.__name__, "stopword_file") self.stemlanguage = self._config_parser.get(self.__class__.__name__, "stem_language") self.topics = [] self.topic = None self.post_id_to_topic_id = None self.topic_id_to_topics = {} self.model = None self.dictionary = None self.corpus = None
def __init__(self, db): AbstractController.__init__(self, db) self._k = self._config_parser.eval(self.__class__.__name__, "k") self._num_of_iterations = int(self._config_parser.get(self.__class__.__name__, "num_of_iterations")) self._index_field_for_predictions = self._config_parser.get(self.__class__.__name__, "index_field_for_predictions") self._connection_types = self._config_parser.get(self.__class__.__name__, "connection_type") self._desicion_models = self._config_parser.eval(self.__class__.__name__, "decision_models") self._result_authors_file_name = self._config_parser.get(self.__class__.__name__, "result_author_file_name") self._result_classifier_file_name = self._config_parser.get(self.__class__.__name__, "result_classifier_file_name") self._train_percent = int(self._config_parser.get(self.__class__.__name__,"train_percent")) self._randomly_classify_unlabelled = self._config_parser.eval(self.__class__.__name__, "randomly_clasify_unlabled") self._classification_options = self._config_parser.eval(self.__class__.__name__, "clasification_options") self._targeted_class = self._config_parser.get(self.__class__.__name__, "targeted_class") self._similarity_function = self._config_parser.get(self.__class__.__name__, "similarity_function") self._update_targeted_class_in_db = self._config_parser.eval(self.__class__.__name__, "update_targeted_class_in_db") self._random_seed =int(self._config_parser.get(self.__class__.__name__,"random_seed")) self._connections = [] self._result_dataframe_of_author = None self._result_dataframe_of_classifier = None self._data_holder = None
def __init__(self, db): AbstractController.__init__(self, db) self._results_path = self._config_parser.get(self.__class__.__name__, "results_path") self._predictions_per_iteration_path = self._config_parser.get( self.__class__.__name__, "predictions_per_iteration_path") self._similarity_functions = self._config_parser.eval( self.__class__.__name__, "similarity_functions") self._k = self._config_parser.eval(self.__class__.__name__, "k") self._compute_knn_based_on_link_prediction = self._config_parser.eval( self.__class__.__name__, "compute_knn_based_on_lp") self._link_prediction_models = self._config_parser.eval( self.__class__.__name__, "link_prediction_models") self._results_averaged_on_report = self._config_parser.eval( self.__class__.__name__, "results_averaged_on_report") self._generate_anchors = self._config_parser.eval( self.__class__.__name__, "generate_anchors") self._num_iterations = self._config_parser.eval( self.__class__.__name__, "num_iterations") self._targeted_class_dict = self._config_parser.eval( self.__class__.__name__, "targeted_class_dict") self._decision_models = self._config_parser.eval( self.__class__.__name__, "decision_models") self._targeted_class_field_name = self._config_parser.eval( self.__class__.__name__, "targeted_class_field_name") self._order_of_results_dictionary = self._config_parser.eval( self.__class__.__name__, "order_of_results_dictionary") self._results_table_file_name = self._config_parser.get( self.__class__.__name__, "results_table_file_name") self._path = self._config_parser.get(self.__class__.__name__, "path") self._column_names_for_results_table = self._config_parser.eval( self.__class__.__name__, "column_names_for_results_table") self._index_field_for_predictions = self._config_parser.get( self.__class__.__name__, "index_field_for_predictions") self._targeted_class_anchors_percent_size = self._config_parser.eval( self.__class__.__name__, "targeted_class_anchors_percent_size") self._targeted_class_num_of_authors_dict = self._config_parser.eval( self.__class__.__name__, "targeted_class_num_of_authors_dict")
def __init__(self, db): AbstractController.__init__(self, db) self._actions = self._config_parser.eval(self.__class__.__name__, "actions") self._minimal_num_of_posts = self._config_parser.eval( self.__class__.__name__, "minimal_num_of_posts") self._limit_friend_follower_number = self._config_parser.eval( self.__class__.__name__, "limit_friend_follower_number") self._maximal_tweets_count_in_timeline = self._config_parser.eval( self.__class__.__name__, "maximal_tweets_count_in_timeline") self._vico_importer_twitter_authors = [] self._found_twitter_users = [] self._social_network_crawler = Twitter_Rest_Api(db) self._suspended_authors = [] self._max_users_without_saving = self._config_parser.eval( self.__class__.__name__, "max_users_without_saving") self._output_path = self._config_parser.eval(self.__class__.__name__, "output_path") self._posts = [] self._authors = [] self._post_citatsions = []
def __init__(self, db): AbstractController.__init__(self, db) self._actions = self._config_parser.eval(self.__class__.__name__, "actions") self._output_path = self._config_parser.eval(self.__class__.__name__, "output_path") self._fake_news_labels = self._config_parser.eval( self.__class__.__name__, "fake_news_labels") self._real_news_labels = self._config_parser.eval( self.__class__.__name__, "real_news_labels") self._min_num_of_distinct_claims_author_involved = self._config_parser.eval( self.__class__.__name__, "min_num_of_distinct_claims_author_involved") self._min_num_of_published_posts = self._config_parser.eval( self.__class__.__name__, "min_num_of_published_posts") self._threshold = self._config_parser.eval(self.__class__.__name__, "threshold") self._original_fake_news_label = u"fake_news" self._original_real_news_label = u"real_news" self._fake_news_promoter_label = u"fake_news_promoter" self._real_news_promoter_label = u"real_news_promoter"
def __init__(self, db): AbstractController.__init__(self, db) self._target_field = self._config_parser.eval(self.__class__.__name__, "target_field") self._data_handler = Data_Handler( db, targeted_class_name=self._target_field) self._classifier_file_path_and_name = self._config_parser.eval( self.__class__.__name__, "classifier_file_path_and_name") self._selected_feature_file_path_and_name = self._config_parser.eval( self.__class__.__name__, "selected_feature_file_path_and_name") self._saved_prediction_file_path_and_name = self._config_parser.eval( self.__class__.__name__, "saved_prediction_file_path_and_name") self._fill_empty = self._config_parser.eval(self.__class__.__name__, "fill_empty") self._remove_features = self._config_parser.eval( self.__class__.__name__, "remove_features") self._select_features = self._config_parser.eval( self.__class__.__name__, "select_features") self._label_text_to_value = self._config_parser.eval( self.__class__.__name__, "label_text_to_value") self._classifier = self._load_file(self._classifier_file_path_and_name)
def __init__(self, db, **kwargs): AbstractController.__init__(self, db) self.config_parser = getConfig() self._pipeline = []
def __init__(self, db): AbstractController.__init__(self, db) self._input_path = self._config_parser.eval(self.__class__.__name__, 'input_path') self._connection_type = self._config_parser.eval( self.__class__.__name__, 'type')
def __init__(self, db): AbstractController.__init__(self, db) self._actions = self._config_parser.eval(self.__class__.__name__, "actions") self._json_path = self._config_parser.eval(self.__class__.__name__, "json_path")
def __init__(self, db): AbstractController.__init__(self, db) configInst = getConfig()
def execute(self, window_start): AbstractController.execute(self, window_start) self._db.deleteBoostAuth(self._window_start) curr_posts = self._db.getPostsListWithoutEmptyRowsByDate(self._window_start, self._window_end) stats_calc = AuthorStatisticsCalculator(self._db) authors_statistics, post_to_pointers_scores = stats_calc.run_authors_stats_calculation(curr_posts) authors_boost_stats_list = [] total = len(authors_statistics) current = 0 for author, stats in authors_statistics.iteritems(): current += 1 print "boost authors model... processing author_boost_stats: " + str(current) + " out of " + str(total) if (author): found_author = self._db.getAuthorByName(author) # We change this line for authors which have the same guid, but different names so some of them not found by the given author name. # This is a known bug in VICO. if len(found_author) > 0: authorDomain = found_author[0].domain author_guid = found_author[0].author_guid author_boost_stats = Author_boost_stats(window_start=self._window_start, window_end=self._window_end, author_name=author, author_domain=authorDomain, boosting_timeslots_participation_count=stats.boosting_timeslots_participation_count, count_of_authors_sharing_boosted_posts=stats.count_of_authors_sharing_boosted_keyposts, num_of_pointers=stats.num_of_pointers, num_of_pointed_posts=stats.num_of_pointed_keyposts, pointers_scores=stats.pointers_scores, scores_sum=stats.scores_sum, scores_avg=stats.scores_avg, scores_std=stats.scores_std, author_guid=author_guid) authors_boost_stats_list.append(author_boost_stats) authors_boost_stats_list = list(set(authors_boost_stats_list)) self._db.addAuthors_boost_stats(authors_boost_stats_list) post_to_pointers_scores_list = [] duplicatesList = [] total_post_to_pointers = len(post_to_pointers_scores) current_post_to_pointers = 0 for post, pointers_scores in post_to_pointers_scores.iteritems(): current_post_to_pointers += 1 print "boost authors model... processing post to pointers : " + str( current_post_to_pointers) + " out of " + str(total_post_to_pointers) total_pointer_scores = len(pointers_scores) current_pointer_scores = 0 for pointer, score in pointers_scores.iteritems(): current_pointer_scores += 1 print "boost authors model... processing pointer scores: " + str( current_pointer_scores) + " out of " + str(total_pointer_scores) temp = [] post_data = self._db.getPostUsingURL(post, self._window_start, self._window_end) if (len(post_data) > 0): post_id = post_data[0].post_id found_author = self._db.getAuthorByName(pointer.author) # We change this line for authors which have the same guid, but different names so some of them not found by the given author name. # This is a known bug in VICO. if len(found_author) > 0: authorDomain = found_author[0].domain temp.extend([post_id, self._window_start, self._window_end, post, authorDomain, pointer.author, str_to_date(pointer.date)]) if temp not in duplicatesList: duplicatesList.append(temp) post_to_pointers = Post_to_pointers_scores(author_name=pointer.author, author_domain=authorDomain, post_id_to=post_id, window_start=self._window_start, window_end=self._window_end, url_to=post, datetime=str_to_date(pointer.date), pointer_score=score) post_to_pointers_scores_list.append(post_to_pointers) self._db.addPosts_to_pointers_scores(post_to_pointers_scores_list)
def __init__(self, db, **kwargs): AbstractController.__init__(self, db) self.config_parser = getConfig() self._pipeline = [] self._load_parameters = self._config_parser.eval( self.__class__.__name__, "load_parameters")
def __init__(self, db): AbstractController.__init__(self, db) self._min_number_of_posts_per_author = self._config_parser.eval( self.__class__.__name__, "min_number_of_posts_per_author") self._random_authors_requirements = self._config_parser.eval( self.__class__.__name__, "random_authors_requirements")
def __init__(self, db, **kwargs): AbstractController.__init__(self, db) self._features_list = self._config_parser.eval(self.__class__.__name__, "features_list") self._good_claim_type = self._config_parser.eval(self.__class__.__name__, "good_claim_type") self._bad_claim_type = self._config_parser.eval(self.__class__.__name__, "bad_claim_type") self._prefix = self.__class__.__name__
def __init__(self, db): AbstractController.__init__(self, db) self._path_for_downloaded_images = self._config_parser.eval( self.__class__.__name__, "path_for_downloaded_images")
def __init__(self, db): AbstractController.__init__(self, db) self._images_folder = self._config_parser.eval(self.__class__.__name__, "images_folder")
def __init__(self, db): AbstractController.__init__(self, db)
def __init__(self, db): AbstractController.__init__(self, db) self._path_for_downloaded_images = self._config_parser.eval( self.__class__.__name__, "path_for_downloaded_images") self._source_target_images_to_download_dicts = self._config_parser.eval( self.__class__.__name__, "source_target_images_to_download_dicts")