def __init__(self, db): #TODO: no kwargs in module constructors
AbstractController.__init__(self, db)
self._connection_type = unicode(
self._config_parser.get(self.__class__.__name__,
"connection_type"))
self._max_objects_without_saving = self._config_parser.eval(
self.__class__.__name__, "max_objects_without_saving")
self._min_number_of_posts_per_author = self._config_parser.eval(
self.__class__.__name__, "min_number_of_posts_per_author")
self._num_of_random_authors_for_graph = self._config_parser.eval(
self.__class__.__name__, "num_of_random_authors_for_graph")
self._author_connections_edges = []
# will save the connection and the reversed in order to make this connection type as un directed.
self._existing_connections_dict = {}
def __init__(self, db):
AbstractController.__init__(self, db)
self._config_parser = getConfig()
self._path = self._config_parser.get(self.__class__.__name__, "path")
self._backup_path = self._config_parser.get(self.__class__.__name__,
"backup_path")
self._bad_actors_file_name = self._config_parser.get(
self.__class__.__name__, "bad_actors_file_name")
self._potential_good_actors_file_name = self._config_parser.get(
self.__class__.__name__, "potential_good_actors_file_name")
self._csv_header = self._config_parser.eval(self.__class__.__name__,
"csv_header")
self._vico_importer_bad_actors = []
self._csv_importer = PostCSVExporter()
def __init__(self, db):
AbstractController.__init__(self, db)
self.necessary_columns = self._config_parser.eval(
self.__class__.__name__, 'columns')
self._snopes_strainer = SoupStrainer("li",
{"class": "breadcrumb-item"})
self._header = {
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
'Accept-Encoding': 'none',
'Accept-Language': 'en-US,en;q=0.8',
'Connection': 'keep-alive'
}
def __init__(self, db):
AbstractController.__init__(self, db)
self._script_name = 'autotopic_model_creation.R'
self._autotopics_dir = self._config_parser.get(self.__class__.__name__, "autotopics_dir")
self._db_name = self._config_parser.get("DB", "DB_name_prefix") + \
self._config_parser.get("DEFAULT", "social_network_name") + \
self._config_parser.get("DB", "DB_name_suffix")
self._min_topic = self._config_parser.get(self.__class__.__name__, "min_topics")
self._max_topic = self._config_parser.get(self.__class__.__name__, "max_topics")
self.single_post_per_author = self._config_parser.get(self.__class__.__name__, "single_post_per_author")
self._thresh_low = self._config_parser.get(self.__class__.__name__, "thresh_low")
self._frequent_keywords_to_remove = self._config_parser.get(self.__class__.__name__,
"frequent_keywords_to_remove")
self._directory = self._autotopics_dir + "/tmp1/"
def __init__(self, db):
AbstractController.__init__(self, db)
self._working_app_number = self._config_parser.eval(self.__class__.__name__, "working_app_number")
self._maximal_get_friend_ids_requests_in_window = self._config_parser.eval(self.__class__.__name__,
"maximal_get_friend_ids_requests_in_window")
self._maximal_get_follower_ids_requests_in_window = self._config_parser.eval(self.__class__.__name__,
"maximal_get_follower_ids_requests_in_window")
self._maximal_get_user_requests_in_window = self._config_parser.eval(self.__class__.__name__,
"maximal_get_user_requests_in_window")
self._maximal_user_ids_allowed_in_single_get_user_request = self._config_parser.eval(self.__class__.__name__,
"maximal_user_ids_allowed_in_single_get_user_request")
self._num_of_twitter_status_id_requests_without_checking = self._config_parser.eval(self.__class__.__name__,
"num_of_twitter_status_id_requests_without_checking")
self._num_of_twitter_timeline_requests_without_checking = self._config_parser.eval(self.__class__.__name__,
"num_of_twitter_timeline_requests_without_checking")
self._max_tweet_ids_allowed_in_single_get_tweets_by_tweet_ids_request = self._config_parser.eval(
self.__class__.__name__,
"max_tweet_ids_allowed_in_single_get_tweets_by_tweet_ids_request")
self._max_num_of_tweet_ids_requests_without_checking = self._config_parser.eval(self.__class__.__name__,
"max_num_of_tweet_ids_requests_without_checking")
self._num_of_get_friend_ids_requests = 0
self._num_of_get_follower_ids_requests = 0
self._num_of_get_timeline_statuses = 0
self._num_of_twitter_status_id_requests = 0
self._num_of_twitter_timeline_requests = 0
self._num_of_get_tweet_ids_requests = 0
self._total_author_connections = []
print("Creating TwitterApiRequester")
self._twitter_api_requester = TwitterApiRequester()
# self._find_source_twitter_id()
logging.info("Setup DB...")
print("Setup DB...")
self._db = DB()
self._db.setUp()
def __init__(self, db):
AbstractController.__init__(self, db)
self._set_type_affiliation = 'set_affiliation'
self._targeted_class_dict = self._config_parser.eval(
self.__class__.__name__, "targeted_class_dict")
self._targeted_class_field_names = self._config_parser.eval(
self.__class__.__name__, "targeted_class_field_names")
self._classifier_type_names = self._config_parser.eval(
self.__class__.__name__, "classifier_type_names")
self._num_of_features = self._config_parser.eval(
self.__class__.__name__, "num_of_features")
self._indentifier_field_name = self._config_parser.get(
self.__class__.__name__, "indentifier_field_name")
self._replace_missing_values = self._config_parser.get(
self.__class__.__name__, "replace_missing_values")
self._selected_features = self._config_parser.eval(
self.__class__.__name__, "selected_features")
self._feature_names_to_remove = self._config_parser.eval(
self.__class__.__name__, "feature_names_to_remove")
self._order_of_results_dictionary = self._config_parser.eval(
self.__class__.__name__, "order_of_results_dictionary")
self._results_file_name = self._config_parser.get(
self.__class__.__name__, "results_file_name")
self._results_table_file_name = self._config_parser.get(
self.__class__.__name__, "results_table_file_name")
self._path = self._config_parser.get(self.__class__.__name__, "path")
self._column_names_for_results_table = self._config_parser.eval(
self.__class__.__name__, "column_names_for_results_table")
self._full_path_model_directory = self._config_parser.get(
self.__class__.__name__, "full_path_model_directory")
self._is_divide_to_training_and_test_sets_by_field_name_then_train_and_evaluate = self._config_parser.eval(
self.__class__.__name__,
"is_divide_to_training_and_test_sets_by_field_name_then_train_and_evaluate"
)
self._is_divide_to_training_and_test_sets_by_k_fold_cross_validation_then_train_and_evaluate = self._config_parser.eval(
self.__class__.__name__,
"is_divide_to_training_and_test_sets_by_k_fold_cross_validation_then_train_and_evaluate"
)
self._k_for_fold = self._config_parser.eval(self.__class__.__name__,
"k_for_fold")
self._training_label = "training"
self._test_label = "test"
def __init__(self, db, targeted_class_name):
AbstractController.__init__(self, db)
self._default_cols = [
'author_guid', 'attribute_name', 'attribute_value', 'author_type'
]
self.optional_classes = ['bad_actor', 'good_actor']
self._fill_empty = self._config_parser.eval(self.__class__.__name__,
"fill_empty")
self._to_replace_to_numerals = self._config_parser.eval(
self.__class__.__name__, "to_replace_to_numerals")
self._to_replace_authors_guid_author_name_to_authors_guid = False
self._target_class_name = targeted_class_name
self._author_guid = self._config_parser.eval(self.__class__.__name__,
"id_field")
self._remove_features_by_prefix = self._config_parser.eval(
self.__class__.__name__, "remove_features_by_prefix")
self._select_features_by_prefix = self._config_parser.eval(
self.__class__.__name__, "select_features_by_prefix")
def __init__(self, db):
AbstractController.__init__(self, db)
self._db_path = self._config_parser.get(self.__class__.__name__,
"DB_path").replace('\\', '/')
self._db_name = self._config_parser.get(self.__class__.__name__, "DB_name_prefix") + \
self._config_parser.get("DEFAULT", "social_network_name") + \
self._config_parser.get(self.__class__.__name__, "DB_name_suffix")
self._single_post_per_author = self._config_parser.get(
self.__class__.__name__, "single_post_per_author")
self._lda_model_path = self._config_parser.get(self.__class__.__name__,
"lda_model_path")
self._k = self._config_parser.get(self.__class__.__name__, "k")
self._script_path = self._config_parser.get(self.__class__.__name__,
"script_path")
self._script_name = 'autotopic_wrapper.R'
def __init__(self, db, **kwargs):
AbstractController.__init__(self, db)
self._input_path = self._config_parser.eval(self.__class__.__name__,
"input_path")
self._topic_statistics_file = self._config_parser.eval(
self.__class__.__name__, "topic_statistics_file")
self._post_id_topic_id_dict_file = self._config_parser.eval(
self.__class__.__name__, "post_id_topic_id_dict_file")
self._classifier_name = self._config_parser.eval(
self.__class__.__name__, "classifier_name")
#self._dataset_name = self._config_parser.eval(self.__class__.__name__, "dataset_name")
self._selected_features_file = self._config_parser.eval(
self.__class__.__name__, "selected_features_file")
self._features_extracted_by_tuple = self._config_parser.eval(
self.__class__.__name__, "features_extracted_by_tuple")
self._aggregated_features = self._config_parser.eval(
self.__class__.__name__, "aggregated_features")
self._prefix = self.__class__.__name__
def __init__(self, db):
AbstractController.__init__(self, db)
self._remove_stopwords = self._config_parser.eval(
self.__class__.__name__, "remove_stopwords")
self._apply_stemming = self._config_parser.eval(
self.__class__.__name__, "apply_stemming")
self._stopwords_file = self._config_parser.get(self.__class__.__name__,
"stopwords_file")
self._stemming_lang = self._config_parser.get(self.__class__.__name__,
"stemming_language")
self._is_preprocess_posts = self._config_parser.eval(
self.__class__.__name__, "is_preprocess_posts")
self._is_preprocess_authors = self._config_parser.eval(
self.__class__.__name__, "is_preprocess_authors")
self._max_objects_without_saving = self._config_parser.eval(
self.__class__.__name__, "max_objects_without_saving")
pattern_regex = "[A-Za-z']+".decode('utf-8')
self.compiled_regex = re.compile(pattern_regex)
def __init__(self, db):
AbstractController.__init__(self, db)
self._graph_types = self._config_parser.eval(self.__class__.__name__,
"graph_types")
self._number_iterations = self._config_parser.eval(
self.__class__.__name__, "number_iterations")
negative_link_method = self._config_parser.eval(
self.__class__.__name__, "negative_link_method")
percent_of_links_to_sample = self._config_parser.eval(
self.__class__.__name__, "percent_of_links_to_sample")
measure_names = self._config_parser.eval(self.__class__.__name__,
"measure_names")
self._combinations = list(
itertools.product(negative_link_method, percent_of_links_to_sample,
measure_names))
self._author_features = []
def __init__(self, db):
AbstractController.__init__(self, db)
self._tsv_input_files_path = self._config_parser.get(
self.__class__.__name__, "tsv_input_files")
self._tsv_authors_file_name = self._config_parser.get(
self.__class__.__name__, "tsv_authors_file_name")
self._tsv_posts_file_name = self._config_parser.get(
self.__class__.__name__, "tsv_posts_file_name")
self._tsv_author_connections_file_name = self._config_parser.get(
self.__class__.__name__, "tsv_author_connections_file_name")
self._actions = self._config_parser.eval(self.__class__.__name__,
"actions")
self._authors_header = self._config_parser.eval(
self.__class__.__name__, "authors_header")
self._posts_header = self._config_parser.eval(self.__class__.__name__,
"posts_header")
self._author_connections_header = self._config_parser.eval(
self.__class__.__name__, "author_connections_header")
self._url_index = 1
def __init__(self, db, **kwargs):
AbstractController.__init__(self, db)
self._features_list = self._config_parser.eval(self.__class__.__name__,
"features_list")
self._prefix = self.__class__.__name__
self._num_of_terms_in_topic = self._config_parser.eval(
self.__class__.__name__, "num_of_terms_in_topic")
self.num_topics = self._config_parser.eval(self.__class__.__name__,
"number_of_topics")
self.stopword_file = self._config_parser.get(self.__class__.__name__,
"stopword_file")
self.stemlanguage = self._config_parser.get(self.__class__.__name__,
"stem_language")
self.topics = []
self.topic = None
self.post_id_to_topic_id = None
self.topic_id_to_topics = {}
self.model = None
self.dictionary = None
self.corpus = None
def __init__(self, db):
AbstractController.__init__(self, db)
self._k = self._config_parser.eval(self.__class__.__name__, "k")
self._num_of_iterations = int(self._config_parser.get(self.__class__.__name__, "num_of_iterations"))
self._index_field_for_predictions = self._config_parser.get(self.__class__.__name__, "index_field_for_predictions")
self._connection_types = self._config_parser.get(self.__class__.__name__, "connection_type")
self._desicion_models = self._config_parser.eval(self.__class__.__name__, "decision_models")
self._result_authors_file_name = self._config_parser.get(self.__class__.__name__, "result_author_file_name")
self._result_classifier_file_name = self._config_parser.get(self.__class__.__name__, "result_classifier_file_name")
self._train_percent = int(self._config_parser.get(self.__class__.__name__,"train_percent"))
self._randomly_classify_unlabelled = self._config_parser.eval(self.__class__.__name__, "randomly_clasify_unlabled")
self._classification_options = self._config_parser.eval(self.__class__.__name__, "clasification_options")
self._targeted_class = self._config_parser.get(self.__class__.__name__, "targeted_class")
self._similarity_function = self._config_parser.get(self.__class__.__name__, "similarity_function")
self._update_targeted_class_in_db = self._config_parser.eval(self.__class__.__name__, "update_targeted_class_in_db")
self._random_seed =int(self._config_parser.get(self.__class__.__name__,"random_seed"))
self._connections = []
self._result_dataframe_of_author = None
self._result_dataframe_of_classifier = None
self._data_holder = None
def __init__(self, db):
AbstractController.__init__(self, db)
self._results_path = self._config_parser.get(self.__class__.__name__,
"results_path")
self._predictions_per_iteration_path = self._config_parser.get(
self.__class__.__name__, "predictions_per_iteration_path")
self._similarity_functions = self._config_parser.eval(
self.__class__.__name__, "similarity_functions")
self._k = self._config_parser.eval(self.__class__.__name__, "k")
self._compute_knn_based_on_link_prediction = self._config_parser.eval(
self.__class__.__name__, "compute_knn_based_on_lp")
self._link_prediction_models = self._config_parser.eval(
self.__class__.__name__, "link_prediction_models")
self._results_averaged_on_report = self._config_parser.eval(
self.__class__.__name__, "results_averaged_on_report")
self._generate_anchors = self._config_parser.eval(
self.__class__.__name__, "generate_anchors")
self._num_iterations = self._config_parser.eval(
self.__class__.__name__, "num_iterations")
self._targeted_class_dict = self._config_parser.eval(
self.__class__.__name__, "targeted_class_dict")
self._decision_models = self._config_parser.eval(
self.__class__.__name__, "decision_models")
self._targeted_class_field_name = self._config_parser.eval(
self.__class__.__name__, "targeted_class_field_name")
self._order_of_results_dictionary = self._config_parser.eval(
self.__class__.__name__, "order_of_results_dictionary")
self._results_table_file_name = self._config_parser.get(
self.__class__.__name__, "results_table_file_name")
self._path = self._config_parser.get(self.__class__.__name__, "path")
self._column_names_for_results_table = self._config_parser.eval(
self.__class__.__name__, "column_names_for_results_table")
self._index_field_for_predictions = self._config_parser.get(
self.__class__.__name__, "index_field_for_predictions")
self._targeted_class_anchors_percent_size = self._config_parser.eval(
self.__class__.__name__, "targeted_class_anchors_percent_size")
self._targeted_class_num_of_authors_dict = self._config_parser.eval(
self.__class__.__name__, "targeted_class_num_of_authors_dict")
def __init__(self, db):
AbstractController.__init__(self, db)
self._actions = self._config_parser.eval(self.__class__.__name__,
"actions")
self._minimal_num_of_posts = self._config_parser.eval(
self.__class__.__name__, "minimal_num_of_posts")
self._limit_friend_follower_number = self._config_parser.eval(
self.__class__.__name__, "limit_friend_follower_number")
self._maximal_tweets_count_in_timeline = self._config_parser.eval(
self.__class__.__name__, "maximal_tweets_count_in_timeline")
self._vico_importer_twitter_authors = []
self._found_twitter_users = []
self._social_network_crawler = Twitter_Rest_Api(db)
self._suspended_authors = []
self._max_users_without_saving = self._config_parser.eval(
self.__class__.__name__, "max_users_without_saving")
self._output_path = self._config_parser.eval(self.__class__.__name__,
"output_path")
self._posts = []
self._authors = []
self._post_citatsions = []
def __init__(self, db):
AbstractController.__init__(self, db)
self._actions = self._config_parser.eval(self.__class__.__name__,
"actions")
self._output_path = self._config_parser.eval(self.__class__.__name__,
"output_path")
self._fake_news_labels = self._config_parser.eval(
self.__class__.__name__, "fake_news_labels")
self._real_news_labels = self._config_parser.eval(
self.__class__.__name__, "real_news_labels")
self._min_num_of_distinct_claims_author_involved = self._config_parser.eval(
self.__class__.__name__,
"min_num_of_distinct_claims_author_involved")
self._min_num_of_published_posts = self._config_parser.eval(
self.__class__.__name__, "min_num_of_published_posts")
self._threshold = self._config_parser.eval(self.__class__.__name__,
"threshold")
self._original_fake_news_label = u"fake_news"
self._original_real_news_label = u"real_news"
self._fake_news_promoter_label = u"fake_news_promoter"
self._real_news_promoter_label = u"real_news_promoter"
def __init__(self, db):
AbstractController.__init__(self, db)
self._target_field = self._config_parser.eval(self.__class__.__name__,
"target_field")
self._data_handler = Data_Handler(
db, targeted_class_name=self._target_field)
self._classifier_file_path_and_name = self._config_parser.eval(
self.__class__.__name__, "classifier_file_path_and_name")
self._selected_feature_file_path_and_name = self._config_parser.eval(
self.__class__.__name__, "selected_feature_file_path_and_name")
self._saved_prediction_file_path_and_name = self._config_parser.eval(
self.__class__.__name__, "saved_prediction_file_path_and_name")
self._fill_empty = self._config_parser.eval(self.__class__.__name__,
"fill_empty")
self._remove_features = self._config_parser.eval(
self.__class__.__name__, "remove_features")
self._select_features = self._config_parser.eval(
self.__class__.__name__, "select_features")
self._label_text_to_value = self._config_parser.eval(
self.__class__.__name__, "label_text_to_value")
self._classifier = self._load_file(self._classifier_file_path_and_name)
def __init__(self, db, **kwargs):
AbstractController.__init__(self, db)
self.config_parser = getConfig()
self._pipeline = []
def __init__(self, db):
AbstractController.__init__(self, db)
self._input_path = self._config_parser.eval(self.__class__.__name__,
'input_path')
self._connection_type = self._config_parser.eval(
self.__class__.__name__, 'type')
def __init__(self, db):
AbstractController.__init__(self, db)
self._actions = self._config_parser.eval(self.__class__.__name__, "actions")
self._json_path = self._config_parser.eval(self.__class__.__name__, "json_path")
def __init__(self, db):
AbstractController.__init__(self, db)
configInst = getConfig()
def execute(self, window_start):
AbstractController.execute(self, window_start)
self._db.deleteBoostAuth(self._window_start)
curr_posts = self._db.getPostsListWithoutEmptyRowsByDate(self._window_start, self._window_end)
stats_calc = AuthorStatisticsCalculator(self._db)
authors_statistics, post_to_pointers_scores = stats_calc.run_authors_stats_calculation(curr_posts)
authors_boost_stats_list = []
total = len(authors_statistics)
current = 0
for author, stats in authors_statistics.iteritems():
current += 1
print "boost authors model... processing author_boost_stats: " + str(current) + " out of " + str(total)
if (author):
found_author = self._db.getAuthorByName(author)
# We change this line for authors which have the same guid, but different names so some of them not found by the given author name.
# This is a known bug in VICO.
if len(found_author) > 0:
authorDomain = found_author[0].domain
author_guid = found_author[0].author_guid
author_boost_stats = Author_boost_stats(window_start=self._window_start,
window_end=self._window_end, author_name=author,
author_domain=authorDomain,
boosting_timeslots_participation_count=stats.boosting_timeslots_participation_count,
count_of_authors_sharing_boosted_posts=stats.count_of_authors_sharing_boosted_keyposts,
num_of_pointers=stats.num_of_pointers,
num_of_pointed_posts=stats.num_of_pointed_keyposts,
pointers_scores=stats.pointers_scores,
scores_sum=stats.scores_sum, scores_avg=stats.scores_avg,
scores_std=stats.scores_std, author_guid=author_guid)
authors_boost_stats_list.append(author_boost_stats)
authors_boost_stats_list = list(set(authors_boost_stats_list))
self._db.addAuthors_boost_stats(authors_boost_stats_list)
post_to_pointers_scores_list = []
duplicatesList = []
total_post_to_pointers = len(post_to_pointers_scores)
current_post_to_pointers = 0
for post, pointers_scores in post_to_pointers_scores.iteritems():
current_post_to_pointers += 1
print "boost authors model... processing post to pointers : " + str(
current_post_to_pointers) + " out of " + str(total_post_to_pointers)
total_pointer_scores = len(pointers_scores)
current_pointer_scores = 0
for pointer, score in pointers_scores.iteritems():
current_pointer_scores += 1
print "boost authors model... processing pointer scores: " + str(
current_pointer_scores) + " out of " + str(total_pointer_scores)
temp = []
post_data = self._db.getPostUsingURL(post, self._window_start, self._window_end)
if (len(post_data) > 0):
post_id = post_data[0].post_id
found_author = self._db.getAuthorByName(pointer.author)
# We change this line for authors which have the same guid, but different names so some of them not found by the given author name.
# This is a known bug in VICO.
if len(found_author) > 0:
authorDomain = found_author[0].domain
temp.extend([post_id, self._window_start, self._window_end, post, authorDomain, pointer.author,
str_to_date(pointer.date)])
if temp not in duplicatesList:
duplicatesList.append(temp)
post_to_pointers = Post_to_pointers_scores(author_name=pointer.author,
author_domain=authorDomain, post_id_to=post_id,
window_start=self._window_start,
window_end=self._window_end, url_to=post,
datetime=str_to_date(pointer.date),
pointer_score=score)
post_to_pointers_scores_list.append(post_to_pointers)
self._db.addPosts_to_pointers_scores(post_to_pointers_scores_list)
def __init__(self, db, **kwargs):
AbstractController.__init__(self, db)
self.config_parser = getConfig()
self._pipeline = []
self._load_parameters = self._config_parser.eval(
self.__class__.__name__, "load_parameters")
def __init__(self, db):
AbstractController.__init__(self, db)
self._min_number_of_posts_per_author = self._config_parser.eval(
self.__class__.__name__, "min_number_of_posts_per_author")
self._random_authors_requirements = self._config_parser.eval(
self.__class__.__name__, "random_authors_requirements")
def __init__(self, db, **kwargs):
AbstractController.__init__(self, db)
self._features_list = self._config_parser.eval(self.__class__.__name__, "features_list")
self._good_claim_type = self._config_parser.eval(self.__class__.__name__, "good_claim_type")
self._bad_claim_type = self._config_parser.eval(self.__class__.__name__, "bad_claim_type")
self._prefix = self.__class__.__name__
def __init__(self, db):
AbstractController.__init__(self, db)
self._path_for_downloaded_images = self._config_parser.eval(
self.__class__.__name__, "path_for_downloaded_images")
def __init__(self, db):
AbstractController.__init__(self, db)
self._images_folder = self._config_parser.eval(self.__class__.__name__,
"images_folder")
def __init__(self, db):
AbstractController.__init__(self, db)
def __init__(self, db):
AbstractController.__init__(self, db)
self._path_for_downloaded_images = self._config_parser.eval(
self.__class__.__name__, "path_for_downloaded_images")
self._source_target_images_to_download_dicts = self._config_parser.eval(
self.__class__.__name__, "source_target_images_to_download_dicts")