def __init__(self, db):
AbstractController.__init__(self, db)
self._trained_classifier_file_name = self._config_parser.eval(
self.__class__.__name__, "trained_classifier_file_name")
self._best_features_file_name = self._config_parser.eval(
self.__class__.__name__, "best_features_file_name")
self._full_path_trained_classifier_directory = self._config_parser.eval(
self.__class__.__name__, "full_path_trained_classifier_directory")
self._targeted_class_field_names = self._config_parser.eval(
self.__class__.__name__, "targeted_class_field_names")
self._replace_missing_values = self._config_parser.eval(
self.__class__.__name__, "replace_missing_values")
self._indentifier_field_name = self._config_parser.eval(
self.__class__.__name__, "indentifier_field_name")
self._feature_names_to_remove = self._config_parser.eval(
self.__class__.__name__, "feature_names_to_remove")
self._selected_features = self._config_parser.eval(
self.__class__.__name__, "selected_features")
self._classifier_type_name = self._config_parser.eval(
self.__class__.__name__, "classifier_type_name")
self._num_of_features = self._config_parser.eval(
self.__class__.__name__, "num_of_features")
self._targeted_class_dict = self._config_parser.eval(
self.__class__.__name__, "targeted_class_dict")
self._path = self._config_parser.eval(self.__class__.__name__, "path")
def __init__(self, db):
AbstractController.__init__(self, db)
self._target_field = self._config_parser.eval(self.__class__.__name__,
"target_field")
self._select_features = self._config_parser.eval(
self.__class__.__name__, "select_features")
self._remove_features = self._config_parser.eval(
self.__class__.__name__, "remove_features")
self._k = self._config_parser.eval(self.__class__.__name__, "k")
self._num_of_features_to_select = self._config_parser.eval(
self.__class__.__name__, "num_of_features_to_select")
self._classifiers_with_parameters_dict = self._config_parser.eval(
self.__class__.__name__, "classifiers_with_parameters_dict")
self._compare_matics_by_order = self._config_parser.eval(
self.__class__.__name__, "compare_matrics_by_order")
self._label_text_to_value = self._config_parser.eval(
self.__class__.__name__, "label_text_to_value")
self._results_file_path = "data/output/expermintal_environment/classification_results_refactored.csv"
self._feature_importance_file = 'data\\output\\expermintal_environment\\feature_importance.csv'
self._saved_classifier_path = self._config_parser.eval(
self.__class__.__name__, "saved_classifier_path")
self._results_dictionary = {}
self._classifiers_by_name = {}
self._data_handler = Data_Handler(
db, targeted_class_name=self._target_field)
def __init__(self, db):
AbstractController.__init__(self, db)
self._kernels = self._config_parser.eval(self.__class__.__name__,
"kernels")
self._index_field_for_predictions = self._config_parser.get(
self.__class__.__name__, "index_field_for_predictions")
self._path = self._config_parser.get(self.__class__.__name__, "path")
def __init__(self, db):
AbstractController.__init__(self, db)
nltk.download('stopwords')
self._num_of_terms_in_topic = self._config_parser.eval(
self.__class__.__name__, "num_of_terms_in_topic")
self.num_topics = self._config_parser.eval(self.__class__.__name__,
"number_of_topics")
self.stopword_file = self._config_parser.get(self.__class__.__name__,
"stopword_file")
self.stemlanguage = self._config_parser.get(self.__class__.__name__,
"stem_language")
self._removed_keywords_file = self._config_parser.eval(
self.__class__.__name__, "removed_keywords_file")
self._post_topic_probability_table_path = self._config_parser.eval(
self.__class__.__name__, "post_topic_probability_table_path")
self._to_print_top_10_terms = self._config_parser.eval(
self.__class__.__name__, "to_print_top_10_terms")
self._output_file = self._config_parser.eval(self.__class__.__name__,
"output_path")
self.topics = []
self.topic = None
self.post_id_to_topic_id = None
self.topic_id_to_topics = {}
self.model = None
self.dictionary = None
self.corpus = None
self._stopwords = set(stopwords.words('english')) | set_of_stopwords(
self._removed_keywords_file)
self._english_words = set(nltk.corpus.words.words())
def __init__(self, db):
AbstractController.__init__(self, db)
self._read_predictions_from_db = self._config_parser.eval(
self.__class__.__name__, "read_predictions_from_db")
self.prediction_csv_path = self._config_parser.eval(
self.__class__.__name__, "prediction_csv_path")
self._viz_output_path = self._config_parser.eval(
self.__class__.__name__, "output_dir")
self._buckets = self._config_parser.eval(self.__class__.__name__,
"buckets")
self._target_author_field = self._config_parser.eval(
self.__class__.__name__, "target_author_field")
self._include_unlabeled_predictions = self._config_parser.eval(
self.__class__.__name__, "include_unlabeled_predictions")
self._include_labeled_authors_in_visualization = self._config_parser.eval(
self.__class__.__name__,
"include_labeled_authors_in_visualization")
self._optional_classes = self._config_parser.eval(
self.__class__.__name__, "optional_classes")
self._extend_posts_by_retweets = self._config_parser.eval(
self.__class__.__name__, "extend_posts_by_retweets")
# if you have problem with config - this is it
# font = "topic_distribution_visualization/Mukadimah.ttf"
self._font_path = self._config_parser.eval(self.__class__.__name__,
"font_path")
self.include_topic_labels = self._config_parser.eval(
self.__class__.__name__, "include_topic_labels")
self.claim_to_topic_connection_json = self._config_parser.eval(
self.__class__.__name__, "claim_to_topic_connection_json")
self.topic_classification_table_args = self._config_parser.eval(
self.__class__.__name__, "topic_classification_table_args")
self._topic_to_type = {}
def __init__(self, db, **kwargs):
AbstractController.__init__(self, db)
self._db = db
self._word_vector_dict = {}
self._table_name = self._config_parser.eval(self.__class__.__name__, "table_name")
self._aggregation_functions = self._config_parser.eval(self.__class__.__name__, "aggregation_functions")
self._pairs_targets = self._config_parser.eval(self.__class__.__name__, "pairs_targets")
self._distance_functions = self._config_parser.eval(self.__class__.__name__, "distance_functions")
def __init__(self, db):
AbstractController.__init__(self, db)
self.threshold = self._config_parser.eval(self.__class__.__name__,
"threshold")
self.output_file_path = self._config_parser.eval(
self.__class__.__name__, "output_file_path")
self.output_file_path = self.output_file_path.format(
datetime.datetime.today().strftime('%d-%m-%Y'), self.threshold)
def __init__(self, db, **kwargs):
AbstractController.__init__(self, db)
self._source_list = self._config_parser.eval(self.__class__.__name__,
"source_list")
self._features_list = self._config_parser.eval(self.__class__.__name__,
"features_list")
self._delta_time = self._config_parser.eval(self.__class__.__name__,
"delta_time")
self._prefix = self.__class__.__name__
def __init__(self, db, **kwargs):
AbstractController.__init__(self, db)
self._features_list = self._config_parser.eval(self.__class__.__name__,
"features_list")
self._good_claim_type = self._config_parser.eval(
self.__class__.__name__, "good_claim_type")
self._bad_claim_type = self._config_parser.eval(
self.__class__.__name__, "bad_claim_type")
self._prefix = self.__class__.__name__
def __init__(self, db):
AbstractController.__init__(self, db)
self._input_path = self._config_parser.eval(self.__class__.__name__,
"input_path")
self._output_path = self._config_parser.eval(self.__class__.__name__,
"output_path")
self._keywords_file_name = self._config_parser.eval(
self.__class__.__name__, "keywords_file_name")
def __init__(self, db): #TODO: no kwargs in module constructors
AbstractController.__init__(self, db)
self._connection_type = unicode(self._config_parser.get(self.__class__.__name__, "connection_type"))
self._max_objects_without_saving = self._config_parser.eval(self.__class__.__name__, "max_objects_without_saving")
self._min_number_of_posts_per_author = self._config_parser.eval(self.__class__.__name__, "min_number_of_posts_per_author")
self._num_of_random_authors_for_graph = self._config_parser.eval(self.__class__.__name__, "num_of_random_authors_for_graph")
self._author_connections_edges = []
# will save the connection and the reversed in order to make this connection type as un directed.
self._existing_connections_dict = defaultdict()
def __init__(self, db):
AbstractController.__init__(self, db)
self.normalize = self._config_parser.get(self.__class__.__name__,
"normalize")
self._all_authors_features = self._config_parser.eval(
self.__class__.__name__, 'all_authors')
self._default_cols = [
'author_guid', 'attribute_name', 'attribute_value'
]
self._author_features_data_frame = {}
def __init__(self, db):
AbstractController.__init__(self, db)
self._k = self._config_parser.eval(self.__class__.__name__, "k")
self._index_field_for_predictions = self._config_parser.get(self.__class__.__name__, "index_field_for_predictions")
self._connection_types = self._config_parser.get(self.__class__.__name__, "connection_type")
self._desicion_models = self._config_parser.eval(self.__class__.__name__, "decision_models")
self._connections = []
self._decision_by_author_to_config = {} # test config = K to desicion function to
def __init__(self, db):
AbstractController.__init__(self, db)
self.output_filename = self._config_parser.eval(
self.__class__.__name__, "arff_file")
self._author_type_classes = self._config_parser.eval(
self.__class__.__name__, "author_type_classes")
self._target_type_attr_name = self._config_parser.eval(
self.__class__.__name__, "target_type_attr_name")
self._author_sub_type_classes = self._config_parser.eval(
self.__class__.__name__, "author_sub_type_classes")
def __init__(self, db):
AbstractController.__init__(self, db)
self._working_app_number = self._config_parser.eval(
self.__class__.__name__, "working_app_number")
self._maximal_get_friend_ids_requests_in_window = self._config_parser.eval(
self.__class__.__name__,
"maximal_get_friend_ids_requests_in_window")
self._maximal_get_follower_ids_requests_in_window = self._config_parser.eval(
self.__class__.__name__,
"maximal_get_follower_ids_requests_in_window")
self._maximal_get_user_requests_in_window = self._config_parser.eval(
self.__class__.__name__, "maximal_get_user_requests_in_window")
self._maximal_user_ids_allowed_in_single_get_user_request = self._config_parser.eval(
self.__class__.__name__,
"maximal_user_ids_allowed_in_single_get_user_request")
self._num_of_twitter_status_id_requests_without_checking = self._config_parser.eval(
self.__class__.__name__,
"num_of_twitter_status_id_requests_without_checking")
self._num_of_twitter_timeline_requests_without_checking = self._config_parser.eval(
self.__class__.__name__,
"num_of_twitter_timeline_requests_without_checking")
self._max_tweet_ids_allowed_in_single_get_tweets_by_tweet_ids_request = self._config_parser.eval(
self.__class__.__name__,
"max_tweet_ids_allowed_in_single_get_tweets_by_tweet_ids_request")
self._max_num_of_tweet_ids_requests_without_checking = self._config_parser.eval(
self.__class__.__name__,
"max_num_of_tweet_ids_requests_without_checking")
self._num_of_get_friend_ids_requests = 0
self._num_of_get_follower_ids_requests = 0
self._num_of_get_timeline_statuses = 0
self._num_of_twitter_status_id_requests = 0
self._num_of_twitter_timeline_requests = 0
self._num_of_get_tweet_ids_requests = 0
self._total_author_connections = []
print("Creating TwitterApiRequester")
self._twitter_api_requester = TwitterApiRequester(
self._working_app_number)
# self._find_source_twitter_id()
logging.info("Setup DB...")
print("Setup DB...")
self._db = DB()
self._db.setUp()
def __init__(self, db):
AbstractController.__init__(self, db)
self._classification_method = self._config_parser.eval(self.__class__.__name__, "classification_method")
self._path = self._config_parser.eval(self.__class__.__name__, "path")
self._good_buckets = self._config_parser.eval(self.__class__.__name__, "good_buckets")
self._bad_buckets = self._config_parser.eval(self.__class__.__name__, "bad_buckets")
self._input_topic_statistics_file_name = self._config_parser.eval(self.__class__.__name__,
"input_topic_statistics_file_name")
self._claim_id_topic_num_dict_json_file_name = self._config_parser.eval(self.__class__.__name__,
"claim_id_topic_num_dict_json_file_name")
self._targeted_classes_dict = self._config_parser.eval(self.__class__.__name__,
"targeted_classes_dict")
def __init__(self, db):
AbstractController.__init__(self, db)
self._graph_types = self._config_parser.eval(self.__class__.__name__, "graph_types")
self._number_iterations = self._config_parser.eval(self.__class__.__name__, "number_iterations")
negative_link_method = self._config_parser.eval(self.__class__.__name__, "negative_link_method")
number_of_links_to_sample = self._config_parser.eval(self.__class__.__name__, "number_of_links_to_sample")
measure_names = self._config_parser.eval(self.__class__.__name__, "measure_names")
self._combinations = list(itertools.product(negative_link_method, number_of_links_to_sample, measure_names))
self._author_features = []
def __init__(self, db, custom_targeted_class_num_of_authors_dict=None):
AbstractController.__init__(self, db)
if custom_targeted_class_num_of_authors_dict is None:
self._targeted_class_num_of_authors_dict = self._config_parser.eval(self.__class__.__name__, "targeted_class_num_of_authors_dict")
else:
self._targeted_class_num_of_authors_dict = custom_targeted_class_num_of_authors_dict
self._targeted_class_field_name = self._config_parser.eval(self.__class__.__name__, "targeted_class_field_name")
self._anchor_authors = []
self._anchor_authors_dict = {}
def __init__(self, db):
AbstractController.__init__(self, db)
self._actions = self._config_parser.eval(self.__class__.__name__, "actions")
self._targeted_twitter_author_ids = self._config_parser.eval(self.__class__.__name__, "targeted_twitter_author_ids")
self._targeted_twitter_post_ids = self._config_parser.eval(self.__class__.__name__, "targeted_twitter_post_ids")
self._targeted_twitter_author_names = self._config_parser.eval(self.__class__.__name__, "targeted_twitter_author_names")
self._social_network_crawler = Twitter_Rest_Api(db)
def __init__(self, db):
AbstractController.__init__(self, db)
self._tsv_input_files_path = self._config_parser.get(self.__class__.__name__, "tsv_input_files")
self._tsv_authors_file_name = self._config_parser.get(self.__class__.__name__, "tsv_authors_file_name")
self._tsv_posts_file_name = self._config_parser.get(self.__class__.__name__, "tsv_posts_file_name")
self._tsv_author_connections_file_name = self._config_parser.get(self.__class__.__name__, "tsv_author_connections_file_name")
self._actions = self._config_parser.eval(self.__class__.__name__, "actions")
self._authors_header = self._config_parser.eval(self.__class__.__name__, "authors_header")
self._posts_header = self._config_parser.eval(self.__class__.__name__, "posts_header")
self._author_connections_header = self._config_parser.eval(self.__class__.__name__, "author_connections_header")
self._url_index = 1
def __init__(self, db, **kwargs):
AbstractController.__init__(self, db)
self._features_list = self._config_parser.eval(self.__class__.__name__,
"features_list")
self._csv_file = self._config_parser.eval(self.__class__.__name__,
"csv_file")
self._group_by = self._config_parser.eval(self.__class__.__name__,
"group_by")
self._source = self._config_parser.eval(self.__class__.__name__,
"source")
self._target = self._config_parser.eval(self.__class__.__name__,
"target")
self._prefix = self.__class__.__name__
def __init__(self, db):
AbstractController.__init__(self, db)
self._actions = self._config_parser.eval(self.__class__.__name__,
"actions")
self._json_path = self._config_parser.eval(self.__class__.__name__,
"json_path")
self._path_bad_actor_csv_path = self._config_parser.eval(
self.__class__.__name__, "path_bad_actor_csv")
self._author_type = str(
self._config_parser.get(self.__class__.__name__, "author_type"))
self._author_dict = {}
self._authors = []
self._posts = []
def __init__(self, db):
AbstractController.__init__(self, db)
self._auth_key = self._config_parser.eval(self.__class__.__name__,
"auth_key")
self._model_id = self._config_parser.eval(self.__class__.__name__,
"model_id")
self._max_objects_without_saving = self._config_parser.eval(
self.__class__.__name__, "max_objects_without_saving")
self._max_request_to_service = self._dataset_folder = self._config_parser.eval(
self.__class__.__name__, "max_request_to_service")
self._dataset_folder = self._config_parser.eval(
self.__class__.__name__, "dataset_folder")
self.min_value_proba = self._config_parser.eval(
self.__class__.__name__, "min_value_proba")
def __init__(self, db):
AbstractController.__init__(self, db)
self._keywords = self._config_parser.eval(self.__class__.__name__,
"keywords")
self._specific_site = str(
self._config_parser.get(self.__class__.__name__, "specific_site"))
self._specific_language = str(
self._config_parser.get(self.__class__.__name__,
"specific_language"))
self._maximal_returned_records = self._config_parser.get(
self.__class__.__name__, "maximal_returned_records")
self._prefix = "http://api.gdeltproject.org/api/v1/search_ftxtsearch/search_ftxtsearch?&"
self._api_parameters_seperator = '%20'
def __init__(self, db):
AbstractController.__init__(self, db)
self._config_parser = getConfig()
self._path = self._config_parser.get(self.__class__.__name__, "path")
self._backup_path = self._config_parser.get(self.__class__.__name__,
"backup_path")
self._bad_actors_file_name = self._config_parser.get(
self.__class__.__name__, "bad_actors_file_name")
self._potential_good_actors_file_name = self._config_parser.get(
self.__class__.__name__, "potential_good_actors_file_name")
self._csv_header = self._config_parser.eval(self.__class__.__name__,
"csv_header")
self._vico_importer_bad_actors = []
self._csv_importer = PostCSVExporter()
def __init__(self, db):
AbstractController.__init__(self, db)
self._script_name = 'autotopic_model_creation.R'
self._autotopics_dir = self._config_parser.get(self.__class__.__name__, "autotopics_dir")
self._db_name = self._config_parser.get("DB", "DB_name_prefix") + \
self._config_parser.get("DEFAULT", "social_network_name") + \
self._config_parser.get("DB", "DB_name_suffix")
self._min_topic = self._config_parser.get(self.__class__.__name__, "min_topics")
self._max_topic = self._config_parser.get(self.__class__.__name__, "max_topics")
self.single_post_per_author = self._config_parser.get(self.__class__.__name__, "single_post_per_author")
self._thresh_low = self._config_parser.get(self.__class__.__name__, "thresh_low")
self._frequent_keywords_to_remove = self._config_parser.get(self.__class__.__name__,
"frequent_keywords_to_remove")
self._directory = self._autotopics_dir + "/tmp1/"
def __init__(self, db):
AbstractController.__init__(self, db)
self.necessary_columns = self._config_parser.eval(
self.__class__.__name__, 'columns')
self._snopes_strainer = SoupStrainer("li",
{"class": "breadcrumb-item"})
self._header = {
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
'Accept-Encoding': 'none',
'Accept-Language': 'en-US,en;q=0.8',
'Connection': 'keep-alive'
}
def __init__(self, db):
AbstractController.__init__(self, db)
self._set_type_affiliation = 'set_affiliation'
self._targeted_class_dict = self._config_parser.eval(
self.__class__.__name__, "targeted_class_dict")
self._targeted_class_field_names = self._config_parser.eval(
self.__class__.__name__, "targeted_class_field_names")
self._classifier_type_names = self._config_parser.eval(
self.__class__.__name__, "classifier_type_names")
self._num_of_features = self._config_parser.eval(
self.__class__.__name__, "num_of_features")
self._indentifier_field_name = self._config_parser.get(
self.__class__.__name__, "indentifier_field_name")
self._replace_missing_values = self._config_parser.get(
self.__class__.__name__, "replace_missing_values")
self._selected_features = self._config_parser.eval(
self.__class__.__name__, "selected_features")
self._feature_names_to_remove = self._config_parser.eval(
self.__class__.__name__, "feature_names_to_remove")
self._order_of_results_dictionary = self._config_parser.eval(
self.__class__.__name__, "order_of_results_dictionary")
self._results_file_name = self._config_parser.get(
self.__class__.__name__, "results_file_name")
self._results_table_file_name = self._config_parser.get(
self.__class__.__name__, "results_table_file_name")
self._path = self._config_parser.get(self.__class__.__name__, "path")
self._column_names_for_results_table = self._config_parser.eval(
self.__class__.__name__, "column_names_for_results_table")
self._full_path_model_directory = self._config_parser.get(
self.__class__.__name__, "full_path_model_directory")
self._is_divide_to_training_and_test_sets_by_field_name_then_train_and_evaluate = self._config_parser.eval(
self.__class__.__name__,
"is_divide_to_training_and_test_sets_by_field_name_then_train_and_evaluate"
)
self._is_divide_to_training_and_test_sets_by_k_fold_cross_validation_then_train_and_evaluate = self._config_parser.eval(
self.__class__.__name__,
"is_divide_to_training_and_test_sets_by_k_fold_cross_validation_then_train_and_evaluate"
)
self._k_for_fold = self._config_parser.eval(self.__class__.__name__,
"k_for_fold")
self._training_label = "training"
self._test_label = "test"
def __init__(self, db):
AbstractController.__init__(self, db)
self._db_path = self._config_parser.get(self.__class__.__name__,
"DB_path").replace('\\', '/')
self._db_name = self._config_parser.get(self.__class__.__name__, "DB_name_prefix") + \
self._config_parser.get("DEFAULT", "social_network_name") + \
self._config_parser.get(self.__class__.__name__, "DB_name_suffix")
self._single_post_per_author = self._config_parser.get(
self.__class__.__name__, "single_post_per_author")
self._lda_model_path = self._config_parser.get(self.__class__.__name__,
"lda_model_path")
self._k = self._config_parser.get(self.__class__.__name__, "k")
self._script_path = self._config_parser.get(self.__class__.__name__,
"script_path")
self._script_name = 'autotopic_wrapper.R'
def __init__(self, db, targeted_class_name):
AbstractController.__init__(self, db)
self._default_cols = [
'author_guid', 'attribute_name', 'attribute_value', 'author_type'
]
self.optional_classes = ['bad_actor', 'good_actor']
self._fill_empty = self._config_parser.eval(self.__class__.__name__,
"fill_empty")
self._to_replace_to_numerals = self._config_parser.eval(
self.__class__.__name__, "to_replace_to_numerals")
self._to_replace_authors_guid_author_name_to_authors_guid = False
self._target_class_name = targeted_class_name
self._author_guid = self._config_parser.eval(self.__class__.__name__,
"id_field")
self._remove_features_by_prefix = self._config_parser.eval(
self.__class__.__name__, "remove_features_by_prefix")
self._select_features_by_prefix = self._config_parser.eval(
self.__class__.__name__, "select_features_by_prefix")