def __init__(self): self.data_file = app_config['data_file'] self.texts_in_file = 'texts_in_file.txt' self.ner_texts_file = 'output.txt' self.utilities = Utilities() self.lemmatizer = WordNetLemmatizer() self.preprocessor = Preprocessor( ['remove_urls', 'remove_mentions', 'remove_hashtags', 'normalize'])
def __init__(self): self.path_to_jar = 'lib/stanford_parser/stanford-parser.jar' self.path_to_models_jar = 'lib/stanford_parser/stanford-english-corenlp-2018-02-27-models.jar' self.path_to_ner_tagger = 'lib/stanford_ner/stanford-ner.jar' self.path_to_ner_model = 'lib/stanford_ner/english.all.3class.distsim.crf.ser.gz' self.ner_tagger = StanfordNERTagger(self.path_to_ner_model, self.path_to_ner_tagger) self.dependency_parser = StanfordDependencyParser( path_to_jar=self.path_to_jar, path_to_models_jar=self.path_to_models_jar) self.lemmatizer = WordNetLemmatizer() self.utilities = Utilities()
def __init__(self, params=list()): self.remove_urls = True if 'remove_urls' in params else False self.remove_mentions = True if 'remove_mentions' in params else False self.remove_hashtags = True if 'remove_hashtags' in params else False self.normalize = True if 'normalize' in params else False self.remove_stopwords = True if 'remove_stopwords' in params else False self.remove_punct = True if 'remove_punctuation' in params else False self.lower = True if 'lower' in params else False self.lemmatize = True if 'lemmatize' in params else False self.stemming = True if 'stemming' in params else False self.remove_non_letters = True if 'remove_non_letters' in params else False self.lemmatizer = WordNetLemmatizer() self.stemmer = PorterStemmer() self.utilities = Utilities()
def get_evaluation_data(self, dataset_file, n_pair): utilities = Utilities() preprocessor = Preprocessor( ['remove_stopwords', 'remove_non_letters', 'lemmatize']) data_rows = utilities.read_from_csv(dataset_file) del data_rows[0] X = [] y = [] for data_row in data_rows[:n_pair]: candidate_causal_pair = eval(data_row[2]) label = 1 if data_row[3] == 'causal' else 0 candidate_causal_phrase = preprocessor.preprocess( candidate_causal_pair[0]) candidate_effect_phrase = preprocessor.preprocess( candidate_causal_pair[1]) if len(candidate_causal_phrase) > 0 and len( candidate_effect_phrase) > 0: X.append((candidate_causal_pair[0], candidate_causal_pair[1])) y.append(label) return X, y
import timeit import collections from utils.utilities import Utilities from preprocessing.preprocesssor import Preprocessor from causality_detection.causal_stength_calculator import CausalStrengthCalculator from causality_detection.itemsest_causality import ItemsetCausality if __name__ == "__main__": start_time = timeit.default_timer() event_file_path = 'events.csv' utilities = Utilities() causal_strength_calculator = CausalStrengthCalculator() itemset_causality = ItemsetCausality() preprocessor = Preprocessor(params=['lower', 'lemmatize']) rows = utilities.read_from_csv(event_file_path) header = rows[0] del rows[0] events_phrases = [] for row in rows: phrases = [phrase.strip() for phrase in row[header.index('event_phrases')].split(',')] events_phrases += phrases sorted_event_phrases = collections.Counter(events_phrases).most_common() low_freq_events = [event[0] for event in sorted_event_phrases if event[1] <= 5] event_rows = [] for row in rows:
def __init__(self): self.data_source_file = None self.utilities = Utilities()
def __init__(self): self.wiki_file = '/home/humayun/enwiki-latest-pages-articles.xml.bz2' self.utilities = Utilities() self.preprocessor = Preprocessor( ['remove_stopwords', 'remove_punctuation', 'lemmatize'])