from utils.utilities import Utilities from preprocessing.event_extractor import EventExtractor import time if __name__ == '__main__': # Original Code # # utilities = Utilities() # event_extraction = EventExtractor() # # # # event_extraction.save_texts_in_file() # # events = event_extraction.extract_events() # # for event in events: # if len(event['event_phrases']) > 0: # utilities.save_or_append_in_csv(event, 'events.csv') utilities = Utilities() event_extractor = EventExtractor() tweet_rows = event_extractor.get_unique_tweets() tweet_sentences = event_extractor.get_tweet_sentences(tweet_rows) events = event_extractor.extract_events2(tweet_sentences) events = sorted(events, key=lambda x: time.strptime(x[0], '%d-%m-%Y %H:%M')) utilities.save_or_append_list_as_csv(events, 'events2.csv')
header = tweet_rows[0] del tweet_rows[0] count = 0 tweet_causal_rows = [] for tweet_row in tweet_rows: tweet = preprocessor.preprocess(tweet_row[header.index('text')]) sentences = sent_tokenize(tweet) # tweet_causal_pairs = [] for sentence in sentences: sentence_causal_pair = apply_causal_rules(sentence) if sentence_causal_pair is not None: tweet_causal_rows.append( [tweet_row, sentence, sentence_causal_pair]) # print(sentence) # print(sentence_causal_pair) # if len(sentence_causal_pair)>1: # print(sentence) # print(sentence_causal_pair) # print(len(sentence_causal_pair)) # count += 1 # event_pair = extract_sentence_event_pairs(list(sentence_causal_pair)) # if len(event_pair) > 0: # event_pairs.append(event_pair) # print(event_pair) # print(count) utilities.save_or_append_list_as_csv(tweet_causal_rows, 'causal_pairs.csv') # event_pairs = extract_batch_sentence_event_pairs(tweet_causal_rows)