def parse_tweets(tweet): if len(tweet) > 0: keep_tweet = tweet tweet = tweet[0].split() cleaned_tweet = tweet_cleaner.clean_tweet(tweet) words = [] final_words = [] for word in cleaned_tweet: if len(word) > 2: checking_words = tweet_cleaner.check_words(word) if checking_words != 0: words.append(checking_words) for word in words: # stripping word twice to account for people's weird punctuation errors word = tweet_cleaner.strip_word(word) word = tweet_cleaner.strip_word(word) if len(word) > 2: checking_words = tweet_cleaner.check_words(word) if checking_words != 0: final_words.append(checking_words) words_in_game = [] if len(final_words) > 0: for word in final_words: if word > 0: # check Wordnik results = wordnik_api.check_all(word) if results != None: # if Wordnik never returned a result > 0, put it into the game words_in_game.append(word) if len(words_in_game) > 0: model.set_game_words_tweets(words_in_game, keep_tweet)
def process_json(screen_name): workbook = xlsxwriter.Workbook('%s_tweets.xlsx' % screen_name) worksheet = workbook.add_worksheet() # Start from the first cell. Rows and columns are zero indexed. row = 0 col = 0 worksheet.write(row, 0, 'id') worksheet.write(row, 1, 'created_at') worksheet.write(row, 2, 'full_text') worksheet.write(row, 3, 'clean_text') row += 1 with open('{}.json'.format(screen_name)) as json_reader: lines = json_reader.readlines() for line in lines: json_tweet = json.loads(line) if 'retweeted_status' in json_tweet: text = json_tweet['retweeted_status']['full_text'] else: text = json_tweet['full_text'] clean_text = tweet_cleaner.clean_tweet(text) clean_text = tweet_cleaner.normalize_arabic(clean_text) clean_text = tweet_cleaner.remove_repeating_char(clean_text) clean_text = tweet_cleaner.keep_only_arabic(clean_text.split()) worksheet.write(row, col, json_tweet['id_str']) worksheet.write(row, col + 1, json_tweet['created_at']) worksheet.write(row, col + 2, text) worksheet.write(row, col + 3, clean_text) row += 1 workbook.close()
def process_json(filters, json_file, xls_file): import preprocessor workbook = xlsxwriter.Workbook(xls_file) worksheet = workbook.add_worksheet() # Start from the first cell. Rows and columns are zero indexed. row = 0 col = 0 worksheet.write(row, 0, 'id') worksheet.write(row, 1, 'created_at') worksheet.write(row, 2, 'text') worksheet.write(row, 3, 'clean_text') row += 1 lines = json_file.readlines() for line in lines: json_tweet = json.loads(line) if 'retweeted_status' in json_tweet: text = json_tweet['retweeted_status']['text'] else: text = json_tweet['text'] clean_text = tweet_cleaner.clean_tweet(text) clean_text = tweet_cleaner.normalize_arabic(clean_text) clean_text = tweet_cleaner.remove_repeating_char(clean_text) clean_text = tweet_cleaner.keep_only_arabic(clean_text.split()) is_filtered = filter_tweet(filters, clean_text) if not is_filtered: # print only if text is not filtered worksheet.write(row, col, json_tweet['id_str']) worksheet.write(row, col + 1, json_tweet['created_at']) worksheet.write(row, col + 2, text) worksheet.write(row, col + 3, clean_text) row += 1 workbook.close()
def parse_tweets(tweet): if len(tweet) > 0: keep_tweet = tweet tweet = tweet[0].split() cleaned_tweet = tweet_cleaner.clean_tweet(tweet) words = [] final_words = [] for word in cleaned_tweet: if len(word) > 2: checking_words = tweet_cleaner.check_words(word) if checking_words != 0: words.append(checking_words) for word in words: # stripping word twice to account for people's weird punctuation errors word = tweet_cleaner.strip_word(word) word = tweet_cleaner.strip_word(word) if len(word) > 2: checking_words = tweet_cleaner.check_words(word) if checking_words != 0: final_words.append(checking_words) words_in_game = [] if len(final_words) > 0: for word in final_words: if word > 0: # check Wordnik results = wordnik_api.check_all(word) if results != None: # if Wordnik never returned a result > 0, put it into the game words_in_game.append(word) if len(words_in_game) > 0: model.set_game_words_tweets(words_in_game, keep_tweet)
def do_embedding(): data_pd = pd.read_csv( DATA_FILE_ORIGINAL_TWEETS, sep=',', skipinitialspace=True, ) data_set = pd.DataFrame(data_pd) # print(data_set.describe()) # print(data_pd.__len__()) # print(data_set.head()) col = data_pd['tweet_text'] # prepare an empty list tweet_list = list() tweets = data_set['tweet_text'].values.tolist() for tweet in tweets[0:params.NBR_TWEETS]: print(tweet) words = tweet_cleaner.clean_tweet(tweet) tweet_list.append(words) len(tweet_list) model = gensim.models.Word2Vec(sentences=tweet_list, size=params.EMBEDDING_DIM, workers=4, min_count=params.EMBEDDING_MIN_SIZE) model.save(STATE_FILE_ORIGINAL_TWEETS)
def extract_tweets_from_json(json_reader, text_writer): json_tweets = json_reader.readlines() print('tweets in json file: {} tweets'.format(len(json_tweets))) tweets_list = list() extracted_tweets_count = 0 if args.include_id: text_writer.write("id\ttweet\n") for json_tweet in json_tweets: try: if json_tweet: # load it as Python dict tweet = json.loads(json_tweet) tid = tweet['id'] text = tweet['text'] text = tweet_cleaner.clean_tweet(text) if args.normalize: text = tweet_cleaner.normalize_arabic(text) if args.remove_repeated_letters: text = tweet_cleaner.remove_repeating_char(text) if args.keep_only_arabic: text = tweet_cleaner.keep_only_arabic(text.split()) if len(text.split()) > 2: if args.exclude_redundant: if text not in tweets_list: tweets_list.append(text) if args.include_id: text_writer.write( str(tid) + "\t" + text + "\n") # print('id:{}'.format(str(tid))) # print('text:', text) # print('tweet:', tweet['text']) # input("press any key...") else: text_writer.write(text + "\n") extracted_tweets_count += 1 else: if args.include_id: text_writer.write(str(tid) + "\t" + text + "\n") else: text_writer.write(text + "\n") extracted_tweets_count += 1 except json.decoder.JSONDecodeError as error: pass except UnicodeDecodeError as error: pass print('extracted tweets: {} tweets'.format(extracted_tweets_count))
def tweets_2_vectors(): # This is the vector of documents / tweets document_vector_list = [] #load the model back model = gensim.models.Word2Vec.load("state_files/w2v.model") data_pd = pd.read_csv( 'data/tweets.csv', sep=',', skipinitialspace=True, ) data_set = pd.DataFrame(data_pd) col = data_pd['tweet_text'] # prepare an empty list tweet_list = list() tweets = data_set['tweet_text'].values.tolist() for tweet in tweets[0:40000]: words = tweet_cleaner.clean_tweet(tweet) count = 0 tweet_vector = [] for w in words: if w in model: if count == 0: tweet_vector = model[w] else: tweet_vector = tweet_vector + model[w] count = count + 1 if len(tweet_vector) > 0: document_vector_list.append(tweet_vector) document_vector_array = np.array(document_vector_list) pickle.dump(document_vector_array, open("state_files/tweets_vector.dat", "wb"))
def tweets_2_vectors( self, active, state_file_tweets_vector='state_files/tweets_vector.dat'): self.state_file_tweets_vector = state_file_tweets_vector if active: # This is the vector of documents / tweets tweets_vector_list = [] # load the model back model = gensim.models.Word2Vec.load(self.model_file_w2v) data_pd = pd.read_csv( self.data_file_tweets, sep=',', skipinitialspace=True, ) data_set = pd.DataFrame(data_pd) tweets = data_set['tweet_text'].values.tolist() for tweet in tweets[0:self.nbr_of_tweets]: words = tweet_cleaner.clean_tweet(tweet) count = 0 tweet_vector = [] for w in words: if w in model: if count == 0: tweet_vector = model[w] else: tweet_vector = tweet_vector + model[w] count = count + 1 if len(tweet_vector) > 0: tweets_vector_list.append(tweet_vector) document_vector_array = np.array(tweets_vector_list) pickle.dump(document_vector_array, open(self.state_file_tweets_vector, "wb"))
def word_embedding(self, active, embedding_dim=100, embedding_min_size=30, model_file_w2v='state_files/w2v.model'): self.embedding_dim = embedding_dim self.embedding_min_size = embedding_min_size self.model_file_w2v = model_file_w2v if active: data_pd = pd.read_csv( self.data_file_tweets, sep=',', skipinitialspace=True, ) data_set = pd.DataFrame(data_pd) # print(data_set.describe()) # print(data_pd.__len__()) # print(data_set.head()) # prepare an empty list tweet_list = list() tweets = data_set['tweet_text'].values.tolist() for tweet in tweets[0:self.nbr_of_tweets]: print(tweet) words = tweet_cleaner.clean_tweet(tweet) tweet_list.append(words) len(tweet_list) model = gensim.models.Word2Vec(sentences=tweet_list, size=self.embedding_dim, workers=4, min_count=self.embedding_min_size) model.save(self.model_file_w2v)
def get_all_tweets(screen_name): # Twitter only allows access to a users most recent 3240 tweets with this method # authorize twitter, initialize tweepy if api_keys.consumer_key == '' or api_keys.consumer_secret == '' \ or api_keys.access_token == '' or api_keys.access_secret == '': print("API key not found. Please check api_keys.py file") sys.exit(-1) auth = tweepy.OAuthHandler(api_keys.consumer_key, api_keys.consumer_secret) auth.set_access_token(api_keys.access_token, api_keys.access_secret) api = tweepy.API(auth) # initialize a list to hold all the tweepy Tweets alltweets = [] # make initial request for most recent tweets (200 is the maximum allowed count) new_tweets = api.user_timeline(screen_name=screen_name, count=200) # save most recent tweets alltweets.extend(new_tweets) # save the id of the oldest tweet less one oldest = alltweets[-1].id - 1 # keep grabbing tweets until there are no tweets left to grab while len(new_tweets) > 0: print("getting tweets before {}".format(oldest)) # all subsiquent requests use the max_id param to prevent duplicates new_tweets = api.user_timeline(screen_name=screen_name, count=200, max_id=oldest) # save most recent tweets alltweets.extend(new_tweets) # update the id of the oldest tweet less one oldest = alltweets[-1].id - 1 print("...{} tweets downloaded so far".format((len(alltweets)))) #print('all tweets\n', alltweets) #print('first tweet:', alltweets[0]) # transform the tweepy tweets into a 2D array that will populate the csv outtweets = [[ tweet.id_str, tweet.created_at.strftime('%m/%d/%Y'), tweet.text.encode("utf-8").decode('utf-8') ] for tweet in alltweets] outtweetsDict = [{ 'id': tweet.id_str, 'created_at': tweet.created_at.strftime('%m/%d/%Y'), 'text': tweet.text.encode("utf-8").decode('utf-8') } for tweet in alltweets] #print('first outtweets:', outtweets[0]) # write the csv with open('%s_tweets.csv' % screen_name, 'w') as csvfile: fieldnames = ["id", "created_at", "text"] writer = csv.DictWriter(csvfile, fieldnames=fieldnames, dialect='excel') writer.writeheader() writer.writerows(outtweetsDict) workbook = xlsxwriter.Workbook('%s_tweets.xlsx' % screen_name) worksheet = workbook.add_worksheet() # Start from the first cell. Rows and columns are zero indexed. row = 0 col = 0 worksheet.write(row, 0, 'id') worksheet.write(row, 1, 'created_at') worksheet.write(row, 2, 'original_text') worksheet.write(row, 3, 'clean_text') row += 1 for tid, tdate, text in outtweets: clean_text = tweet_cleaner.clean_tweet(text) clean_text = tweet_cleaner.normalize_arabic(clean_text) clean_text = tweet_cleaner.remove_repeating_char(clean_text) clean_text = tweet_cleaner.keep_only_arabic(clean_text.split()) worksheet.write(row, col, tid) worksheet.write(row, col + 1, tdate) worksheet.write(row, col + 2, text) worksheet.write(row, col + 3, clean_text) row += 1 workbook.close()
data_pd = pd.read_csv( 'data/tweets.csv', sep=',', skipinitialspace=True, ) data_set = pd.DataFrame(data_pd) col = data_pd['tweet_text'] # prepare an empty list tweet_list = list() tweets = data_set['tweet_text'].values.tolist() for tweet in tweets[0:40000]: words = tweet_cleaner.clean_tweet(tweet) count = 0 tweet_vector = [] for w in words: if w in model: if count == 0: tweet_vector = model[w] else: tweet_vector = tweet_vector + model[w] count = count + 1 if len(tweet_vector) > 0: document_vector_list.append(tweet_vector) document_vector_array = np.array(document_vector_list)