def _prepare_data(self): return [ Tweet("1", "qwerty 1", "qazxsw 1", "plmnko 1"), Tweet("2", "qwerty 2", "qazxsw 2", "plmnko 2"), Tweet("3", "qwerty 3", "qazxsw 3", "plmnko 3"), Tweet("4", "qwerty 4", "qazxsw 4", "plmnko 4"), Tweet("5", "qwerty 5", "qazxsw 5", "plmnko 5"), ]
def _prepare_expected_data(self): return [ Tweet("1", self._highlighted_text, self._highlighted_text, "plmnko 1"), Tweet("2", self._highlighted_text, self._highlighted_text, "plmnko 2"), Tweet("3", self._highlighted_text, self._highlighted_text, "plmnko 3"), Tweet("4", self._highlighted_text, self._highlighted_text, "plmnko 4"), Tweet("5", self._highlighted_text, self._highlighted_text, "plmnko 5"), ]
def request_tweets(subject): params_map = { 'q': subject, 'tweet_mode': 'extended', 'count': 100, 'geocode': geocode_param } headers = {'Authorization': token} r = requests.get(tweets_search_url, params=params_map, headers=headers) result = r.json() return [ Tweet(tweet['retweeted_status']['full_text'], []) if 'retweeted_status' in tweet else Tweet(tweet['full_text'], []) for tweet in result['statuses'] ]
def run(self): # url does't support `#` symbol, it should be converted into %23 # https://www.w3schools.com/tags/ref_urlencode.asp quoted_term = quote(self.hash_tag) # get webpage of URL constant, see above, where q=hash_tag&s=mode, by default mode=SearchMode.EXACT # which is sprv (see search_mode.py) request = get(self.URL.format(quoted_term, self.mode)) # get response text response = request.text # remove BAD_HEADER constant, because <?xml version="1.0" encoding="utf-8"?> is incompatible with PyQuery # and here we create feed pyQuery object # https://pypi.org/project/pyquery/ filter_idx = len(self.BAD_HEADER) feed = PyQuery(response[filter_idx:]) # like in jquery ".tweet" -> class="tweet", returns us an array tweet_html_array = feed(".tweet") for element in tweet_html_array.items(): time = element(".timestamp a").attr("name") # because the time object is tweet_123178237914 - we need to get rid of `tweet_` in order to get a number time = time[len(self.TWEET_TIME):] # compare time by comparing 2 strings if str_number_is_greater(time, str(self.time_start)): author = element(".username").text() text = element(".tweet-text .dir-ltr").text() tweet = Tweet(author, time, text) self.tweets.append(tweet)
def get_tweet_list(): tweets: List[Status] = TweepyWrapper().get_trending_tweets_in_LA(limit=10) result = [] for t in tweets: tweet = Tweet.parse_from_status(t) result.append(tweet.__dict__) return result
def test_handle(self): return_val = [Tweet()] self.repository.get_tweets_by_hashtag.return_value = return_val response = self.handler.handle(self.create_request()) self.repository.get_tweets_by_hashtag.assert_called_with(expected_hashtag, expected_limit) self.assertIsInstance(response, TweetsResponse) self.assertEqual(response.tweets, return_val)
def parse(doc): source = doc['_source'] # Comprobamos que el proveedor del documento es twitter provider = source['provider'] if not provider == 'twitter': raise Exception() id = source['post_id'] text = next(iter(source['body'].values())) is_retweet = source['is_retweet'] is_reply = source['is_reply'] post_type = 'retweet' if is_retweet else ('reply' if is_reply else 'original') num_retweets = source['retweet_count'] retweet_id = source['retweet_id'] if post_type == 'retweet' else None reply_id = source['retweet_id'] if post_type == 'reply' else None author = None result = match('^(\d{4})\-(\d{1,2})\-(\d{1,2})T(\d{1,2})\:(\d{1,2})\+.*$', source['published_at']) if not result: raise Exception() timestamp = datetime(*[int(strnum) for strnum in result.groups()]).strftime('%s') user_data = source['user'] num_followers = user_data['followers_count'] num_friends = user_data['friends_count'] screen_name = user_data['screenname'] author = TwitterUser(screen_name, num_followers, num_friends) tweet = Tweet(author, id, text, post_type, num_retweets, timestamp, retweet_id, reply_id) return tweet
def __init__(self, tweets): tweet_list = [] for tweet in tweets: tweet_id = tweet['id'] message = tweet['message'] tweet_data = Tweet(tweet_id, message) tweet_list.append(tweet_data) self.tweets = tweet_list
def find() -> list: tweet_table_name = get_tweet_table_name() tweet_table = Table(tweet_table_name) results = tweet_table.scan() tweet_list = [] for result in results: todo = Tweet.from_dict(result) tweet_list.append(todo) return tweet_list
def create(params) -> Tweet: tweet_id = int(uuid.uuid4().int) now_datetime = datetime.now().strftime('%Y/%m/%d') delete_status = False tweet = Tweet(tweet_id=TweetId(tweet_id), content=Content(params['content']), create_date=CreateDate(now_datetime), delete_status=DeleteStatus(delete_status), user_id=UserId(params['user_id'])) return tweet
def find_by_user_id(user_id: str) -> list: tweet_table_name = get_tweet_table_name() tweet_table = Table(tweet_table_name) results = tweet_table.scan() tweet_list = [] for result in results: if result['user_id'] == user_id: tweet = Tweet.from_dict(result) tweet_list.append(tweet) return tweet_list
def get_user_tweets(self, target, days, party): """ Get a specific user's tweets from the last X number of days, and associate the tweets with specific paryy :param target: Target name :param days: Number of days to query back :param party: The party associated with the person queried :return: Array of tweets for the specific person """ tweets = [] end_date = datetime.utcnow() - timedelta(days=days) for status in Cursor(self._api.user_timeline, id=target, count=1000, tweet_mode='extended').items(): tweet = Tweet() tweet.load_from_status(status, party) tweets.append(tweet) if status.created_at < end_date: break return tweets
def get_similar_tweets(tweet_id): print(tweet_id) print("==============================") tweets = [] # get the tweet tweet = TweepyWrapper().get_tweet(tweet_id) original = Tweet.parse_from_status(tweet) # extract entities entities: List[str] = get_entities(original.text) retry_count = 0 limit = 5 while len(tweets) < limit and retry_count < 5: # find tweets with entities statuses = [] for entity in entities: statuses.extend(TweepyWrapper().get_tweets_by_keyword( keyword=entity, limit=15)) potential_tweets: List[Tweet] = [ Tweet.parse_from_status(t) for t in statuses ] # filter similarities #print(len(tweets), retry_count, "===========================================") for potential_tweet in potential_tweets: similarity = sentence_distance.similarity(original.text, potential_tweet.text) #print(similarity) # todo judge whether eligible tweets.append(potential_tweet.__dict__) if len(tweets) > limit: break # repeat if less than 15 retry_count += 1 return jsonify(tweets)
def getTweetsforCandidate(self, screen_name): storedTweets = Tweet.query().filter( Tweet.candidate == screen_name).order(Tweet.id).fetch() if not storedTweets: tweets = self.api.GetUserTimeline(screen_name=screen_name, count=100) else: tweets = self.api.GetUserTimeline(screen_name=screen_name, since_id=storedTweets[-1].id, count=100) for t in tweets: tweet = Tweet() tweet.candidate = screen_name tweet.text = t.text tweet.id = t.id tweet.dateCreated = datetime.strptime( t.created_at, '%a %b %d %H:%M:%S +0000 %Y') tweet.put() self.response.write('\n' + str(tweet))
def clicked(self): tweet = self.tekst.get("1.0", tk.END).rstrip() if len(tweet) > 140: self.label['text'] = "Tweet is too long" elif len(tweet) == 0: self.label['text'] = 'Tweet is empty' else: self.label['text'] = self.standard_text self.tekst.delete('1.0', tk.END) showinfo(title='Sent', message='Message has been sent') TwitterIO.add_tweet_to_file( 'tweets/tweets.json', Tweet(str(uuid.uuid4()), time.strftime('%m/%d/%Y,%H:%M:%S'), tweet))
def query_twitter_for_tweets(geo_code): tweets = set() for tweet in tweepy.Cursor(api.search, q="*", geocode=geo_code, lang="en").items(50): tweet_id = tweet.id tweet_date = datetime.strptime(str(tweet.created_at)[:10], '%Y-%m-%d').strftime('%d-%m-%Y') tweet_text = tweet.text tweet_score = round(TextBlob(tweet_text).sentiment.polarity, 4) new_tweet = Tweet(tweet_id, tweet_date, tweet_text, tweet_score) tweets.add(new_tweet) return tweets
def request_tweets(term): tweets = db.tweets.find() filtered_list = [] for tweet in tweets: text = '' if 'retweeted_status' in tweet: text = tweet['retweeted_status']['full_text'] else: text = tweet['full_text'] if term in text: filtered_list.append(text) return [Tweet(text, []) for text in filtered_list]
def get_search_tweets(self, country_code, days, target): """ Free text tweets search :param country_code: (Optional) Twitter's internal ID for the country, set None if we don't want this filter :param days: Number of days to query back from Twitter :param target: The search query we want to invoke against Twitter :return: Array of tweets with party set to Unknown """ tweets = [] end_date = datetime.utcnow() - timedelta(days=days) query = target if country_code is not None: query = 'place:{} AND {}'.format(country_code, target) for status in Cursor(self._api.search, q=query, count=1000, tweet_mode='extended').items(): if status.created_at > end_date: tweet = Tweet() tweet.load_from_status(status, Parties.Unknown) tweets.append(tweet) return tweets
def tweet_post(): if request.method == 'POST': username = request.form['username'] content = request.form['content'] images = request.form.getlist('images[]') if username: user = User.query.filter_by(username=username).first() if user: tweet = Tweet(user.id, content) image_ids = [] if images: for img in images: image = Image(img) db.session.add(image) db.session.flush() image_ids.append(image.id) tweet.images = ','.join(map(str, image_ids)) db.session.add(tweet) db.session.commit() return jsonify(message="200 OK") else: return jsonify(error="No such user: %s" % username) else: return jsonify(error="Error param")
def tweet_post(): if request.method == 'POST': username = request.form['username'] content = request.form['content'] images = request.form.getlist('images[]') if username: user = User.query.filter_by(username=username).first() if user: tweet = Tweet(user.id, content) image_ids = [] if images: for img in images: image = Image(img) db.session.add(image) db.session.flush() image_ids.append(image.id) tweet.images = ','.join(map(str, image_ids)) db.session.add(tweet) db.session.commit() return jsonify(message="200 OK") else: return jsonify(error="No such user: %s" % username) else: return jsonify(error="Error param")
def _extract_tweet(self, soup: BeautifulSoup) -> Tweet: stats_container = soup.find('div', class_='ProfileTweet-actionCountList') stats = self._extract_stats(stats_container) header_container = soup.select('div.stream-item-header')[0] account = self._extract_account(header_container) texts = soup.select('p.TweetTextSize.tweet-text')[0].findAll(text=True) tweet = Tweet() tweet.retweets = stats['retweet'] tweet.likes = stats['favorite'] tweet.replies = stats['reply'] tweet.text = ''.join(texts) tweet.hashtags = re.findall(r"(#\w+)", tweet.text) tweet.date = datetime.utcfromtimestamp( int( header_container.find('span', class_='_timestamp').attrs['data-time'])) tweet.account = account return tweet
def tweet_action(tweet): user_creds = random.choice(USERS_CREDS) api = twitter_api_for(user_creds) screen_name = tweet["user"]["screen_name"] action = get_action(tweet) tweet_text = "@" + screen_name + " " + action.get_tweet_text(tweet) user_id = user_creds["id"] in_reply_to_tweet_id = tweet["id"] in_reply_to_user_id = tweet["user"]["id"] db.session.add(Tweet(tweet)) db.session.commit() sched_favorite(api, in_reply_to_tweet_id) sched_follow(api, in_reply_to_user_id) sched_post_tweet(api, action, tweet_text, user_id, in_reply_to_tweet_id)
def home(): start, amount = pager_args() articles = Article().get_paged(start=start, amount=amount) tot_count = Article().get_count_filtered() tweet_count = Tweet().get_tweetcount_filtered() payload = render_template( 'home.html', articles=articles, start=int(start), amount=int(amount), tot_count=tot_count, tweet_count=tweet_count, cssver=os.path.getmtime( os.path.join(os.path.dirname(__file__), 'static/thcrrspndnt.css')), site=settings.CONFIG.get('site', 'thecorrespondent.com'), version=settings.corres_version, ) return payload
def _process_tweet(self, status): ''' Procesa un tweet. :param status: Es un objeto tweepy.Status de donde se extaerá la información del tweet. :return: Devuelve el tweet procesado, una instancia de la clase Tweet, o None si hubo un error al procesar la información del tweet. ''' try: # ID del tweet. id = status.id_str # Cuerpo del tweet text = status.text # Es un retweet ? is_retweet = hasattr(status, 'retweeted_status') # Es una reply ? is_reply = hasattr( status, 'in_reply_to_status_id' ) and not status.in_reply_to_status_id is None post_type = 'retweet' if is_retweet else ( 'reply' if is_reply else 'original') # Número de retweets num_retweets = status.retweet_count # Timestamp UNIX timestamp = status.created_at.strftime('%s') # ID del tweet retweeteado. retweet_id = status.retweeted_status if is_retweet else None # ID del tweet respondido. reply_id = status.in_reply_to_status_id if is_reply else None # Info del usuario. user_status = status.user author = self._process_user(user_status) if author is None: raise Exception() tweet = Tweet(author, id, text, post_type, num_retweets, timestamp, retweet_id, reply_id) return tweet except: pass return None
def get_tweets_from_file(file_location): """ Haalt tweets op uit bestand en geeft terug als json :param file_location: str :return: List(Tweet) """ try: tweets = [] data = TwitterIO.read_file(file_location) for tweet in data: tweets.append( Tweet(tweet['id'], tweet['datetime'], tweet['message'])) return tweets except IOError: print('Bestand niet gevonden.') except Exception as e: print( 'Er is iets fout gegaan bij het ophalen van alle tweets uit een bestand' ) print(e.message)
def search(): query = request.args.get('query') if not query: return home() start, amount = pager_args() # split the query on whitespace, but keep quoted strings together tokens = re.findall('\w+|"[\w\s]*"', query) articles = Article().get_search(tokens, start=start, amount=amount) tot_count = Article().get_searchcount(tokens) tweet_count = Tweet().get_tweetcount_searchquery(tokens) payload = render_template('search.html', articles=articles, start=int(start), amount=int(amount), version=settings.corres_version, tot_count=tot_count, tweet_count=tweet_count, tokens=tokens, cssver=os.path.getmtime( os.path.join(os.path.dirname(__file__), 'static/thcrrspndnt.css'))) return payload
def harvest(self): twitter_session = OAuth1Session( self.settings['CONSUMER_KEY'], client_secret=self.settings['CONSUMER_SECRET'], resource_owner_key=self.settings['OAUTH_KEY'], resource_owner_secret=self.settings['OAUTH_SECRET']) result = twitter_session.get( 'https://api.twitter.com/1.1/search/tweets.json', params={ 'q': self.query, 'count': 100, 'result_type': 'recent' }) tweets = result.json() print("\nHarvesting %s" % datetime.datetime.now(), end=' ') if 'statuses' in tweets: for tweet in tweets['statuses']: parsed_tweet = Tweet.parse_json(tweet) if parsed_tweet: # it's up to the article to decide what to do with the found URL Article.maybe_find_or_create(parsed_tweet.corres_url) else: print('No tweets found?')
def run(self): # init api # https://github.com/bear/python-twitter api = twitter.Api(consumer_key=self.apiKey.consumer_key, consumer_secret=self.apiKey.consumer_secret, access_token_key=self.apiKey.access_token_key, access_token_secret=self.apiKey.access_token_secret) # search for hashtag search = api.GetSearch(self.hash_tag) for tweet in search: # extract time time = tweet.created_at_in_seconds # compare if time of the current tweet is less the our desirable time then go back to for loop if self.time_start >= time: continue message = tweet.text author = tweet.user.name # convert into tweeter_obj and store it in the array tweet_obj = Tweet(author, time, message) self.tweets.append(tweet_obj)
def register(tweet: Tweet) -> None: tweet_table_name = get_tweet_table_name() tweet_table = Table(tweet_table_name) tweet_table.put_item(tweet.to_dict())
from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker from data_access.db_access import Db_Access from model.tweet import Tweet from datetime import date # instantiate a database access db = Db_Access() # create a list of tweets for test tweet_list = [] # fill the list with tweets tweet_list.append( Tweet('323K2K3L2', 'tweet 1', 'positif', 0.6, 'Samsung', date(1292, 4, 26))) tweet_list.append( Tweet('Y9SD8DS0D', 'tweet 2', 'neutral', 0.5, 'Samsung', date(1292, 4, 26))) tweet_list.append( Tweet('2LSD9809S', 'tweet 3', 'negative', 0.2, 'Samsung', date(1292, 4, 26))) # insert tweet list # db.insert_tweet(tweet_list) # get all tweets tweets = db.get_all(Tweet) # get tweets by hashtag # tweets_by_hashtag = db.get_tweets_by_hashtag("Samsung")
def get_model(self): """ :rtype: model.tweet.Tweet """ return Tweet()