def _prepare_data(self):
return [
Tweet("1", "qwerty 1", "qazxsw 1", "plmnko 1"),
Tweet("2", "qwerty 2", "qazxsw 2", "plmnko 2"),
Tweet("3", "qwerty 3", "qazxsw 3", "plmnko 3"),
Tweet("4", "qwerty 4", "qazxsw 4", "plmnko 4"),
Tweet("5", "qwerty 5", "qazxsw 5", "plmnko 5"),
]
def _prepare_expected_data(self):
return [
Tweet("1", self._highlighted_text, self._highlighted_text,
"plmnko 1"),
Tweet("2", self._highlighted_text, self._highlighted_text,
"plmnko 2"),
Tweet("3", self._highlighted_text, self._highlighted_text,
"plmnko 3"),
Tweet("4", self._highlighted_text, self._highlighted_text,
"plmnko 4"),
Tweet("5", self._highlighted_text, self._highlighted_text,
"plmnko 5"),
]
def request_tweets(subject):
params_map = {
'q': subject,
'tweet_mode': 'extended',
'count': 100,
'geocode': geocode_param
}
headers = {'Authorization': token}
r = requests.get(tweets_search_url, params=params_map, headers=headers)
result = r.json()
return [
Tweet(tweet['retweeted_status']['full_text'], [])
if 'retweeted_status' in tweet else Tweet(tweet['full_text'], [])
for tweet in result['statuses']
]
def run(self):
# url does't support `#` symbol, it should be converted into %23
# https://www.w3schools.com/tags/ref_urlencode.asp
quoted_term = quote(self.hash_tag)
# get webpage of URL constant, see above, where q=hash_tag&s=mode, by default mode=SearchMode.EXACT
# which is sprv (see search_mode.py)
request = get(self.URL.format(quoted_term, self.mode))
# get response text
response = request.text
# remove BAD_HEADER constant, because <?xml version="1.0" encoding="utf-8"?> is incompatible with PyQuery
# and here we create feed pyQuery object
# https://pypi.org/project/pyquery/
filter_idx = len(self.BAD_HEADER)
feed = PyQuery(response[filter_idx:])
# like in jquery ".tweet" -> class="tweet", returns us an array
tweet_html_array = feed(".tweet")
for element in tweet_html_array.items():
time = element(".timestamp a").attr("name")
# because the time object is tweet_123178237914 - we need to get rid of `tweet_` in order to get a number
time = time[len(self.TWEET_TIME):]
# compare time by comparing 2 strings
if str_number_is_greater(time, str(self.time_start)):
author = element(".username").text()
text = element(".tweet-text .dir-ltr").text()
tweet = Tweet(author, time, text)
self.tweets.append(tweet)
def get_tweet_list():
tweets: List[Status] = TweepyWrapper().get_trending_tweets_in_LA(limit=10)
result = []
for t in tweets:
tweet = Tweet.parse_from_status(t)
result.append(tweet.__dict__)
return result
def test_handle(self):
return_val = [Tweet()]
self.repository.get_tweets_by_hashtag.return_value = return_val
response = self.handler.handle(self.create_request())
self.repository.get_tweets_by_hashtag.assert_called_with(expected_hashtag, expected_limit)
self.assertIsInstance(response, TweetsResponse)
self.assertEqual(response.tweets, return_val)
def parse(doc):
source = doc['_source']
# Comprobamos que el proveedor del documento es twitter
provider = source['provider']
if not provider == 'twitter':
raise Exception()
id = source['post_id']
text = next(iter(source['body'].values()))
is_retweet = source['is_retweet']
is_reply = source['is_reply']
post_type = 'retweet' if is_retweet else ('reply' if is_reply else 'original')
num_retweets = source['retweet_count']
retweet_id = source['retweet_id'] if post_type == 'retweet' else None
reply_id = source['retweet_id'] if post_type == 'reply' else None
author = None
result = match('^(\d{4})\-(\d{1,2})\-(\d{1,2})T(\d{1,2})\:(\d{1,2})\+.*$', source['published_at'])
if not result:
raise Exception()
timestamp = datetime(*[int(strnum) for strnum in result.groups()]).strftime('%s')
user_data = source['user']
num_followers = user_data['followers_count']
num_friends = user_data['friends_count']
screen_name = user_data['screenname']
author = TwitterUser(screen_name, num_followers, num_friends)
tweet = Tweet(author, id, text, post_type, num_retweets, timestamp, retweet_id, reply_id)
return tweet
def __init__(self, tweets):
tweet_list = []
for tweet in tweets:
tweet_id = tweet['id']
message = tweet['message']
tweet_data = Tweet(tweet_id, message)
tweet_list.append(tweet_data)
self.tweets = tweet_list
def find() -> list:
tweet_table_name = get_tweet_table_name()
tweet_table = Table(tweet_table_name)
results = tweet_table.scan()
tweet_list = []
for result in results:
todo = Tweet.from_dict(result)
tweet_list.append(todo)
return tweet_list
def create(params) -> Tweet:
tweet_id = int(uuid.uuid4().int)
now_datetime = datetime.now().strftime('%Y/%m/%d')
delete_status = False
tweet = Tweet(tweet_id=TweetId(tweet_id),
content=Content(params['content']),
create_date=CreateDate(now_datetime),
delete_status=DeleteStatus(delete_status),
user_id=UserId(params['user_id']))
return tweet
def find_by_user_id(user_id: str) -> list:
tweet_table_name = get_tweet_table_name()
tweet_table = Table(tweet_table_name)
results = tweet_table.scan()
tweet_list = []
for result in results:
if result['user_id'] == user_id:
tweet = Tweet.from_dict(result)
tweet_list.append(tweet)
return tweet_list
def get_user_tweets(self, target, days, party):
"""
Get a specific user's tweets from the last X number of days, and associate the tweets with specific paryy
:param target: Target name
:param days: Number of days to query back
:param party: The party associated with the person queried
:return: Array of tweets for the specific person
"""
tweets = []
end_date = datetime.utcnow() - timedelta(days=days)
for status in Cursor(self._api.user_timeline,
id=target,
count=1000,
tweet_mode='extended').items():
tweet = Tweet()
tweet.load_from_status(status, party)
tweets.append(tweet)
if status.created_at < end_date:
break
return tweets
def get_similar_tweets(tweet_id):
print(tweet_id)
print("==============================")
tweets = []
# get the tweet
tweet = TweepyWrapper().get_tweet(tweet_id)
original = Tweet.parse_from_status(tweet)
# extract entities
entities: List[str] = get_entities(original.text)
retry_count = 0
limit = 5
while len(tweets) < limit and retry_count < 5:
# find tweets with entities
statuses = []
for entity in entities:
statuses.extend(TweepyWrapper().get_tweets_by_keyword(
keyword=entity, limit=15))
potential_tweets: List[Tweet] = [
Tweet.parse_from_status(t) for t in statuses
]
# filter similarities
#print(len(tweets), retry_count, "===========================================")
for potential_tweet in potential_tweets:
similarity = sentence_distance.similarity(original.text,
potential_tweet.text)
#print(similarity)
# todo judge whether eligible
tweets.append(potential_tweet.__dict__)
if len(tweets) > limit:
break
# repeat if less than 15
retry_count += 1
return jsonify(tweets)
def getTweetsforCandidate(self, screen_name):
storedTweets = Tweet.query().filter(
Tweet.candidate == screen_name).order(Tweet.id).fetch()
if not storedTweets:
tweets = self.api.GetUserTimeline(screen_name=screen_name,
count=100)
else:
tweets = self.api.GetUserTimeline(screen_name=screen_name,
since_id=storedTweets[-1].id,
count=100)
for t in tweets:
tweet = Tweet()
tweet.candidate = screen_name
tweet.text = t.text
tweet.id = t.id
tweet.dateCreated = datetime.strptime(
t.created_at, '%a %b %d %H:%M:%S +0000 %Y')
tweet.put()
self.response.write('\n' + str(tweet))
def clicked(self):
tweet = self.tekst.get("1.0", tk.END).rstrip()
if len(tweet) > 140:
self.label['text'] = "Tweet is too long"
elif len(tweet) == 0:
self.label['text'] = 'Tweet is empty'
else:
self.label['text'] = self.standard_text
self.tekst.delete('1.0', tk.END)
showinfo(title='Sent', message='Message has been sent')
TwitterIO.add_tweet_to_file(
'tweets/tweets.json',
Tweet(str(uuid.uuid4()), time.strftime('%m/%d/%Y,%H:%M:%S'),
tweet))
def query_twitter_for_tweets(geo_code):
tweets = set()
for tweet in tweepy.Cursor(api.search, q="*", geocode=geo_code,
lang="en").items(50):
tweet_id = tweet.id
tweet_date = datetime.strptime(str(tweet.created_at)[:10],
'%Y-%m-%d').strftime('%d-%m-%Y')
tweet_text = tweet.text
tweet_score = round(TextBlob(tweet_text).sentiment.polarity, 4)
new_tweet = Tweet(tweet_id, tweet_date, tweet_text, tweet_score)
tweets.add(new_tweet)
return tweets
def request_tweets(term):
tweets = db.tweets.find()
filtered_list = []
for tweet in tweets:
text = ''
if 'retweeted_status' in tweet:
text = tweet['retweeted_status']['full_text']
else:
text = tweet['full_text']
if term in text:
filtered_list.append(text)
return [Tweet(text, []) for text in filtered_list]
def get_search_tweets(self, country_code, days, target):
"""
Free text tweets search
:param country_code: (Optional) Twitter's internal ID for the country, set None if we don't want this filter
:param days: Number of days to query back from Twitter
:param target: The search query we want to invoke against Twitter
:return: Array of tweets with party set to Unknown
"""
tweets = []
end_date = datetime.utcnow() - timedelta(days=days)
query = target
if country_code is not None:
query = 'place:{} AND {}'.format(country_code, target)
for status in Cursor(self._api.search,
q=query,
count=1000,
tweet_mode='extended').items():
if status.created_at > end_date:
tweet = Tweet()
tweet.load_from_status(status, Parties.Unknown)
tweets.append(tweet)
return tweets
def tweet_post():
if request.method == 'POST':
username = request.form['username']
content = request.form['content']
images = request.form.getlist('images[]')
if username:
user = User.query.filter_by(username=username).first()
if user:
tweet = Tweet(user.id, content)
image_ids = []
if images:
for img in images:
image = Image(img)
db.session.add(image)
db.session.flush()
image_ids.append(image.id)
tweet.images = ','.join(map(str, image_ids))
db.session.add(tweet)
db.session.commit()
return jsonify(message="200 OK")
else:
return jsonify(error="No such user: %s" % username)
else:
return jsonify(error="Error param")
def tweet_post():
if request.method == 'POST':
username = request.form['username']
content = request.form['content']
images = request.form.getlist('images[]')
if username:
user = User.query.filter_by(username=username).first()
if user:
tweet = Tweet(user.id, content)
image_ids = []
if images:
for img in images:
image = Image(img)
db.session.add(image)
db.session.flush()
image_ids.append(image.id)
tweet.images = ','.join(map(str, image_ids))
db.session.add(tweet)
db.session.commit()
return jsonify(message="200 OK")
else:
return jsonify(error="No such user: %s" % username)
else:
return jsonify(error="Error param")
def _extract_tweet(self, soup: BeautifulSoup) -> Tweet:
stats_container = soup.find('div',
class_='ProfileTweet-actionCountList')
stats = self._extract_stats(stats_container)
header_container = soup.select('div.stream-item-header')[0]
account = self._extract_account(header_container)
texts = soup.select('p.TweetTextSize.tweet-text')[0].findAll(text=True)
tweet = Tweet()
tweet.retweets = stats['retweet']
tweet.likes = stats['favorite']
tweet.replies = stats['reply']
tweet.text = ''.join(texts)
tweet.hashtags = re.findall(r"(#\w+)", tweet.text)
tweet.date = datetime.utcfromtimestamp(
int(
header_container.find('span',
class_='_timestamp').attrs['data-time']))
tweet.account = account
return tweet
def tweet_action(tweet):
user_creds = random.choice(USERS_CREDS)
api = twitter_api_for(user_creds)
screen_name = tweet["user"]["screen_name"]
action = get_action(tweet)
tweet_text = "@" + screen_name + " " + action.get_tweet_text(tweet)
user_id = user_creds["id"]
in_reply_to_tweet_id = tweet["id"]
in_reply_to_user_id = tweet["user"]["id"]
db.session.add(Tweet(tweet))
db.session.commit()
sched_favorite(api, in_reply_to_tweet_id)
sched_follow(api, in_reply_to_user_id)
sched_post_tweet(api, action, tweet_text, user_id, in_reply_to_tweet_id)
def home():
start, amount = pager_args()
articles = Article().get_paged(start=start, amount=amount)
tot_count = Article().get_count_filtered()
tweet_count = Tweet().get_tweetcount_filtered()
payload = render_template(
'home.html',
articles=articles,
start=int(start),
amount=int(amount),
tot_count=tot_count,
tweet_count=tweet_count,
cssver=os.path.getmtime(
os.path.join(os.path.dirname(__file__), 'static/thcrrspndnt.css')),
site=settings.CONFIG.get('site', 'thecorrespondent.com'),
version=settings.corres_version,
)
return payload
def _process_tweet(self, status):
'''
Procesa un tweet.
:param status: Es un objeto tweepy.Status de donde se extaerá la información del tweet.
:return: Devuelve el tweet procesado, una instancia de la clase Tweet, o None si hubo
un error al procesar la información del tweet.
'''
try:
# ID del tweet.
id = status.id_str
# Cuerpo del tweet
text = status.text
# Es un retweet ?
is_retweet = hasattr(status, 'retweeted_status')
# Es una reply ?
is_reply = hasattr(
status, 'in_reply_to_status_id'
) and not status.in_reply_to_status_id is None
post_type = 'retweet' if is_retweet else (
'reply' if is_reply else 'original')
# Número de retweets
num_retweets = status.retweet_count
# Timestamp UNIX
timestamp = status.created_at.strftime('%s')
# ID del tweet retweeteado.
retweet_id = status.retweeted_status if is_retweet else None
# ID del tweet respondido.
reply_id = status.in_reply_to_status_id if is_reply else None
# Info del usuario.
user_status = status.user
author = self._process_user(user_status)
if author is None:
raise Exception()
tweet = Tweet(author, id, text, post_type, num_retweets,
timestamp, retweet_id, reply_id)
return tweet
except:
pass
return None
def get_tweets_from_file(file_location):
"""
Haalt tweets op uit bestand en geeft terug als json
:param file_location: str
:return: List(Tweet)
"""
try:
tweets = []
data = TwitterIO.read_file(file_location)
for tweet in data:
tweets.append(
Tweet(tweet['id'], tweet['datetime'], tweet['message']))
return tweets
except IOError:
print('Bestand niet gevonden.')
except Exception as e:
print(
'Er is iets fout gegaan bij het ophalen van alle tweets uit een bestand'
)
print(e.message)
def search():
query = request.args.get('query')
if not query:
return home()
start, amount = pager_args()
# split the query on whitespace, but keep quoted strings together
tokens = re.findall('\w+|"[\w\s]*"', query)
articles = Article().get_search(tokens, start=start, amount=amount)
tot_count = Article().get_searchcount(tokens)
tweet_count = Tweet().get_tweetcount_searchquery(tokens)
payload = render_template('search.html',
articles=articles,
start=int(start),
amount=int(amount),
version=settings.corres_version,
tot_count=tot_count,
tweet_count=tweet_count,
tokens=tokens,
cssver=os.path.getmtime(
os.path.join(os.path.dirname(__file__),
'static/thcrrspndnt.css')))
return payload
def harvest(self):
twitter_session = OAuth1Session(
self.settings['CONSUMER_KEY'],
client_secret=self.settings['CONSUMER_SECRET'],
resource_owner_key=self.settings['OAUTH_KEY'],
resource_owner_secret=self.settings['OAUTH_SECRET'])
result = twitter_session.get(
'https://api.twitter.com/1.1/search/tweets.json',
params={
'q': self.query,
'count': 100,
'result_type': 'recent'
})
tweets = result.json()
print("\nHarvesting %s" % datetime.datetime.now(), end=' ')
if 'statuses' in tweets:
for tweet in tweets['statuses']:
parsed_tweet = Tweet.parse_json(tweet)
if parsed_tweet:
# it's up to the article to decide what to do with the found URL
Article.maybe_find_or_create(parsed_tweet.corres_url)
else:
print('No tweets found?')
def run(self):
# init api
# https://github.com/bear/python-twitter
api = twitter.Api(consumer_key=self.apiKey.consumer_key,
consumer_secret=self.apiKey.consumer_secret,
access_token_key=self.apiKey.access_token_key,
access_token_secret=self.apiKey.access_token_secret)
# search for hashtag
search = api.GetSearch(self.hash_tag)
for tweet in search:
# extract time
time = tweet.created_at_in_seconds
# compare if time of the current tweet is less the our desirable time then go back to for loop
if self.time_start >= time:
continue
message = tweet.text
author = tweet.user.name
# convert into tweeter_obj and store it in the array
tweet_obj = Tweet(author, time, message)
self.tweets.append(tweet_obj)
def register(tweet: Tweet) -> None:
tweet_table_name = get_tweet_table_name()
tweet_table = Table(tweet_table_name)
tweet_table.put_item(tweet.to_dict())
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from data_access.db_access import Db_Access
from model.tweet import Tweet
from datetime import date
# instantiate a database access
db = Db_Access()
# create a list of tweets for test
tweet_list = []
# fill the list with tweets
tweet_list.append(
Tweet('323K2K3L2', 'tweet 1', 'positif', 0.6, 'Samsung', date(1292, 4,
26)))
tweet_list.append(
Tweet('Y9SD8DS0D', 'tweet 2', 'neutral', 0.5, 'Samsung', date(1292, 4,
26)))
tweet_list.append(
Tweet('2LSD9809S', 'tweet 3', 'negative', 0.2, 'Samsung',
date(1292, 4, 26)))
# insert tweet list
# db.insert_tweet(tweet_list)
# get all tweets
tweets = db.get_all(Tweet)
# get tweets by hashtag
# tweets_by_hashtag = db.get_tweets_by_hashtag("Samsung")
def get_model(self):
"""
:rtype: model.tweet.Tweet
"""
return Tweet()