def database():
search_term = "bills.com"
print search_term
api = Api(consumer_key='uy3Utit3PhWPmM5284sh7w',
consumer_secret='zA6ps76R3ID2oRXXdUjQ7jsPgcmXD1rPsNYVSw',
access_token_key='61793468-PoarjfBLEa8KxtmFvNcLwFlvSGsh03U2LyNbroXye',
access_token_secret='qg6cEIa33rX725oVAqmN4CtIgG0fmuPeI7pb2Vjeyxs')
results = api.GetSearch(search_term)
tweets = []
for r in results:
user = r.GetUser()
text = r.GetText()
seconds_ago = r.GetCreatedAtInSeconds()
hours_ago = ConvertTime(seconds_ago)
source_url = r.GetSource()
screenname = user.GetScreenName()
image = user.GetProfileImageUrl()
tweet = {"text": text, "hours_ago":hours_ago, "screenname": screenname,
"img": image, "view_tweet": source_url}
tweets.append(tweet)
#Add each tweet to database.
t = Tweet( author= screenname, body = text, is_read = False )
db.session.add(t)
for t in tweets:
print t
db.session.commit()
return "Connected to db. Check db."
def search_loop(api: twitter.Api, logger=None, *args, **kwargs):
result = []
max_id = None
loop_count = 0
while True:
loop_count += 1
tweets = api.GetSearch(*args, **kwargs, count=100, max_id=max_id)
if logger:
print(f'search{loop_count}:{str(len(tweets))} tweets')
if not tweets:
break
else:
result.extend(tweets)
max_id = tweets[-1].id
return sorted(list(set(result)), key=lambda x: x.created_at_in_seconds)
def pull_and_publish(
consumer_key: str,
consumer_secret: str,
access_token: str,
access_token_secret: str,
slack_token: str,
slack_channel: str,
wait_time: int,
):
"""Continuously pull recent Twitter statuses and publish them to Slack."""
twitter_api = Api(consumer_key, consumer_secret, access_token,
access_token_secret)
slack_client = WebClient(slack_token)
users_to_exclude = ['Ethereum_MXN', 'someOtherUser', 'someOtherUser2']
since_id = None
while True:
#statuses = twitter_api.GetSearch(raw_query="q=-Precio%20(from%3Abitso)%20(to%3Abitso)%20(%40bitso)&src=typed_query&f=live")
statuses = twitter_api.GetSearch(
raw_query="q=bitso%20&result_type=recent&since_id={}".format(
since_id))
if statuses:
logger.info(f"Got {len(statuses)} statuses from Twitter.")
count = 0
publishable_statuses = set()
for status in reversed(statuses):
user = status.user
screen_name = user.screen_name
if any(u in screen_name for u in users_to_exclude):
count += 1
else:
twitter_link = f"http://twitter.com/{user.screen_name}/status/{status.id}"
_publish(slack_channel, slack_client, twitter_link, user)
since_id = status.id
logger.info(f"Skipped {count} statuses from excluded users.")
else:
logger.info("No new twitter statuses.")
time.sleep(wait_time)
def get_twits_list(request):
request.content_type = 'application/json'
request_id = request.json.get('requestId')
version = request.json.get('version')
queries = request.json.get('queries')
period = request.json.get('period')
queries_search = {}
for word in queries:
queries_search[word] = 'q=' + word.replace('#', '%23') + '%20'
queries_search[word] += 'since%3A' + period['fromDate'] + '%20until%3A' + period['toDate'] + '&count=100'
api = Api(consumer_key=CONSUMER_KEY,
consumer_secret=CONSUMER_SECRET,
access_token_key=ACCESS_TOKEN,
access_token_secret=ACCESS_TOKEN_SECRET)
tweets = {}
rv = {
'requestId': request_id,
'version': version,
'grabber': {
'name': 'TwitterGrabber',
'version': version,
'authors': [
{
'name': 'Иларионов Дмитрий',
'email': '*****@*****.**'
}
],
'source': {
'title': 'Твиттер',
'url': 'https://twitter.com',
'logo': twitter_image_link
}
},
'results': [
]
}
try:
for word in queries:
tweets[word] = api.GetSearch(raw_query=queries_search[word])
response.body = return_result(rv, queries, tweets)
return return_result(rv, queries, tweets)
except Exception as e:
response.body = return_error(rv, e)
return return_error(rv, e)
def index():
search_term = "bills.com"
print search_term
api = Api(consumer_key='uy3Utit3PhWPmM5284sh7w',
consumer_secret='zA6ps76R3ID2oRXXdUjQ7jsPgcmXD1rPsNYVSw',
access_token_key='61793468-PoarjfBLEa8KxtmFvNcLwFlvSGsh03U2LyNbroXye',
access_token_secret='qg6cEIa33rX725oVAqmN4CtIgG0fmuPeI7pb2Vjeyxs')
results = api.GetSearch(search_term)
print results[0]
tweets = []
for r in results:
user = r.GetUser()
text = r.GetText()
seconds_ago = r.GetCreatedAtInSeconds()
hours_ago = ConvertTime(seconds_ago)
source_url = r.GetSource()
print source_url
screenname = user.GetScreenName()
image = user.GetProfileImageUrl()
tweet = {"text": text, "hours_ago":hours_ago, "screenname": screenname,
"img": image, "view_tweet": source_url}
tweets.append(tweet)
return render_template("index.html", tweets = tweets)
class TwitterTrainer(Trainer):
"""
Allows the chat bot to be trained using data
gathered from Twitter.
:param random_seed_word: The seed word to be used to get random tweets from the Twitter API.
This parameter is optional. By default it is the word 'random'.
:param twitter_lang: Language for results as ISO 639-1 code.
This parameter is optional. Default is None (all languages).
"""
def __init__(self, storage, **kwargs):
super(TwitterTrainer, self).__init__(storage, **kwargs)
from twitter import Api as TwitterApi
# The word to be used as the first search term when searching for tweets
self.random_seed_word = kwargs.get('random_seed_word', 'random')
self.lang = kwargs.get('twitter_lang')
self.api = TwitterApi(
consumer_key=kwargs.get('twitter_consumer_key'),
consumer_secret=kwargs.get('twitter_consumer_secret'),
access_token_key=kwargs.get('twitter_access_token_key'),
access_token_secret=kwargs.get('twitter_access_token_secret'))
def random_word(self, base_word, lang=None):
"""
Generate a random word using the Twitter API.
Search twitter for recent tweets containing the term 'random'.
Then randomly select one word from those tweets and do another
search with that word. Return a randomly selected word from the
new set of results.
"""
import random
random_tweets = self.api.GetSearch(term=base_word, count=5, lang=lang)
random_words = self.get_words_from_tweets(random_tweets)
random_word = random.choice(list(random_words))
tweets = self.api.GetSearch(term=random_word, count=5, lang=lang)
words = self.get_words_from_tweets(tweets)
word = random.choice(list(words))
return word
def get_words_from_tweets(self, tweets):
"""
Given a list of tweets, return the set of
words from the tweets.
"""
words = set()
for tweet in tweets:
tweet_words = tweet.text.split()
for word in tweet_words:
# If the word contains only letters with a length from 4 to 9
if word.isalpha() and len(word) > 3 and len(word) <= 9:
words.add(word)
return words
def get_statements(self):
"""
Returns list of random statements from the API.
"""
from twitter import TwitterError
statements = []
# Generate a random word
random_word = self.random_word(self.random_seed_word, self.lang)
self.chatbot.logger.info(
'Requesting 50 random tweets containing the word {}'.format(
random_word))
tweets = self.api.GetSearch(term=random_word, count=50, lang=self.lang)
for tweet in tweets:
statement = Statement(tweet.text)
if tweet.in_reply_to_status_id:
try:
status = self.api.GetStatus(tweet.in_reply_to_status_id)
statement.in_response_to = status.text
statements.append(statement)
except TwitterError as error:
self.chatbot.logger.warning(str(error))
self.chatbot.logger.info('Adding {} tweets with responses'.format(
len(statements)))
return statements
def train(self):
for _ in range(0, 10):
statements = self.get_statements()
for statement in statements:
self.chatbot.storage.create(
text=statement.text,
in_response_to=statement.in_response_to,
conversation=statement.conversation,
tags=statement.tags)
class TwitterTrainer(Trainer):
"""
Allows the chat bot to be trained using data
gathered from Twitter.
"""
def __init__(self, storage, **kwargs):
super(TwitterTrainer, self).__init__(storage, **kwargs)
from twitter import Api as TwitterApi
self.api = TwitterApi(
consumer_key=kwargs.get('twitter_consumer_key'),
consumer_secret=kwargs.get('twitter_consumer_secret'),
access_token_key=kwargs.get('twitter_access_token_key'),
access_token_secret=kwargs.get('twitter_access_token_secret'))
def random_word(self, base_word='random'):
"""
Generate a random word using the Twitter API.
Search twitter for recent tweets containing the term 'random'.
Then randomly select one word from those tweets and do another
search with that word. Return a randomly selected word from the
new set of results.
"""
import random
random_tweets = self.api.GetSearch(term=base_word, count=5)
random_words = self.get_words_from_tweets(random_tweets)
random_word = random.choice(list(random_words))
tweets = self.api.GetSearch(term=random_word, count=5)
words = self.get_words_from_tweets(tweets)
word = random.choice(list(words))
return word
def get_words_from_tweets(self, tweets):
"""
Given a list of tweets, return the set of
words from the tweets.
"""
words = set()
for tweet in tweets:
# TODO: Handle non-ascii characters properly
cleaned_text = ''.join(
[i if ord(i) < 128 else ' ' for i in tweet.text])
tweet_words = cleaned_text.split()
for word in tweet_words:
# If the word contains only letters with a length from 4 to 9
if word.isalpha() and len(word) > 3 and len(word) <= 9:
words.add(word)
return words
def get_statements(self):
"""
Returns list of random statements from the API.
"""
from twitter import TwitterError
statements = []
# Generate a random word
random_word = self.random_word()
self.logger.info(
u'Requesting 50 random tweets containing the word {}'.format(
random_word))
tweets = self.api.GetSearch(term=random_word, count=50)
for tweet in tweets:
statement = Statement(tweet.text)
if tweet.in_reply_to_status_id:
try:
status = self.api.GetStatus(tweet.in_reply_to_status_id)
statement.add_response(Response(status.text))
statements.append(statement)
except TwitterError as error:
self.logger.warning(str(error))
self.logger.info('Adding {} tweets with responses'.format(
len(statements)))
return statements
def train(self):
for _ in range(0, 10):
statements = self.get_statements()
for statement in statements:
self.storage.update(statement, force=True)
def spotify_info(sitem):
artist = sitem['item']['artists'][0]['name']
song = sitem['item']['name']
write_line(u'Sptfy', 3)
write_line(song, 1)
write_line(artist, 2)
def disp_message(message):
write_line(u'Msg', 3)
write_line(message, 1)
while 1:
#print(u'ok')
metweet = api.GetSearch(u'#metronäyttö',since = strftime("%Y-%m-%d",gmtime()))
if len(metweet) > 0:
mtime = strptime(metweet[0].created_at, '%a %b %d %H:%M:%S +0000 %Y')
if mktime(gmtime()) - mktime(mtime) < 3600:
metweet = metweet[0].text.replace(u'#metronäyttö',u'')
disp_message(metweet)
if sp.currently_playing() != None:
if sp.currently_playing()['is_playing']==True:
spotify_info(sp.currently_playing())
else:
for cc in chromecasts:
cc.wait()
mc = cc.media_controller.status
cc.media_controller._fire_status_changed()
if mc.player_state == u'PLAYING':
from collections import defaultdict
maxid = 0
search_word = "#XXXXXX"
api = Api(base_url="https://api.twitter.com/1.1",
consumer_key='XXXXXXXX',
consumer_secret='XXXXXXXX',
access_token_key='XXXXXXXX',
access_token_secret='XXXXXXXX')
count = 0
file = open(name='../data/result.json', mode='w')
found = api.GetSearch(term=search_word,
count=100,
lang="ja",
result_type='mixed',
until="yyyy-mm-dd")
while count < 3000:
for result in found:
file.write(str(result) + os.linesep)
count += 1
maxid = result.id
found = api.GetSearch(term=search_word,
count=100,
result_type='mixed',
max_id=maxid - 1)
file.close()
class TweeterSearcher:
"""
ツイッターの情報を管理するデーターベース
"""
def __init__(self, consumer_key, consumer_secret, access_token_key,
access_token_secret):
self._api = Api(base_url="https://api.twitter.com/1.1",
consumer_key=consumer_key,
consumer_secret=consumer_secret,
access_token_key=access_token_key,
access_token_secret=access_token_secret)
self.UpdateRateLimitStatus()
self._tweets = []
self._users = {}
def UpdateRateLimitStatus(self):
self._rate_limit = self._api.GetRateLimitStatus()
def GetSearchRemaining(self):
"""
GetSearchの残り実行可能回数を取得
"""
search = self._rate_limit['resources']['search']
return search['/search/tweets']['remaining']
def GetSearchReset(self):
search = self._rate_limit['resources']['search']
return time.localtime(search['/search/tweets']['reset'])
def StartSearch(self, term, since_id=None, max_id=None):
self._tweets = []
self._users = {}
self.max_id = 0
self.min_id = 0
ret = self._GetSearch(term, since_id, max_id)
while True:
if ret == -1:
return False
if ret == 0:
return True
ret = self._GetSearch(term, since_id, self.min_id - 1)
def GetTweets(self):
return self._tweets
def _GetSearch(self, term, since_id, max_id):
"""
Twitterの検索を行う
@param term 検索文字
@param since_id 検索開始ID このID以上のIDが検索される
@param max_id 検索最大ID このID以下のIDのみ検索される
@return 検索した最低ID -1の場合はAPIの上限 ０はすべて検索できた
"""
if self.GetSearchRemaining() == 0:
return -1
search = self._rate_limit['resources']['search']
--search['/search/tweets']['remaining']
try:
found = self._api.GetSearch(term=term,
count=100,
result_type='recent',
lang='ja',
max_id=max_id,
since_id=since_id)
except twitter.TwitterError, ex:
print ex
return -1
for f in found:
if self.min_id > f.id or self.min_id == 0:
self.min_id = f.id
if self.max_id < f.id or self.max_id == 0:
self.max_id = f.id
rec = {
'id': f.id,
'text': f.text,
'user_id': f.user.id,
'createdtime': f.created_at
}
self._tweets.append(rec)
return len(found)
