def get(self, request, *args, **kwargs):
"""
The GET method controller for the API endpoint and saves the results
to a CSV file.
Accepts query strings:
- keyword: the keyword that searched for
- count: number of results retrieved (default = 100)
"""
keyword = request.GET.get('keyword')
count = request.GET.get('count', 100)
if not keyword:
return Response(HTTP_400_BAD_REQUEST)
auth = twitter_auth()
api = API(auth, parser=JSONParser())
data = []
results = api.search(q=keyword, count=count)
filename = 'search_{}_{}.csv'.format(keyword,
datetime.now().isoformat())
for result in results['statuses']:
data.append({
'user': parse_tweet(result['user']['name']),
'tweet': parse_tweet(result['text'])
})
save_to_csv(filename, data)
response_text = 'Saved {} objects.'.format(
results['search_metadata']['count'])
return Response(response_text)
def get_tweets_info(self, status_ids, output_file='replies.tsv'):
"""Get the info of the list of tweet ids from twitter api into a tsv file
The file has one json entry per line
:param status_ids:
:param output_file:
:return:
"""
print('fetching the tweets via API from the file: ' + output_file)
auth = tweepy.OAuthHandler(TwitterCollector.consumer_key,
TwitterCollector.consumer_secret)
auth.set_access_token(TwitterCollector.access_key,
TwitterCollector.access_secret)
api = tweepy.API(auth_handler=auth,
parser=JSONParser(),
wait_on_rate_limit=True,
wait_on_rate_limit_notify=True)
l = []
with codecs.open(os.path.join(self.base_folder, output_file),
'w',
encoding='utf8') as outFile:
chunks = chunk_array(status_ids, 100)
for chunk in chunks:
rst = api.statuses_lookup(id_=chunk)
for tweet in rst:
outFile.write(json.dumps(tweet) + "\n")
def get_person(person, count):
consumer_key = "vAFqO3jFbVUXkSVKSitaUiU4Q"
consumer_secret = "4aG1sNq2pwDjEvpvG7FbR63GMCrQQfbTkP8Z0kdrLvkE8MCv8A"
access_key = "711632049711554560-Co5AtwXqrg5Qf3m2TwAzHnw6C5rWfxS"
access_secret = "VXMe4nJKHX1mb8QNh5LW3TMAdZSQ0L9FuwZADlG6GxHFB"
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
api = tweepy.API(auth, parser=JSONParser())
query = person
max_tweets = count
# запрос твитов
new_tweets = api.search(q=query, count=count)
# json для записи исходных твитов в базу данных
person = person.replace(" ", "_").lower()
file_name = 'JSON/raw_data_' + translit.translit(person)
file = open(file_name + '.json', 'w')
json_str = json.dumps(new_tweets['statuses'], indent=4, sort_keys=True)
file.write(u''.join(json_str))
file.close()
# список для приведения твитов в "человеческий" вид
all_tweets = list()
for tweet in new_tweets["statuses"]:
all_tweets.append(tweet["text"])
prepare_to_learn(all_tweets, translit.translit(person))
def rebuild(source_path):
conf = load_conf()
auth = OAuthHandler(conf["twitter"]["api"]["app"]["key"],
conf["twitter"]["api"]["app"]["secret"])
auth.set_access_token(conf["twitter"]["api"]["app"]["token"],
conf["twitter"]["api"]["app"]["token_secret"])
twitter = Twitter(auth, parser=JSONParser())
source_tweets = read(source_path)
print("%d tweets found" % len(source_tweets))
estimated = conf["twitter"]["api"]["pause"] * len(source_tweets)
print("dataset rebuilding would require about %s..." %
timedelta(seconds=estimated))
target_path = re.sub("\.json$", "-full.json", source_path)
tweets = []
try:
for tweet in source_tweets:
try:
tweets.append(rebuilt_tweet(tweet, twitter))
if (len(tweets) % 20) == 0:
print("downloaded %d tweets" % len(tweets))
except TweepError as e:
print("Error retrieving tweet #%s: %s" % (tweet["id"], str(e)))
sleep(conf["twitter"]["api"]["pause"])
dump(tweets, target_path)
except KeyboardInterrupt:
print("Exit forced...")
print("rebuilt %d tweets to %s" % (len(tweets), target_path))
def init_tweeter():
TweeterWrapper = Init.TweeterConnection()
auth = OAuthHandler(TweeterWrapper.get_consumer_key(),
TweeterWrapper.get_consumer_secret())
auth.set_access_token(TweeterWrapper.get_access_token(),
TweeterWrapper.get_access_secret())
api = tweepy.API(auth, parser=JSONParser())
return api
def find_userID_from_userName(self, user_name):
api = API(self.auth, parser=JSONParser())
result = api.lookup_users(screen_names=[user_name])
if len(result) == 1:
return result[0]["id"]
else:
print("Multiple Users with username")
sys.exit()
def __init__(self):
with open('twitter_api_creds.json', "rb") as f:
self.conf = json.load(f)
self.auth = OAuthHandler(self.conf["twitter"]["api"]["app"]["key"],
self.conf["twitter"]["api"]["app"]["secret"])
self.auth.set_access_token(self.conf["twitter"]["api"]["app"]["token"],
self.conf["twitter"]["api"]["app"]["token_secret"])
self.twitter = Twitter(self.auth, parser=JSONParser())
def __init__(self):
self.consumer_key = 'UtIptL4lyPuXK8Nwbdf3rw'
self.consumer_secret = 'UjZ4gpT9y8imJmh63uf528SpBLmE4K6XjkR9DKw8zkk'
self.access_token = '1928368088-0Orce72vcWcRgXHrmxsU1vVNYUxfb3PUcGbL8y3'
self.access_secret = 'l1JCaFALvn3pFrMXgq9JLL7l93atUxWPcCvBsuJDQ'
self.auth = tweepy.auth.OAuthHandler(self.consumer_key,
self.consumer_secret)
self.auth.set_access_token(self.access_token, self.access_secret)
self.api = tweepy.API(self.auth, parser=JSONParser())
def handle(self, *args, **options):
t = tweepy.API(_read_tw_auth, parser=JSONParser())
mentions = t.mentions_timeline(count=200,
include_entities=True,
tweet_mode='extended')
for tweet in mentions:
if ('full_text' not in tweet) and (len(tweet['text']) == 140):
tweet = t.get_status(tweet['id'], tweet_mode='extended')
Vote.handle_tweet(tweet)
def twitterApi(auth, terms):
api = API(auth, parser=JSONParser())
parsed_data = api.search(terms)
with open("outputAPI.txt", "wb") as f:
for i in range(len(parsed_data['statuses'])):
output = '@' + parsed_data['statuses'][i]['user'][
'screen_name'] + ': ' + parsed_data['statuses'][i]['text']
output = output.replace('\n', '') + '\n'
output = output.encode('utf-8')
print(output)
f.write(output)
def __init__(self):
# Keys for Twitter API (maybe reading it from a .txt)
self.consumer_key = 'EfbgNEMgmXNSweNDcWmoaSwm0'
self.consumer_secret = 'u3HlNeQNhG4whVzbilCxvswfJTMLG4ppxisaqtB4exHvGgDxsc'
self.access_token_key = '3940337423-CC2NFNG4zX9t3Z4Hl5vAbseYmlhlz6CXbuDlQNr'
self.access_token_secret = 'tmK2f3ZPrOWSkqY2bzu9St0LqDzJVIp5IV8PWPwENh69z'
self.auth = tweepy.OAuthHandler(self.consumer_key, self.consumer_secret)
self.auth.set_access_token(self.access_token_key, self.access_token_secret)
self.api = tweepy.API(self.auth, parser = JSONParser())
def get_api(self):
if not hasattr(self, '_api'):
oauth = tweepy.OAuthHandler(
settings.CONNECTED_ACCOUNTS_TWITTER_CONSUMER_KEY,
settings.CONNECTED_ACCOUNTS_TWITTER_CONSUMER_SECRET)
oauth.set_access_token(self.account.get_token(),
self.account.get_token_secret())
self._api = tweepy.API(oauth, parser=JSONParser())
return self._api
def init_with_settings(cls,
consumer_key=None,
consumer_secret=None,
access_token_key=None,
access_token_secret=None,
**kwargs):
auth = cls.auth_from_settings(consumer_key, consumer_secret,
access_token_key, access_token_secret)
if 'parser' not in kwargs:
kwargs['parser'] = JSONParser()
api = cls.make_api(auth_handler=auth, **kwargs)
return api
def __init__(self, account, rate_limit_buffer=1, **kwargs):
auth = OAuthHandler(current_app.config['TWITTER_CONSUMER_KEY'],
current_app.config['TWITTER_CONSUMER_SECRET'])
auth.set_access_token(account.access_token_key,
account.access_token_secret)
self.rate_limit_buffer = rate_limit_buffer
kwargs['parser'] = JSONParser()
kwargs['api_root'] = '/1.1'
self.api = API(auth, **kwargs)
self.rate_limits = {}
def authenticated_api(username, api_root=None, parser=None):
"""Return an oauthenticated tweety API object."""
auth = tweepy.OAuthHandler(settings.TWITTER_CONSUMER_KEY,
settings.TWITTER_CONSUMER_SECRET)
try:
user = User.objects.get(username__iexact=username)
sa = user.social_auth.all()[0]
auth.set_access_token(sa.tokens['oauth_token'],
sa.tokens['oauth_token_secret'])
return tweepy.API(auth,
api_root=api_root or settings.TWITTER_API_ROOT,
parser=parser or JSONParser())
except:
return None
def get_search_tweets(search_topic, tweet_count):
consumer_key = "aaaaaaaaaaaaaaaa"
consumer_secret = "aaaaaaaaaaaaaaaaaaaaaaaaaa"
access_token = "aaaaaaaaaaaaa-taaaaaaaaaaaaaaaaaaaaaaaaaaaa"
access_token_secret = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth, parser=JSONParser())
res_json = api.search(q=search_topic, lang="en", count=tweet_count)
for res in res_json['statuses']:
with open("tweets/twitter.txt", "a", encoding="utf_16") as f:
f.write(res['text'].replace("\n", " ") + " \n")
print(res['text'])
print("--------------------------------")
def _authorize():
global auth
global api
CONSUMER_SECRET = open('Credentials/Consumer Secret.txt', 'r').read()
CONSUMER_KEY = open('Credentials/Consumer Key.txt', 'r').read()
#PASSWORD = open('Credentials/password.txt','r').read()
#USER = open('Credentials/user.txt','r').read()
ACCESS_TOKEN = open('Credentials/Access Token.txt', 'r').read()
ACCESS_TOKEN_SECRET = open('Credentials/Access Token Secret.txt',
'r').read()
auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
api = tweepy.API(auth, parser=JSONParser())
def post(self, url, data=None, json=None, **kwargs):
kwargs['auth'] = self.oauth
response = requests.post(url=url, data=data, json=json, **kwargs)
if response.status_code and not 200 <= response.status_code < 300:
try:
error_msg, api_error_code = \
JSONParser().parse_error(response.text)
except Exception:
error_msg = "Twitter error response: status code = %s" % response.status_code
api_error_code = None
if is_rate_limit_error_message(error_msg):
raise RateLimitError(error_msg, response)
else:
raise TweepError(error_msg, response, api_code=api_error_code)
return response
def handle(self, *args, **options):
t = tweepy.API(_read_tw_auth, parser=JSONParser())
for vote in Vote.objects.exclude(
tweet_id__isnull=True).order_by('-date'):
if len(vote.text) != 280:
continue
print(vote.text)
try:
tweet_obj = t.get_status(vote.tweet_id, tweet_mode='extended')
except tweepy.TweepError as e:
print(e)
continue
vote.text = tweet_obj.get('full_text', None) or tweet_obj.get(
'text', None)
print(vote.text)
vote.save()
def rebuild(source_tweets):
conf = load_conf()
auth = OAuthHandler(conf["twitter"]["api"]["app"]["key"],
conf["twitter"]["api"]["app"]["secret"])
auth.set_access_token(conf["twitter"]["api"]["app"]["token"],
conf["twitter"]["api"]["app"]["token_secret"])
twitter = Twitter(auth, parser=JSONParser())
try:
for tweet in source_tweets:
try:
tw = rebuilt_tweet(tweet, twitter)
if tw is not None:
tweets.append(tw)
if (len(tweets) % 20) == 0:
print("Downloaded %d Tweets" % len(tweets))
except TweepError as e:
print("Error retrieving tweet #%s: %s" % (tweet["id"], str(e)))
sleep(conf["twitter"]["api"]["pause"])
except KeyboardInterrupt:
print("Exit forced...")
def create_twitter_client() -> API:
"""
tweepyのAPIを生成する
:return: API
"""
# Twitter検索用のクライアント生成
auth = OAuthHandler(config.CONSUMER_KEY, config.CONSUMER_SECRET)
auth.set_access_token(config.ACCESS_TOKEN_KEY, config.ACCESS_TOKEN_SECRET)
# JSONで結果を受け取りたいので、JSONParserを設定する。
# 検索の上限に達してもライブラリ側でよろしくやってくれる。はず。
twitter_api = API(auth,
parser=JSONParser(),
wait_on_rate_limit=True,
wait_on_rate_limit_notify=True)
if twitter_api is None:
logger.error("Can't Authenticate")
sys.exit(-1)
return twitter_api
def enable_activity(self, *args, **kwargs):
""" :reference: https://developer.twitter.com/en/docs/accounts-and-users/subscribe-account-activity/api-reference/aaa-premium
:allowed_param:'url', 'env'
"""
post_data = {}
env = kwargs.pop('env')
if env is not None:
apiPath = '/account_activity/all/' + env + '/webhooks.json'
else:
apiPath = '/account_activity/all/webhooks.json'
self.parser = JSONParser()
return bind_api(
api=self,
path=apiPath,
method='POST',
payload_type='status',
allowed_param=['url'],
require_auth=True,
use_cache=False,
)(post_data=post_data, *args, **kwargs)
def parse(self, method, payload):
try:
if method.payload_type is None:
return
model = BRSearchResults
except Exception as e:
print(e)
json = JSONParser.parse(self, method, payload)
if isinstance(json, tuple):
json, cursors = json
else:
cursors = None
if method.payload_list:
result = model.parse_list(method.api, json)
else:
result = model.parse(method.api, json)
if cursors:
return result, cursors
else:
return result
def __init__(self):
'''
Class constructor or initialization method.
'''
# keys and tokens from the Twitter Dev Console
consumer_key = twitter_credentials.CONSUMER_KEY
consumer_secret = twitter_credentials.CONSUMER_SECRET
access_token = twitter_credentials.ACCESS_TOKEN
access_token_secret = twitter_credentials.ACCESS_TOKEN_SECRET
# attempt authentication
try:
# create OAuthHandler object
self.auth = OAuthHandler(consumer_key, consumer_secret)
# set access token and secret
self.auth.set_access_token(access_token, access_token_secret)
# create tweepy API object to fetch tweets
#self.api = tweepy.API(self.auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)
self.api = tweepy.API(self.auth,
parser=JSONParser(),
wait_on_rate_limit=True,
wait_on_rate_limit_notify=True)
except:
print("Error: Twitter Authentication Failed")
def __init__(self, model_factory=None):
JSONParser.__init__(self)
def lambda_handler(event, context):
client = boto3.resource('dynamodb')
client_comprehend = boto3.client('comprehend')
print("Get credentials")
consumer_key = os.getenv("CONSUMER_KEY")
consumer_secret = os.getenv("CONSUMER_SECRET")
access_token = os.getenv("ACCESS_TOKEN")
access_token_secret = os.getenv("ACCESS_TOKEN_SECRET")
print("Authenticate")
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth, parser=JSONParser())
print("Get search from csv file")
tweets_file = ROOT / "possible_disasters.csv"
recent_tweets = api.user_timeline()[:3]
search = get_tweet(tweets_file)
print(f"Search tweet: {search}")
returned_tweets = api.search(search, result_type='top', count=50)
dynamodb = boto3.client('dynamodb')
table = client.Table("tweets")
print('printing table status..\n')
print(table.table_status)
for i in range(len(returned_tweets["statuses"])):
tweet = returned_tweets["statuses"][i]
lang = tweet['lang']
if lang is None:
continue
if lang not in [
'ar', 'hi', 'ko', 'zh-TW', 'ja', 'zh', 'de', 'pt', 'en', 'it',
'fr', 'es'
]:
continue
tweet_id = tweet["id_str"]
text = tweet["text"]
str_arr = search.split(", ")
location = str_arr[0]
disaster = str_arr[1]
date = tweet['created_at']
sentiment_data = client_comprehend.detect_sentiment(Text=text,
LanguageCode=lang)
sentiment_score = Decimal(
sentiment_data["SentimentScore"]["Positive"] -
sentiment_data["SentimentScore"]["Negative"]).quantize(
Decimal('1.00'))
table.put_item(
Item={
'tweet_id': tweet_id,
'location#disaster': f"{location}+{disaster}",
'location': location,
'disaster': disaster,
'date': date,
'sentiment_score': sentiment_score
})
json_str = json.dumps(returned_tweets)
return {"statusCode": 200, "tweets": json_str}
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
from tweepy import API
from tweepy import Cursor
from tweepy.parsers import JSONParser
access_token = "YOUR ACCESS TOKEN"
access_token_secret = "YOUR ACCESS TOKEN SECRET"
consumer_key = "YOUR CONSUMER KEY"
consumer_secret = "YOUR CONSUMER SECRET"
if __name__ == '__main__':
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = API(auth)
uid = 'user to search'
query = 'to:' + uid
out = api.search(q=query, parser=JSONParser(),rpp=10,count=10,tweet_mode="extended")
print(out)
def handle(self, *args, **options):
t = tweepy.API(_read_tw_auth, parser=JSONParser())
mentions = t.mentions_timeline(count=200, include_entities=True)
for tweet in mentions:
Vote.handle_tweet(tweet)
consumer_secret = ssm.get_parameter(
Name='/jtweets/tweepy/consumer_secret',
WithDecryption=True).get("Parameter").get("Value")
access_token = ssm.get_parameter(
Name='/jtweets/tweepy/access_token',
WithDecryption=True).get("Parameter").get("Value")
access_token_secret = ssm.get_parameter(
Name='/jtweets/tweepy/access_token_secret',
WithDecryption=True).get("Parameter").get("Value")
# login
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
# set up tweepy
api = tweepy.API(auth, parser=JSONParser())
# get the most recent triplejplays tweet
search_results = api.user_timeline(screen_name="triplejplays", count=10000)
# for testing only!
#search_results[5]["text"]="Live At The Wireless - Two Door Cinema Club (Splendour In The Grass, Byron Bay Parklands 2017) [20:02]"
# iterate through results, convert timezone and dump out tweet details
for tweet in search_results:
# keep for debugging only - so can compare
# thisText = u' '.join((tweet["user"]["screen_name"],tweet["text"])).encode('utf-8')
# print tweet["id"] , unescape(thisText)
# the strings that come through are not unicode for some reason
tweet["text"] = tweet["text"].encode('utf-8')
####### Access Information #################
# Parameter you need to specify
consumer_key = 'QtJ2zFy4kMmPrJKWQpCUvlgpI'
consumer_secret = 'CpH8C0IHlhnAI9PjWkDxYr3RUVb6rf7jspNml70vFvM4T27GbO'
access_key = '126251520-feNdhpCqglrLDTkqp4B6UqESILVXgHO2KrLWmknD'
access_secret = 'Z0i6VHHxeO1Xh05BIwgrvLW1ZyUFDfm4WaPEnkZq1eMNN'
inputFile = 'tweet_id'
outputFile = 'tweet.json'
#############################################
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
api = tweepy.API(auth_handler=auth, parser=JSONParser())
l = []
with open(inputFile, 'r') as inFile:
with codecs.open(outputFile, 'w', encoding='utf8') as outFile:
for line in inFile.readlines():
l.append(line.rstrip())
if len(l) >= 99:
rst = api.statuses_lookup(id_=l)
for tweet in rst:
outFile.write(json.dumps(tweet) + "\n")
l = []
if len(l) > 0:
rst = api.statuses_lookup(id_=l)
for tweet in rst:
outFile.write(json.dumps(tweet) + "\n")
