class TwitterNotificationService(BaseNotificationService):
"""Implementation of a notification service for the Twitter service."""
def __init__(self, consumer_key, consumer_secret, access_token_key,
access_token_secret, username):
"""Initialize the service."""
from TwitterAPI import TwitterAPI
self.user = username
self.api = TwitterAPI(consumer_key, consumer_secret, access_token_key,
access_token_secret)
def send_message(self, message="", **kwargs):
"""Tweet a message."""
if self.user:
resp = self.api.request(
'direct_messages/new', {'text': message, 'user': self.user})
else:
resp = self.api.request('statuses/update', {'status': message})
if resp.status_code != 200:
import json
obj = json.loads(resp.text)
error_message = obj['errors'][0]['message']
error_code = obj['errors'][0]['code']
_LOGGER.error("Error %s : %s (Code %s)", resp.status_code,
error_message,
error_code)
def fetch(self):
count = 0
api = TwitterAPI(self.consumerKey, self.consumerSecret, self.AccessKey, self.AccessToken, auth_type='oAuth2')
r = api.request('search/tweets', {'q': 'AAPL', 'count':1, 'result_type': 'recent'})
for item in r:
count+=1
print(count, item['id'], "\t", item['created_at'] if 'text' in item else item + '\n')
self.lastId = item['id']-1
temp = item['id']
while self.lastId > (self.previousLast + 1):
r = api.request('search/tweets',
{'q': 'AAPL', 'count':10, 'result_type': 'recent',
'max_id': self.lastId})
for item in r:
count+=1
print(count, item['id'], "\t", item['created_at'] if 'text' in item else item + '\n')
print ("Done up to: ", item['id'], "\t", self.lastId, "\t", self.previousLast)
self.lastId = item['id'] - 1
self.previousLast = temp
print ("Done upto ", self.previousLast)
def tweet_movie(fname, which_fun):
bytes_sent = 0
total_bytes = os.path.getsize(fname)
file = open(fname, 'rb')
api = TwitterAPI(t_keys['CONSUMER_KEY'], t_keys['CONSUMER_SECRET'], t_keys['ACCESS_KEY'], t_keys['ACCESS_SECRET'])
r = api.request('media/upload', {'command':'INIT', 'media_type':'video/mp4', 'total_bytes':total_bytes})
check_status(r)
media_id = r.json()['media_id']
segment_id = 0
while bytes_sent < total_bytes:
chunk = file.read(4*1024*1024)
r = api.request('media/upload', {'command':'APPEND', 'media_id':media_id, 'segment_index':segment_id}, {'media':chunk})
check_status(r)
segment_id = segment_id + 1
bytes_sent = file.tell()
print('[' + str(total_bytes) + ']', str(bytes_sent))
r = api.request('media/upload', {'command':'FINALIZE', 'media_id':media_id})
check_status(r)
r = api.request('statuses/update', {'status': which_fun, 'media_ids':media_id})
check_status(r)
print("Tweeted about " + which_fun)
return(r)
def execute(PLAYER_ONE, PLAYER_TWO):
api = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN_KEY, ACCESS_TOKEN_SECRET)
count = 30
lang = "en"
# List of all the tweets
collection1 = []
collection2 = []
followerCount1 = []
followerCount2 = []
r1 = api.request("search/tweets", {"lang": lang, "q": PLAYER_ONE, "count": count, "since": last_sunday})
for item in r1:
ID = item["user"]["id"]
followCount = getFollowerCount(ID)
followerCount1.append(followCount)
collection1.append(item["text"])
print
print
r2 = api.request("search/tweets", {"lang": lang, "q": PLAYER_TWO, "count": count, "since": last_sunday})
for item in r2:
ID = item["user"]["id"]
followCount = getFollowerCount(ID)
followerCount2.append(followCount)
collection2.append(item["text"])
score1 = calculate_sentiment(collection1, followerCount1)
score2 = calculate_sentiment(collection2, followerCount2)
print score1
print score2
if score1 < 0 and score2 > 0:
return write_json(PLAYER_ONE, PLAYER_TWO, 0, 100)
elif score1 > 0 and score2 < 0:
return write_json(PLAYER_ONE, PLAYER_TWO, 100, 0)
combinedScore = score1 + score2
print float(score1) / combinedScore
return write_json(PLAYER_ONE, PLAYER_TWO, float(score1) / combinedScore * 100, float(score2) / combinedScore * 100)
def get_happy_sad_count(location):
"""
Gets the happy and sad count on the current query
Returns the happy count and the sad count
"""
api = TwitterAPI('SSuspEaugDP7vcITlMA', '6FdUGOTcqlcoufjz62frkKHJ5cgm1vnaIHqjlboOHg', '78925893-xs57TGEPdYpSmbfBKi1XPKVliu5i2TrDRFOEPwVmt', 'fAVXZr4cnEi2hNtVvLyh68mxh4JysMbyJVlBbefhLvf5v')
happy_count = 0
sad_count = 0
happy_query = {'q':':)', 'lang':'en', 'count':'100', 'result_type':'recent'}
sad_query = {'q':':(', 'lang':'en', 'count':'100', 'result_type':'recent'}
if location:
happy_query['locations'] = '{0}, {1}'.format(location[0], location[1])
sad_query['locations'] = '{0}, {1}'.format(location[0], location[1])
happy = api.request('search/tweets', happy_query)
sad = api.request('search/tweets', sad_query)
h_list = []
s_list = []
for item in happy:
happy_count += 1
h_list.append(item)
for item in sad:
sad_count += 1
s_list.append(item)
tweet = (h_list[int(random()*len(h_list))]['text'], "positive")
if random()*2 < 1:
tweet = (s_list[int(random()*len(s_list))]['text'], "negative")
return happy_count, sad_count, tweet
class TwitterClient(object):
"""This class is used for the communication with the
Twitter API"""
def __init__(self, options):
self.api = TwitterAPI(\
options['consumer_key'],\
options['consumer_secret'],\
options['access_token_key'],\
options['access_token_secret'])
def _sleep(self):
sleep(10)
def get_followers(self):
followers = []
for id in self.api.request('followers/ids'):
followers.append(id)
self._sleep()
return followers
def get_following(self):
following = []
for id in self.api.request('friends/ids'):
following.append(id)
self._sleep()
return following
def follow_user(self, user_id):
r = self.api.request('friendships/create', {'user_id': user_id })
if r.status_code == 200:
status = r.json()
print 'followed %s' % status['screen_name']
self._sleep()
def unfollow_user(self, user_id):
r = self.api.request('friendships/destroy', {'user_id': user_id})
if r.status_code == 200:
status = r.json()
print 'unfollowed %s' % status['screen_name']
self._sleep()
def get_users_for_topic(self, topic):
r = self.api.request('search/tweets', {
'count':200,
'q':topic,
'lang':"de"})
statuses = r.json()['statuses']
how_many = len(statuses)
print("fetched " + str(how_many) + " tweets")
self._sleep()
users = map(lambda s : s['user'], statuses)
users_id_unique = set()
distinct = []
for user in users:
if(user['id_str'] not in users_id_unique):
distinct.append(user)
users_id_unique.add(user['id_str'])
return distinct
def handle(self, *args, **options):
api = TwitterAPI(**settings.TWITTER_CREDENTIALS)
users = settings.EGA_ADMINS.keys()
# matches in the last hour
matches = Match.objects.filter(
when__isnull=False,
when__range=(now() - timedelta(minutes=30), now()))
for m in matches:
predictions = m.prediction_set.filter(
user__username__in=users).order_by('user')
if predictions:
data = ', '.join([
"%s dice %s %d-%d %s" % (
settings.EGA_ADMINS[p.user.username],
m.home.code, p.home_goals,
p.away_goals, m.away.code)
for p in predictions
if p.home_goals is not None and p.away_goals is not None])
tweet = u"En juego: %s vs %s\n%s" % (
m.home.name, m.away.name, data)
api.request('statuses/update', {'status': tweet})
# get predictions count and tweet
counts = get_top_predictions(m)
preds = '\n'.join([
'#%s %d - %d #%s (%d)' % (m.home.code, r[0], r[1],
m.away.code, r[2])
for r in counts
])
tweet = 'Los resultados más pronosticados:\n' + preds
api.request('statuses/update', {'status': tweet})
def share_yap_on_twitter(user,yap,twitter_access_token_key,twitter_access_token_secret):
twitter = TwitterAPI(consumer_key=settings.TWITTER_CONSUMER_KEY,consumer_secret=settings.TWITTER_CONSUMER_SECRET,access_token_key=twitter_access_token_key,access_token_secret=twitter_access_token_secret)
status = "@" + str(yap.user.username) + " yapped \"" + str(yap.title) + '\" - web.yapster.co/yap/' + str(yap.pk)
length_of_status = len(status)
if yap.picture_flag == True:
if length_of_status > 118:
extra_length_of_title = length_of_status - 121
title = yap.title[:(-extra_length_of_title)].upper() + "..." # -3 for three dots ...
elif length_of_status <= 118:
title = yap.title
b = connect_s3(bucket_name="yapsterapp")
yap_picture_key = b.get_key(yap.picture_path)
fp = StringIO.StringIO()
yap_picture_key.get_file(fp)
yap_picture = fp.getvalue()
r = twitter.request('statuses/update_with_media',{'status':status}, {'media[]': yap_picture})
elif yap.picture_flag == False:
if length_of_status > 140:
extra_length_of_title = length_of_status - 143
title = yap.title[:(-extra_length_of_title)].upper() + "..."
elif length_of_status > 140:
title = yap.title
r = twitter.request('statuses/update',{'status':status})
#print("Success" if r.status_code == 200 else 'Failure')
if r.status_code == 200:
return json.loads(r.text)['id']
else:
pass
def sendToTwitter(blurb, imgName):
# These tokens are necessary for twitter contact and must be obtained by creating
# an app on app.twitter.com, linking to an account, and setting perms to read/write
accToken = "XXXXX"
accTokenSecret = "XXXXX"
apiKey = "XXXXX"
apiSecret = "XXXXX"
api = TwitterAPI(apiKey, apiSecret, accToken, accTokenSecret)
if (imgName != "" and imgName != "fail"):
open_file = open(imgName, "rb")
r = api.request("statuses/update_with_media",
{"status": blurb},
{"media[]": open_file.read()})
if r.status_code is not 200:
# Try once more, truncating period
r = api.request("statuses/update_with_media",
{"status": blurb[:len(blurb)-1]},
{"media[]": open_file.read()})
if r.status_code is not 200:
print("Image tweet failed (status code " + str(r.status_code) +
"). Trying tweet without image...")
r = api.request("statuses/update",
{"status": blurb})
else:
r = api.request("statuses/update",
{"status": blurb})
return r.status_code
def post_video():
context = make_context(asset_depth=2)
video_id = request.args.get('vid_id')
movie = 'www/assets/%s.mp4' % video_id
nbytes = os.path.getsize(movie)
file = open(movie, 'rb')
data = file.read()
access_token = request.args.get('token')
access_secret = request.args.get('secret')
tweet = request.args.get('tweet')
print tweet
api = TwitterAPI(
'RCapaO1LKpGu7W5FsGRI4g',
'N6tavbh66HuGCguIiDff1Bi78aeHWTNpJraZK0pXGdo',
access_token,
access_secret
)
r = api.request(
'media/upload',
{
'command': 'INIT',
'media_type': 'video/mp4',
'total_bytes': nbytes
}
)
check_status(r)
media_id = r.json()['media_id']
r = api.request(
'media/upload',
{
'command': 'APPEND',
'media_id': media_id,
'segment_index': 0
},
{
'media': data
}
)
check_status(r)
r = api.request('media/upload', {
'command': 'FINALIZE',
'media_id': media_id
})
r = api.request('statuses/update', {
'status': tweet,
'media_ids': media_id
})
check_status(r)
return ('', 204)
def twitter_stream_loc_json(fn_app_auth, outfile,
num_tweets=0, xduration=0, bb=''):
""" (filename, list of strings, filename, int, int) -> None
fn_app_auth - file of twitter application credential in json format
outfile - filename for outputing tweets in json format
num_tweets - integer for the number of tweets to collect
xduration - integer for number of seconds to listen for tweets
bb - bounding box with lon, lat for SW followed by lon, lat for NE
>>>twitter_stream_json("c:/dev/twitter/auth/app_auth.json",
['#Transcendence', '#Divergent', '#AHauntedHouse2'],
"c:/dev/twitter/output/tweets_output.json",
1000, 0)
"""
with open(fn_app_auth) as json_file:
key = json.load(json_file)
#print(key)
api = TwitterAPI(key["consumer_key"], key["consumer_secret"],
key["access_token_key"], key["access_token_secret"])
counter=0
if num_tweets != 0:
# Stream Tweets:
r = api.request('statuses/filter', {'locations': bb})
with open(outfile, 'a') as outfile:
for item in r:
json.dump(item, outfile)
counter+=1
if counter < num_tweets:
outfile.write('\n')
#print counter
else:
break
elif xduration != 0:
stop = time.time()+xduration
# Stream Tweets:
r = api.request('statuses/filter', {'locations': bb})
with open(outfile, 'a') as outfile:
for item in r:
json.dump(item, outfile)
if time.time() < stop:
outfile.write('\n')
else:
break
def twitter_post(tweet_title="hi"):
con_k = u'uEmrTJlrsXcQheimdjilVRgpi'
con_s = u'xPPpr6kvMqMIOe3cyj0hE4Et8y08AehbFsAJW2qqXR7p7KKMHA'
acc_k = u'2545183884-MlXG41q8NF0inB1RtpqfSfpokT8fYnUuOBrVf0r'
acc_s = u'IzPbYqQzRHm5foiA5AWHccYfWYe0FMZ2wG9lCJZGuS2Lq'
twit = TwitterAPI(con_k, con_s, acc_k, acc_s)
twit.request('statuses/update', {'status': str(tweet_title +
"-Tweeted from " + request.url_root)})
return redirect(url_for('show_entries'))
def tweet_gifsm(fname, which_fun):
api = TwitterAPI(t_keys['CONSUMER_KEY'], t_keys['CONSUMER_SECRET'], t_keys['ACCESS_KEY'], t_keys['ACCESS_SECRET'])
total_bytes = os.path.getsize(fname)
with open(fname, 'rb') as file:
data = file.read()
r = api.request('media/upload', {'command':'INIT', 'media_type':'image/gif', 'total_bytes':total_bytes})
check_status(r)
media_id = r.json()['media_id']
r = api.request('media/upload', {'command':'APPEND', 'media_id':media_id, 'segment_index':0}, {'media':file})
check_status(r)
r = api.request('media/upload', {'command':'FINALIZE', 'media_id':media_id})
check_status(r)
print(r.status_code)
print(r.text)
def main():
db.connect()
db.create_table(Rows, safe=True)
api = TwitterAPI('65dxiPhn10SE3zJT6fVEPWsVx', 'VmK0rQFapjymwtSNpidi0Yfe16mjMdHXBhZTmYVc8dwb1joAxX',
'109566527-ZufkixJ3XInW91ZE34hGFtxQcrXGOzBS7vBdApUP', '0N5poNnJoDsWO8Yvf1FfNECfOJKJm7nKthPVzow7apyPu')
user_ids = [x.twi_id for x in Tweeple.select().where(Tweeple.filled_tweets == 0)]
counter = 0
bad_conter = 0
iter = len(user_ids)
print(iter)
sleep(15*60)
for twi_id in user_ids:
r = api.request('statuses/user_timeline', {'user_id': twi_id, 'count': '100'})
with db.transaction():
user = Tweeple.get(twi_id=twi_id)
for tweet in r:
tweet['user'] = user
if 'id' not in tweet.keys():
iter -= 1
counter += 1
bad_conter += 1
print(iter, 'Bad')
continue
del tweet['id']
Rows.create(**tweet)
user.filled_tweets = 1
user.save()
counter += 1
if counter > 290:
sleep(15*60)
counter = 0
iter -= 1
print(iter)
print(bad_conter)
class TwitterListener:
def __init__(self, redis_con, classifier):
self.redis_con = redis_con
self.classifier = classifier
self.twitter = TwitterAPI(settings.CONSUMER_KEY, settings.CONSUMER_SECRET, settings.ACCESS_TOKEN, settings.ACCESS_TOKEN_SECRET)
def run(self):
req = self.twitter.request('statuses/filter',{'track':settings.TOPICS})
for tweet in req.get_iterator():
if "retweeted_status" in tweet:
text = 'RT @'+tweet['retweeted_status']['user']['screen_name']+': '+tweet['retweeted_status']['text']
else:
text = tweet['text']
sys.stdout.write(text+"\n")
'''if(any([x.lower() in text.lower() for x in settings.AID_KEYWORDS])):
label="A"
else:
label="NA"
sys.stdout.write(label+"\n")
sys.stdout.flush()
self.classifier.addDocument(text,label)'''
pred = self.classifier.predict([text])
if pred != -1:
sys.stdout.write(str(pred[0])+"\n")
tweet['label']=pred[0]
self.redis_con.publish(settings.REDIS_CLASSIFIED_CHANNEL,json.dumps(tweet))
if pred[0]=='A':
self.redis_con.publish(settings.REDIS_FILTERED_CHANNEL,json.dumps(tweet))
else:
sys.stdout.write("No prediction")
sys.stdout.flush()
gevent.sleep(0)
def start(self):
gevent.spawn(self.run)
def follower_crawler(name, key):
api = TwitterAPI(**key)
while not q.empty():
team = q.get()
ids = set()
resps = list()
cursor = -1
while cursor != 0:
request = api.request('followers/ids', {'screen_name': team,
'count': 5000,
'cursor': cursor})
print '{name} [{code}]: {team} - {cursor}'.format(team = team, cursor = cursor, name = name, code = request.status_code)
if request.status_code != 429:
resp = [i for i in request][0]
resps.append(resp)
cursor = resp['next_cursor']
ids = ids.union(set(resp['ids']))
if int(request.headers['x-rate-limit-remaining']) == 0:
print '{name} sleeping'.format(name = name)
time.sleep(15*60)
with lock:
follower_lists[team] = {'ids': ids,
'responses': resps}
with open('../followers/{}.txt'.format(team), 'w') as f:
f.writelines([str(id)+'\n' for id in ids])
q.task_done()
def search():
global api
api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret)
r = api.request('search/tweets', {'q':'hate speech twitter bot','count':10})
for t in r:
tweet_id = str(t[u'id'])
#print tweet_id
try:
f = open(idfile)
except IOError:
with open(idfile,'w') as f:
print "Creating file: " + idfile
f.write("tweet id" + '\n')
f = open(idfile)
s = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
if s.find(tweet_id) != -1:
print '= FOUND = ' + tweet_id
f.close()
else:
print '= NEW TWEET = ' + tweet_id
f.close()
with open(idfile,'a') as f:
print "Writing tweet_id to file..."
f.write(tweet_id + '\n')
response(t)
print '\t'+t['user']['name']+" :: "+t['user']['screen_name']
print '\t'+t[u'text']
createURL(t)
print
def get_twitter_result(search):
CONSUMER_KEY = 'SECRET'
CONSUMER_SECRET = 'SECRET'
ACCESS_TOKEN_KEY = 'SECRET'
ACCESS_TOKEN_SECRET = 'SECRET'
api = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN_KEY, ACCESS_TOKEN_SECRET)
search_pattern = [search.replace(" ",""), search, search.split()[0], search[1:], '@' + search.replace(" ","")[1:]]
twitter_result = []
MAX_TWEET = 3
count = MAX_TWEET
try:
for pattern in search_pattern:
r = api.request('search/tweets',{'q':pattern,
'count':count,
'result_type':'mixed',
'lang':'en'})
for item in r.get_iterator():
twitter_result.append([item['user']['screen_name'],
item['user']['profile_image_url'],
item['text']])
if len(twitter_result) > MAX_TWEET-1:
break
else:
count = MAX_TWEET - len(twitter_result)
except:
twitter_result = []
return twitter_result
def twitter(request, search):
consumer_key = ''
consumer_secret = ''
access_token_key = ''
access_token_secret = ''
api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret)
r = api.request('search/tweets', {'q':str(search)})
data = json.loads(r.text)
tweetInfo = '['
for i in range(0,len(data['statuses'])):
tweetInfo += '{'
tweetInfo += '"id":"'+data['statuses'][i]['id_str']+'",'
tweetInfo += '"avatar":"'+data['statuses'][i]['user']['profile_image_url']+'",'
tweetInfo += '"tweetedOn":"'+data['statuses'][i]['created_at']+'",'
tweetInfo += '"tweetedOnParsed":"'+datetime.datetime.strptime(data['statuses'][i]['created_at'], "%a %b %d %X %z %Y").strftime("%X")+'",'
tweetInfo += '"screenname":"'+data['statuses'][i]['user']['screen_name']+'",'
tweetInfo += '"status":"'+data['statuses'][i]['text'].replace('"',"'").replace('\n','').replace('\r','')+'",'
tweetInfo += '"truncated":"'+str(data['statuses'][i]['truncated'])+'",'
tweetInfo += '"retweeted":"'+str(data['statuses'][1]['retweeted'])+'"'
tweetInfo += '}'
if i+1 < len(data['statuses']):
tweetInfo += ','
tweetInfo += ']'
return HttpResponse(tweetInfo)
def check_stream(self):
api = TwitterAPI(consumer_key=config.get("twitter.consumer_key"),
consumer_secret=config.get("twitter.consumer_secret"),
access_token_key=config.get("twitter.access_token"),
access_token_secret=config.get("twitter.access_token_secret")
)
while True:
tweeter_stream = api.request('statuses/filter', {'locations': "-123.66,32.54,-113.77,39.57,-93.82,24.32,-65.08,47.84"})
# tweeter_stream = api.request('statuses/filter', {'locations': self.get_geo_str()})
# print(self.get_geo_str())
start_time = time.time()
# print("len")
# print((tweeter_stream.text))
# print((tweeter_stream.stream))
# Stream data
for tweet in tweeter_stream:
# Break out for loop at specified time interval to query new sets of Geo Coordinates
if time.time() > start_time + self.REFRESH_INTERVAL:
print("breaktime")
break
# Publish tweets to Kafka
print(tweet)
def run_async(self):
auth = TwitterOAuth.read_file_object(self.wrapper.getFileObject(__file__, "TwitterAPI/credentials.txt"))
api = TwitterAPI(auth.consumer_key, auth.consumer_secret, auth.access_token_key, auth.access_token_secret)
# list of user id to follow (by Nazli)
""" '@StiftenAGF','@jpaarhus','#Aarhus','@AarhusKultur','@smagaarhus','@AarhusNyheder','@AarhusPortaldk',
'@Aarhus2017','@OpenDataAarhus', '@aarhusupdate','@AskVest','@SundhedOgOmsorg','@ArhusVejr','@aarhus_ints',
'@AGFFANdk','@AarhusCykelby','@VisitAarhus','@larshaahr' """
users = ['3601200017', '3370065087', '3330970365', '2911388001', '2706568231',
'2647614595', '2201213730', '1324132976', '1065597596', '816850003', '763614246',
'210987829', '159110346', '112585923', '77749562', '38227253', '36040150']
# list of terms to track
tracks = ['#Aarhus']
print "TAA ready."
try:
r = api.request('statuses/filter', {'locations': self.location, 'follow': ','.join(users), 'track': ','.join(tracks)})
except:
print r.text
for item in r:
self.tweet = item
# print item
self.wrapper.update()
if self.abort.is_set():
break
class TwitterGetter(object):
def __init__(self):
self.stop = False
self.registerApi()
@property
def headers(self):
return self.api
def registerApi(self):
from TwitterAPI import TwitterAPI
self.api = TwitterAPI("BGMUpePgUoFU13l3NOd26DVUD",
"pmiwqlftwhj6tPEDBzRrxvObSw3E8Kt29PIUo0XjGW60jtpOZO",
"2719912512-l6a59JAHMpbs8s5t72BHLJqL19sCvUluBw9Ewhh",
"1QNsRQKJ6eRTxY9x8HQ3RcdwNd48IF4ywk0MWFI0ILqG7")
def registerCallback(self, callback):
self.callback = callback
def startFlow(self):
r = self.api.request('statuses/filter', {'locations':'-179.1506, 18.9117, -66.9406, 71.4410'})
for item in r.get_iterator():
# if item['coordinates'] != None:
# print item['coordinates']['coordinates']
self.callback(item)
if self.stop : break
def stopFlow(self):
self.stop = True
def search_twitter_by_term(term, geo_search=False, lat="", lng=""):
consumer_key = TWITTER_API_KEYS['consumer_key']
consumer_secret = TWITTER_API_KEYS['consumer_secret']
access_token_key = TWITTER_API_KEYS['access_token_key']
access_token_secret = TWITTER_API_KEYS['access_token_secret']
api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret)
lang = 'en'
count = 10
try:
items = []
options = {'q': term, 'lang': lang, 'count': count}
if geo_search and lat and lng:
options['geocode'] = lat + ',' + lng + ',100mi'
r = list(api.request('search/tweets', options))
for item in r:
if 'text' in item and 'created_at' in item and 'user' in item:
text = process_twitter_text(item['text'])
created_at = format_date(item['created_at'])
user_name = item['user']['name']
user_screen_name = item['user']['screen_name']
profile_url = item['user']['profile_image_url']
item = TwitterItem(text, created_at, user_name, user_screen_name, profile_url)
items.append(item)
except:
raise TwitterAPIQueryError
return items
class Twitter:
def __init__(self):
self.api = TwitterAPI(
'LmACQX3XV1c9rA05MfYVbB9OG',
'ik7nVuZ0AtXffKjnqCjdCN1rTFVmod7Leehwbx1mezEDXCZAx8',
'2899747854-L4CzMtqL6BJQLsXNjs64xKavFqAPf3bPgspn4Aq',
'7zy5QY2hIKwzyUbJ5X9gPDluAoLbOzrFongjqFAiMI8dN'
)
self.influential = (
'CNBC', 'RANsquawk', 'the_real_fly', 'Wsjmarkets', 'Business'
)
def search(self, symbol, name, user=None, until=None):
q = symbol + ' OR ' + name
if user:
q += ' from:' + user
if until:
q += ' until:' + until
req = self.api.request('search/tweets', {'q': q, 'count': 100})
return [twt['text'] for twt in req]
def influential_tweets(self, symbol, name, until=None):
influentials = []
for user in self.influential:
req = self.search(symbol, name, user, until)
influentials += req
return influentials
def get_tweets(self, symbol, name, until=None):
return self.search(symbol, name, None, until) + self.influential_tweets(symbol, name, until)
def get_photo_stream():
api = TwitterAPI(
os.environ['consumer_key'],
os.environ['consumer_secret'],
os.environ['access_token_key'],
os.environ['access_token_secret']
)
# Any geo-tagged tweet, don't have access to firehose :(
r = api.request('statuses/filter', {'locations': '-180,-90,180,90'})
for item in r.get_iterator():
try:
if 'media_url' in item['entities']['media'][0] and item['entities']['media'][0]['type'] == "photo":
media_url = item['entities']['media'][0]['media_url']
content = urllib.urlopen(media_url).read()
img_array = np.asarray(bytearray(content), dtype=np.uint8)
img = cv2.imdecode(img_array, 0)
if img is not None and img.shape[::-1] in [(576, 1024), (600, 900)]: # Compressed iPhone 4/5 sizes
if is_snap(img):
print media_url
except KeyError:
continue
def loadTweets(request):
path = (os.path.dirname(os.path.dirname(os.path.realpath(__file__)))+"\\files\\access.json").replace('\\','/')
#url = static('/sentiments/us-states.json')
with open(path) as f:
data = json.load(f)
ACCESS_TOKEN_KEY = data['ACCESS_TOKEN_KEY']
ACCESS_TOKEN_SECRET = data['ACCESS_TOKEN_SECRET']
CONSUMER_KEY = data['CONSUMER_KEY']
CONSUMER_SECRET = data['CONSUMER_SECRET']
api = TwitterAPI(CONSUMER_KEY,CONSUMER_SECRET,ACCESS_TOKEN_KEY,ACCESS_TOKEN_SECRET)
SEARCH_TERM = 'trump'
count = 0
lang = 'en'
geocode = "37.6,-95.665,1400mi"
r = api.request('search/tweets', {'lang': lang, 'q': SEARCH_TERM, 'count': count, 'geocode': geocode})
for item in r:
if item['place'] != None :
text = unicode(item['text']).encode('utf-8')
state = unicode(item['place']['full_name'][-2:]).encode("utf-8")
score = getScore(text)
if score > -2:
queryState = State.objects.filter(abbrev=state)
queryTweet = Tweet.objects.filter(text=text)
#Check if tweet not already in the Database & if state exists
if queryState.count()>0 and queryTweet.count() == 0 :
state = queryState[0]
_, created = Tweet.objects.get_or_create(text=text,state=state,score=score,)
count +=1
return HttpResponse("Load successful "+ str(count)+" tweets has been added to the DB")
def suck(save_item, handle_error, source):
api = TwitterAPI(settings.TWITTER['consumer_key'],
settings.TWITTER['consumer_secret'],
settings.TWITTER['access_token'],
settings.TWITTER['access_token_secret'])
if 'lastRetrieved' not in source:
source['lastRetrieved'] = {}
for l in lists.items:
request_filters = {
'slug':l['slug'],
'owner_screen_name':l['owner_screen_name'],
'per_page': 100
}
if l['owner_screen_name'] in source['lastRetrieved']:
request_filters['since_id'] = source['lastRetrieved'][l['owner_screen_name']]
r = api.request('lists/statuses', request_filters)
new_since_id = None
if r.status_code == 200:
for record in r.get_iterator():
if not new_since_id:
new_since_id = record['id_str']
source['lastRetrieved'][l['owner_screen_name']] = new_since_id
item = transform(record)
save_item(item)
return source['lastRetrieved']
def get_tweets_by_GPS(location, text_only=False):
api = TwitterAPI(
Config.CONSUMER_KEY,
Config.CONSUMER_SECRET,
Config.ACCESS_TOKEN_KEY,
Config.ACCESS_TOKEN_SECRET
)
response = api.request('search/tweets', {'geocode': location})
tweets = []
for item in response.get_iterator():
data = {}
if "text" in item:
if text_only:
data['text'] = ' '.join(
re.sub('(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+://\S+)', " ", item['text']).split())
else:
data['text'] = item['text']
if 'profile_image_url' in item:
data['profile_image'] = item['profile_image_url']
elif 'profile_image_url_https' in item:
data['profile_image'] = item['profile_image_url_https']
tweets.append(data)
return tweets
def post_to_twitter(sender, instance, *args, **kwargs):
"""
Post new saved objects to Twitter.
"""
if settings.DEBUG == True:
return
# print("Status", instance._state.adding)
if instance.twitter == False:
# print("Not Twitte ")
return
data = {"status": str(get_body_twitte(instance.get_twitter_message()))}
media = {}
tw_url = 'statuses/update'
image = instance.image
if image:
file = open(os.path.join(settings.MEDIA_ROOT, image.name), 'rb')
media.update({'media[]': file.read()})
tw_url = 'statuses/update_with_media'
consumer_key = settings.TWITTER_CONSUMER_KEY
consumer_secret = settings.TWITTER_CONSUMER_SECRET
access_token_key = settings.TWITTER_ACCESS_TOKEN_KEY
access_token_secret = settings.TWITTER_ACCESS_TOKEN_SECRET
api = TwitterAPI(
consumer_key, consumer_secret, access_token_key, access_token_secret)
r = api.request(tw_url, data, media)
def mineTweets(username):
tweet_list = []
api = TwitterAPI(CONSUMER_KEY,CONSUMER_SECRET,ACCESS_TOKEN_KEY,ACCESS_TOKEN_SECRET)
r = api.request('statuses/user_timeline', {'screen_name': username})
for item in r:
tweet_list.append((item['text'],item['created_at']))
return tweet_list
import descartes
import geopandas as gpd
import json
consumer_key= ""
consumer_secret= ""
access_token_key= ""
access_token_secret= ""
api = TwitterAPI(consumer_key,
consumer_secret,
access_token_key,
access_token_secret)
response = api.request("statuses/filter", {"track": ["harry", "and", "meghan"]})
tweets = response.get_iterator()
coordinates = []
count = 0
while count < 100:
tweet = next(tweets)
if "place" in tweet and tweet["place"] != None:
location = tweet["place"]["bounding_box"]["coordinates"][0][0]
coordinates.append(location)
count += 1
world_map = gpd.read_file("C:/Users/Brian/Desktop/TM_WORLD_BORDERS_SIMPL-0.3/TM_WORLD_BORDERS_SIMPL-0.3.shp")
fig, ax = plt.subplots(figsize = (15, 15))
class TwitterURLs():
"""
Twitter URLs enables access to the URLs posted by the authorised user and
the followers of that user
"""
def __init__(self, responseCount=40):
#set response count
self.responseCount = responseCount
#configure TwitterAPI AUTH
self.api = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN_KEY,
ACCESS_TOKEN_SECRET)
#set class variables to store responses
self.tweets = {}
self.urls = {}
self.followers = []
def setFollowers(self):
"""
setFollowers sets the class variable to a list of user IDs that are
following the authorised user
"""
self.followers = []
response = self.api.request(
'followers/list', {
'skip_status': 'true',
'include_user_entities': 'false',
'count': self.responseCount
})
for item in response:
for user in item['users']:
self.followers.append(user['screen_name'])
def setTweets(self, screenName='owner'):
"""
setTweets adds a dictionary key value pair to the class variable tweets where
the key is a screenName (or owner if authorised user) and the value are tweets
"""
if (screenName == 'owner'):
response = self.api.request('statuses/user_timeline',
{'count': self.responseCount})
else:
response = self.api.request('statuses/user_timeline', {
'screen_name': screenName,
'count': self.responseCount
})
self.tweets[screenName] = []
for item in response:
self.tweets[screenName].append(item)
def setURLs(self, screenName='owner'):
"""
setURLS adds a key value pair to the urls class variable where the key is
a screenName and the value is a list of recent URLs they have tweeted
"""
if (screenName not in self.tweets.keys()): self.setTweets(screenName)
self.urls[screenName] = []
for tweet in self.tweets[screenName]:
try:
urls = tweet['entities']['urls']
except KeyError:
print "Key Error for user {}".format(screenName)
urls = []
for url in urls:
self.urls[screenName].append(url['expanded_url'])
def getFollowers(self):
"printFollowers prints all followers in the class variable followers"
if (len(self.followers) == 0): self.setFollowers()
if VERBOSE:
for follower in self.followers:
print follower
return self.followers
def getTweets(self, screenName='owner'):
"printTweets prints all the tweet text for the given screenName"
if (screenName not in self.tweets.keys()): self.setTweets(screenName)
tweets = []
for tweet in self.tweets[screenName]:
if VERBOSE: print tweet['text']
tweets.append(tweet['text'])
return tweets
def getURLs(self, screenName='owner'):
"printURLs prints all the URLs shared by the given screenName"
if (screenName not in self.urls.keys()): self.setURLs(screenName)
if VERBOSE:
for url in self.urls[screenName]:
print url
return self.urls[screenName]
def getAllURLs(self):
"getAllURLs gets all the the URLs shared by a users followers"
if (len(self.followers) == 0): self.setFollowers()
#set the urls for owner
self.setURLs()
#get the urls for all owners
for follower in self.followers:
self.setURLs(follower)
#return the urls dictionary object
return self.urls
PRODUCT = 'fullarchive'
LABEL = 'Testing'
api = TwitterAPI("nWQzqafsC0XcH370OkSlwxonQ",
"ZZWqC0b1hO3N8bGO5yfpTj98mnptVBTJX6sZ9fbBdBmH9h0SUJ",
"706273045527314432-GymkKoqae8a3uNBSALfjG2pcnr6oT34",
"EUQ40A65Bck7yVQceSClcGX3Q3eKvNSVq3VXCZk1o7yBq")
csvFile = open('Gillibrand.csv', 'a', encoding="utf-8")
csvWriter = csv.writer(csvFile)
tnext = ' '
i = 0
r = api.request('tweets/search/%s/:%s' % (PRODUCT, LABEL), {
'query': query,
'fromDate': '201809010000',
'toDate': '201811052200'
})
for item in r:
csvWriter.writerow([
item['created_at'], item['user']['screen_name'],
item['text'] if 'text' in item else item
])
json = r.json()
if 'next' not in json:
print("fail")
tnext = json['next']
while (i < 15):
r = api.request(
import json
import csv
import re
# Pass API permissions
consumer_key = ('')
consumer_secret = ('')
access_token = ('')
access_token_secret = ('')
# Create API frame
api = TwitterAPI(consumer_key, consumer_secret, access_token, access_token_secret)
# Set API to request tweets on response object
response = api.request("statuses/filter", {"track": ["BLM"]}) # filter using keywords attribute
print(response.response)
# Get geographic coordinates
coordinates = []
# Iterate over tweets
tweets = response.get_iterator()
# Create a while loop to extract set amount of tweets
count = 0
while count < 50:
tweet = next(tweets)
if "place" in tweet and tweet["place"] != None:
place = tweet["place"]["bounding_box"]["coordinates"][0][0]
coordinates.append(place)
class TwitterInterface(SingleCommandInterface):
log = logging.getLogger(__name__)
user_manager: UserManager
data: CovidData
viz: Visualization
twitter: TwitterAPI
handle_regex = re.compile('@(\w){1,15}')
location_service: LocationService
INFECTIONS_UID = "infections"
VACCINATIONS_UID = "vaccinations"
ICU_UID = "icu"
def __init__(self,
consumer_key: str,
consumer_secret: str,
access_token_key: str,
access_token_secret: str,
user_manager: UserManager,
covid_data: CovidData,
visualization: Visualization,
no_write: bool = False):
super().__init__(user_manager, covid_data, visualization, 15, no_write)
self.twitter = TwitterAPI(consumer_key,
consumer_secret,
access_token_key,
access_token_secret,
api_version='1.1')
self.twitter.CONNECTION_TIMEOUT = 120
self.twitter.REST_TIMEOUT = 120
self.twitter.STREAMING_TIMEOUT = 120
self.rki_name = "@rki_de"
self.bmg_name = "@BMG_Bund"
self.update_follower_number()
def update_follower_number(self):
response = self.twitter.request('users/show',
{'user_id': 1367862514579542017})
if response.status_code == 200:
number = response.json()['followers_count']
self.user_manager.set_platform_user_number(number)
def write_message(self,
messages: List[BotResponse],
reply_obj: Optional[object] = None) -> bool:
if reply_obj and type(reply_obj) != int:
raise ValueError("Twitter client needs reply_obj to be int")
for message in messages:
data = {'status': message.message}
if message.images:
# Upload filenames
media_ids = []
for file in message.images:
with open(file, "rb") as f:
upload_resp = self.twitter.request(
'media/upload', None, {'media': f.read()})
if upload_resp.status_code != 200:
if upload_resp.status_code == 429: # Rate Limit exceed
reset_time = int(
upload_resp.headers.get(
"x-rate-limit-reset", 0))
if reset_time:
sleep_time = (
datetime.fromtimestamp(
reset_time, timezone.utc) -
datetime.now(tz=timezone.utc)).seconds
self.log.warning(
f"Rate Limit exceed: Wait for reset in {sleep_time}s"
)
time.sleep(sleep_time)
return False
raise ValueError(
f"Could not upload graph to twitter. API response {upload_resp.status_code}: "
f"{upload_resp.text}")
media_ids.append(upload_resp.json()['media_id'])
data['media_ids'] = ",".join(map(str, media_ids))
if reply_obj:
data['in_reply_to_status_id'] = reply_obj
data['auto_populate_reply_metadata'] = True
with API_RESPONSE_TIME.labels(platform='twitter').time():
response = self.twitter.request('statuses/update', data)
if 200 <= response.status_code < 300:
self.log.info(
f"Tweet sent successfully {len(data['status'])} chars), response: {response.status_code}"
)
SENT_MESSAGE_COUNT.inc()
self.update_twitter_metrics(response)
reply_obj = response.json()['id']
else:
if upload_resp.status_code == 429: # Rate Limit exceed
reset_time = int(
upload_resp.headers.get("x-rate-limit-reset", 0))
if reset_time:
sleep_time = (
datetime.fromtimestamp(reset_time, timezone.utc) -
datetime.now(tz=timezone.utc)).seconds
self.log.warning(
f"Rate Limit exceed: Wait for reset in {sleep_time}s"
)
time.sleep(sleep_time)
return False
raise ValueError(
f"Could not send tweet: API Code {response.status_code}: {response.text}"
)
return True
@staticmethod
def update_twitter_metrics(response: TwitterResponse):
quota = response.get_quota()
if 'limit' in quota and quota['limit']:
API_RATE_LIMIT.labels(platform='twitter',
type='limit').set(quota['limit'])
if 'remaining' in quota and quota['remaining']:
API_RATE_LIMIT.labels(platform='twitter',
type='remaining').set(quota['remaining'])
API_RESPONSE_CODE.labels(platform='twitter',
code=response.status_code).inc()
def get_mentions(self) -> Iterable[SingleArgumentRequest]:
try:
with API_RESPONSE_TIME.labels(platform='twitter').time():
response = self.twitter.request(f"statuses/mentions_timeline",
params={
'tweet_mode': 'extended',
'count': 200,
'trim_user': 1
})
except TwitterConnectionError as e:
self.log.warning(
f"TwitterConnectionError while fetching mentions: {e}",
exc_info=e)
API_ERROR.inc()
return []
self.update_twitter_metrics(response)
mentions = []
if 200 <= response.status_code < 300:
for tweet in response:
if self.user_manager.is_message_answered(tweet['id']):
continue
mention_position = 0
for mention in tweet['entities']['user_mentions']:
if mention['id'] == 1367862514579542017:
mention_position = mention['indices'][1]
break
arguments = self.handle_regex.sub(
"", tweet['full_text'][mention_position:]).strip()
if arguments:
# As our locale is different, we have to adapt Twitter Time & Date String
day_en = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
day_de = ["Mo", "Di", "Mi", "Do", "Fr", "Sa", "So"]
month_en = [
"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug",
"Sep", "Oct", "Nov", "Dec"
]
month_de = [
"Jan", "Feb", "Mär", "Apr", "Mai", "Jun", "Jul", "Aug",
"Sep", "Okt", "Nov", "Dez"
]
localized_str = replace_by_list(tweet['created_at'],
day_en + month_en,
day_de + month_de)
created = None
try:
created = datetime.strptime(localized_str,
"%a %b %d %H:%M:%S %z %Y")
except ValueError as e:
self.log.warning(
f"Cant parse twitters date string {localized_str}:",
exc_info=e)
mentions.append(
SingleArgumentRequest(tweet['id'], arguments,
tweet['id'], created))
return mentions
def main():
"""
Sends Donald Trump, the saddest excuse of a President the United States has seen, a tweet urging him to delete
his account. And because he might not know how to do that, we'll even provide him a link.
:return: None
"""
i = 0
recent_messages = deque(maxlen=3)
retry_post = None
while True:
time_now = time()
time_now_hr = strptime(
ctime(time_now),
'%a %b %d %H:%M:%S %Y',
)
# Every morning at 8 am, try to spread how people can join the Revolution against Trump.
if time_now_hr.tm_min == 0 and time_now_hr.tm_sec <= 10 and time_now_hr.tm_hour == 8:
try:
api = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN,
ACCESS_SECRET)
request = api.request(
'statuses/update', {
'status':
'Want to tell @realDonaldTrump how bad he is doing every hour? '
'Download this Python script. https://github.com/annoyTrump/annoyTrump'
})
if request.status_code == 200:
print('\033[92m' + '\r\nSent ad.' + '\033[0m')
else:
print('Ad could not be sent.')
except (TimeoutError, TwitterConnectionError):
pass
# Every hour, tweet at 45, calling him out on his lack of leadership and overuse of Twitter and golfing.
if time_now % 3600 < 10 or (retry_post is not None
and retry_post is True):
retry_post = False
try:
link = randint(0, len(DELETE_TWITTER_LINKS) - 1)
noun = randint(0, len(NOUNS) - 1)
action = randint(0, len(ACTIONS) - 1)
message = MESSAGE.format(link=DELETE_TWITTER_LINKS[link],
noun=NOUNS[noun],
action=ACTIONS[action])
while message in recent_messages:
link = randint(0, len(DELETE_TWITTER_LINKS) - 1)
noun = randint(0, len(NOUNS) - 1)
action = randint(0, len(ACTIONS) - 1)
message = MESSAGE.format(link=DELETE_TWITTER_LINKS[link],
noun=NOUNS[noun],
action=ACTIONS[action])
recent_messages.append(message)
# tell Trump how to delete his account
api = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN,
ACCESS_SECRET)
request = api.request('statuses/update', {'status': message})
if request.status_code == 200:
print('\033[92m' + '\r\nSent {msg}.'.format(msg=message) +
'\033[0m')
else:
errors = json.loads(request.text)['errors']
# if Twitter says this is a duplicate, then increase the maxlen of the deque
if len(errors) == 1 and errors[0]['code'] == 187:
# create a new deque with the old deque items in it. Increase maxlen by one.
recent_messages = deque(recent_messages,
maxlen=recent_messages.maxlen +
1)
# append the most recent message that was denied.
recent_messages.append(message)
print(
'\r\nPost was denied. Trying to post another one.')
retry_post = True
else:
print('\r\nError ' + request.status_code + '. ' +
request.text + ': ' + message)
sleep(60)
except (TimeoutError, TwitterConnectionError):
pass
else:
if i == 0:
print('\r\nsleeping')
i = 255
else:
print(['z', 'Z', ' '][randint(0, 2)], end='', flush=True)
i -= 1
sleep(10)
class Twitter:
url_length = 23
def __init__(self, config_json_file):
with open(config_json_file) as json_data_file:
config_json = json.load(json_data_file)
self.api = TwitterAPI(config_json["consumer_key"],
config_json["consumer_secret"],
config_json["token_key"],
config_json["token_secret"])
self.logger = logging.getLogger('__main__')
self.admin_profile = config_json["admin_profile"]
def post_tweet(self, text):
self.logger.info("Posting tweet: {}".format(text))
r = self.api.request('statuses/update', {'status': text})
if r.status_code != 200:
self.logger.error("Posting tweet failed with code {}".format(
r.status_code))
self.logger.error("Response content {}".format(r.response.content))
try:
code_error = json.loads(
r.response.content)['errors'][0]['code']
except KeyError:
self.logger.error
raise TweetNotSent
if code_error == 187:
self.logger.error('Status is a duplicate.')
raise DuplicateTweet
if code_error == 326:
self.logger.error(
'Twitter SPAM protection locked sending status updates.')
raise TwitterLockedForSpam
else:
self.logger.error(
'Error code {} not recognized'.format(code_error))
raise TweetNotSent
def delete_last_tweets(self, count=1):
self.logger.info("Deleting last {} tweets".format(count))
r = self.api.request('statuses/user_timeline', {'count': count})
if r.status_code == 200:
for item in r:
tweet_id = item['id']
r2 = self.api.request('statuses/destroy/:%d' % tweet_id)
if r2.status_code != 200:
self.logger.error(
"Posting tweet failed with code {}".format(
r.status_code))
self.logger.error("Response content {}".format(
r.response.content))
raise TwitterError
else:
self.logger.error("Response content {}".format(r.response.content))
raise TwitterError
return True
def send_dm(self, username, message):
user_id = self.get_user_id(username)
event = {
"event": {
"type": "message_create",
"message_create": {
"target": {
"recipient_id": user_id
},
"message_data": {
"text": message
}
}
}
}
r = self.api.request('direct_messages/events/new', json.dumps(event))
if r.status_code != 200:
self.logger.error("Direct Message failed {}".format(r.status_code))
self.logger.error("Response content {}".format(r.response.content))
raise TwitterError
def get_user_id(self, username):
r = self.api.request('users/lookup',
{'screen_name': username.strip('@')})
if r.status_code == 200:
return r.json()[0]['id']
self.logger.error("Getting User id failed with code {}".format(
r.status_code))
self.logger.error("Response content {}".format(r.response.content))
return None
class TwitterConnector:
def __init__(self):
"""
twitter API 를 사용하기 위해 API KEY를 이용해 연결하고, 요청에 필요한 파라미터를 초기화
실시간 데이터를 응답받기 위해선 twitter 에서 정해놓은 query 형식에 따라 rule 을 등록한 후 요청해야 함
모든 rule 은 계정(API KEY)에 연동되고 각가 id로 구분되며, 추가적인 등록/삭제가 없다면 변경되지 않음
여기서 rule 이란 특정한 조건을 만족하는 streaming data 를 받기위한 filtering 방법
"""
try:
auth_info = TwitterOAuth.read_file()
self.api = TwitterAPI(
auth_info.consumer_key,
auth_info.consumer_secret,
auth_type="oAuth2",
api_version="2",
)
except TwitterRequestError as e:
msg_list = []
for msg in iter(e):
msg_list.append(msg)
err_msg = "RequestError: " + "|".join(msg_list)
Log.e(err_msg)
except TwitterConnectionError as e:
Log.e(f"ConnectionError: {e}")
self.prompt_reconnect_msg(2)
except Exception as e:
Log.e(f"BaseException: {e}")
self.prompt_reconnect_msg(2)
# twitter conifg 파일
twitter_config = config.Config('configs/twitter_config.conf')
self.query_string = twitter_config.load_config('API_QUERY', 'rules')
self.output_file_name = twitter_config.load_config(
'OUTPUT_FILES', 'stream_result')
# comma-seperated list with no space between fields
self.expansions = twitter_config.load_config('CONTENTS', 'expansions')
self.media_fields = twitter_config.load_config('CONTENTS',
'media_fields')
self.tweet_fields = twitter_config.load_config('CONTENTS',
'tweet_fields')
self.user_fields = twitter_config.load_config('CONTENTS',
'user_fields')
self.place_fields = twitter_config.load_config('CONTENTS',
'place_fields')
def init_stream_rules(self):
"""
기존에 등록된 모든 rule 을 제거하고, 초기 rule 을 등록
:return: -
"""
cur_rules = self.get_stream_rules()
self.delete_stream_rules(cur_rules)
self.add_stream_rules(self.query_string)
def add_stream_rules(self, request_query):
"""
계정에 특정 rule 을 등록
:param request_query: [str] 문법에 맞는 쿼리문
:return: -
"""
try:
response = self.api.request("tweets/search/stream/rules",
{"add": [{
"value": request_query
}]})
Log.i(f"[{response.status_code}] RULE ADDED: {response.text}")
if response.status_code != 201:
raise Exception(response.text)
except TwitterRequestError as e:
msg_list = []
for msg in iter(e):
msg_list.append(msg)
err_msg = "RequestError: " + "|".join(msg_list)
Log.e(err_msg)
except TwitterConnectionError as e:
Log.e(f"ConnectionError: {e}")
self.prompt_reconnect_msg(2)
except Exception as e:
Log.e(f"BaseException: {e}")
self.prompt_reconnect_msg(2)
def get_stream_rules(self):
"""
현재 계정에 등록된 rule 목록을 조회
:return: [list(str)] rule's ids
"""
rule_ids = []
try:
response = self.api.request("tweets/search/stream/rules",
method_override="GET")
Log.i(f"[{response.status_code}] RULES: {response.text}")
if response.status_code != 200:
raise Exception(response.text)
else:
for item in response:
if "id" in item:
rule_ids.append(item["id"])
else:
Log.i(json.dumps(item, ensure_ascii=False))
return rule_ids
except TwitterRequestError as e:
msg_list = []
for msg in iter(e):
msg_list.append(msg)
err_msg = "RequestError: " + "|".join(msg_list)
Log.e(err_msg)
except TwitterConnectionError as e:
Log.e(f"ConnectionError: {e}")
self.prompt_reconnect_msg(2)
except Exception as e:
Log.e(f"BaseException: {e}")
self.prompt_reconnect_msg(2)
def delete_stream_rules(self, rule_ids):
"""
특정 rule 을 삭제
:param rule_ids: [list(str)] 삭제 하기위한 rule 의 id 목록
:return: -
"""
try:
if len(rule_ids) > 0:
response = self.api.request("tweets/search/stream/rules",
{"delete": {
"ids": rule_ids
}})
Log.i(
f"[{response.status_code}] RULES DELETED: {json.dumps(response.json())}"
)
if response.status_code != 200:
raise Exception(response.text)
except TwitterRequestError as e:
msg_list = []
for msg in iter(e):
msg_list.append(msg)
err_msg = "RequestError: " + "|".join(msg_list)
Log.e(err_msg)
except TwitterConnectionError as e:
Log.e(f"ConnectionError: {e}")
self.prompt_reconnect_msg(2)
except Exception as e:
Log.e(f"BaseException: {e}")
self.prompt_reconnect_msg(2)
def start_stream(self, producer, topic):
"""
계정에 등록된 rule 에 따라 filtered streaming data 를 받아옴
data를 정상적으로 받아오고 있다면 오류/종료 전까지 반복문에서 빠져나오지 않음
에러 원인에 따라 재연결을 시도하여 서버가 유지될 수 있도록 함
rule 과 별개로 하나의 tweet 에서 받아오고자 하는 정보를 쿼리에 포함시킬 수 있음
kafka 의 prodcer 역할을 하는 부분으로 받아오는 data 를 producer 와 연결된 broker 로 전달함
:param producer: [kafka.producer] kafka 의 produce 객체
:param topic: [str] 데이터를 전달하고자하는 broker 의 특정 topic
:return: -
"""
total_cnt = 0
while True:
try:
response = self.api.request(
"tweets/search/stream",
{
"expansions": self.expansions,
"media.fields": self.media_fields,
"tweet.fields": self.tweet_fields,
"user.fields": self.user_fields,
"place.fields": self.place_fields,
},
)
Log.i(f"[{response.status_code}] START...")
Log.i(response.get_quota()) # API connect 회수 조회
if (response.status_code != 200 and response.status_code != 400
and response.status_code != 429):
# 에러 원인별 다른 처리를 위해 응답 코드로 구분
raise Exception(response)
with open(self.output_file_name, "a",
encoding="utf-8") as output_file, open(
"logs/data_count.txt", "a") as cnt_file:
# data_count.txt : 유실을 확인하기위해 count
# [데이터를 받아온 시간] 해당 tweet 의 게시 시간 (파일이 open 된 후 받아온 데이터의 수 / 프로그램이 실행된 후 받아온 데이터의 수)
print(f"[{datetime.datetime.now()}] file re-open",
file=cnt_file)
for no, item in enumerate(response):
self.check_error_response(item)
data = json.dumps(item, ensure_ascii=False)
print(data, file=output_file, flush=True)
producer.send_data(topic=topic, data=data)
print(
f"[{datetime.datetime.now()}] {item['data']['created_at']} ({no} / {total_cnt})",
file=cnt_file,
flush=True,
)
total_cnt += 1
except TwitterRequestError as e:
# ConnectionException will be caught here
msg_list = []
for msg in iter(e):
msg_list.append(msg)
err_msg = "RequestError: " + "|".join(msg_list)
Log.e(err_msg)
if e.status_code >= 500:
self.prompt_reconnect_msg(3)
elif e.status_code == 429:
self.prompt_reconnect_msg(63)
else:
exit()
except TwitterConnectionError as e:
Log.e(f"ConnectionError: {e}")
self.prompt_reconnect_msg(3)
except Exception as e:
Log.e(f"Exception: {e}")
self.prompt_reconnect_msg(3)
def check_error_response(self, item):
"""
twitter api 의 response 가 error 메세지 일 경우 데이터 포멧이 다르므로 따로 처리
:param item: [dict] twitter api response data
:return: terminate | raise Exception
"""
error_type = {
# 에러 원인 중 해당 정보를 제공하는 겨우 재연결 시도가 의미없으므로 프로그램 종료
"CRITICAL": [
"https://api.twitter.com/2/problems/usage-capped",
"https://api.twitter.com/2/problems/rule-cap",
"https://api.twitter.com/2/problems/invalid-rules",
"https://api.twitter.com/2/problems/duplicate-rules",
]
}
if "data" in item:
return
elif "errors" in item:
title_ = item["errors"][0]["title"]
detail_ = item["errors"][0]["detail"]
type_ = item["errors"][0]["type"]
ret_msg = [title_, detail_, type_]
ret_msg = "|".join(ret_msg)
if type_ in error_type["CRITICAL"]:
Log.c(ret_msg)
exit()
else:
raise Exception(ret_msg)
def prompt_reconnect_msg(self, wait_secs):
"""
안내문구 출력 후 특정시간 동안 대기
아래와 같은 이용 정책을 준수하기 위해 최소 요구되는 대기 시간은 2~3초로 판단
Twitter API has usage rate limit (Per 15m)
- 50 connect
- 450 adding, deleting, listing
:param wait_secs: [int] 대기 시간 (초)
:return: -
"""
Log.i(f"wait {wait_secs}secs for reconnecting..")
time.sleep(wait_secs)
from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError
import json
QUERY = 'pizza"'
try:
o = TwitterOAuth.read_file()
api = TwitterAPI(o.consumer_key,
o.consumer_secret,
auth_type='oAuth2',
api_version='2')
# ADD STREAM RULES
r = api.request('tweets/search/stream/rules', {'add': [{'value': QUERY}]})
print(f'[{r.status_code}] RULE ADDED: {json.dumps(r.json(), indent=2)}\n')
except TwitterRequestError as e:
print(e.status_code)
for msg in iter(e):
print(msg)
except TwitterConnectionError as e:
print(e)
except Exception as e:
print(e)
def findFollowers(self):
#check if cherrypy session is initialized otherwise reload token
if ("consumer_key" not in cherrypy.session.keys()
or cherrypy.session['consumer_key'] == ""):
raise cherrypy.HTTPRedirect("/thirdpartyrats/twitter/tokens")
else:
api = TwitterAPI(cherrypy.session['consumer_key'],
cherrypy.session['consumer_secret'],
cherrypy.session['oauth_token'],
cherrypy.session['oauth_secret'])
r = api.request('followers/list')
#creating followers list
follower_list = {"": []}
for follower in r.json()['users']:
id = follower['id']
screen_name = follower['screen_name']
name = follower['name']
created_at = datetime.strftime(
datetime.strptime(follower['created_at'],
'%a %b %d %H:%M:%S +0000 %Y'),
'%Y-%m-%d %H:%M:%S')
protected = follower['protected']
geo_enabled = follower['geo_enabled']
verified = follower['verified']
json_object = {
"id": id,
"screen_name": screen_name,
"name": name,
"created_at": created_at,
"protected": protected,
"geo_enabled": geo_enabled,
"verified": verified
}
follower_list[""].append(json_object)
html = "<label>Followers List: </label><br><br>" + convert(
follower_list,
build_direction="LEFT_TO_RIGHT",
table_attributes={"id": "followersList"})
print(html)
cherrypy.session["followers"] = html
#creating following list
r = api.request('friends/list')
following_list = {"": []}
for following in r.json()['users']:
id = following['id']
screen_name = following['screen_name']
name = following['name']
created_at = datetime.strftime(
datetime.strptime(following['created_at'],
'%a %b %d %H:%M:%S +0000 %Y'),
'%Y-%m-%d %H:%M:%S')
protected = following['protected']
geo_enabled = following['geo_enabled']
verified = following['verified']
json_object = {
"id": id,
"screen_name": screen_name,
"name": name,
"created_at": created_at,
"protected": protected,
"geo_enabled": geo_enabled,
"verified": verified
}
following_list[""].append(json_object)
html = "<label>Following List: </label><br><br>" + convert(
following_list,
build_direction="LEFT_TO_RIGHT",
table_attributes={"id": "followersList"})
print(html)
cherrypy.session["following"] = html
raise cherrypy.HTTPRedirect("/thirdpartyrats/twitter/getContacts")
import json
from twitterCreds import *
from keywords import *
api = TwitterAPI(consumer_key, consumer_secret, access_token_key,
access_token_secret)
kinesis = boto3.client('kinesis')
# Keywords
terms = iphone
# Tweet Language
language = 'en'
# Request Twitter api and filter on keywords and language
request = api.request('statuses/filter', {
'track': terms,
'language': language
})
# Ingest into kinesis stream
while True:
try:
for item in request:
if 'text' in item:
kinesis.put_record(StreamName="twitter",
Data=json.dumps(item),
PartitionKey="filler")
except:
pass
QUERY = '"pizza" OR "burger"'
EXPANSIONS = 'author_id,referenced_tweets.id,referenced_tweets.id.author_id,in_reply_to_user_id,attachments.media_keys,attachments.poll_ids,geo.place_id,entities.mentions.username'
TWEET_FIELDS = 'author_id,conversation_id,created_at,entities,geo,id,lang,public_metrics,source,text'
USER_FIELDS = 'created_at,description,entities,location,name,profile_image_url,public_metrics,url,username'
try:
o = TwitterOAuth.read_file()
api = TwitterAPI(o.consumer_key,
o.consumer_secret,
auth_type=OAuthType.OAUTH2,
api_version='2')
# ADD STREAM RULES
r = api.request('tweets/search/stream/rules', {'add': [{'value': QUERY}]})
print(f'[{r.status_code}] RULE ADDED: {json.dumps(r.json(), indent=2)}\n')
if r.status_code != 201: exit()
# GET STREAM RULES
r = api.request('tweets/search/stream/rules', method_override='GET')
print(f'[{r.status_code}] RULES: {json.dumps(r.json(), indent=2)}\n')
if r.status_code != 200: exit()
# START STREAM
r = api.request('tweets/search/stream', {
'expansions': EXPANSIONS,
'tweet.fields': TWEET_FIELDS,
'user.fields': USER_FIELDS,
def process_item(self, item, spider):
try:
# Insert item into kith table.
if isinstance(item, KithItem):
self.cursor.execute(
"INSERT INTO kith (name, link, size, date) VALUES (%s, %s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'),
item['size'].encode('utf-8'), DATE))
# Insert item into ruvilla table.
elif isinstance(item, RuvillaItem):
self.cursor.execute(
"INSERT INTO ruvilla (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into footlocker table.
elif isinstance(item, FootLockerItem):
self.cursor.execute(
"INSERT INTO footlocker (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into footaction table.
elif isinstance(item, FootActionItem):
self.cursor.execute(
"INSERT INTO footaction (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into champs table.
elif isinstance(item, ChampsItem):
self.cursor.execute(
"INSERT INTO champs (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into eastbay table.
elif isinstance(item, EastBayItem):
self.cursor.execute(
"INSERT INTO eastbay (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into finishline table.
elif isinstance(item, FinishLineItem):
self.cursor.execute(
"INSERT INTO finishline (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into adidas table.
elif isinstance(item, AdidasItem):
self.cursor.execute(
"INSERT INTO adidas (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into nike table.
elif isinstance(item, NikeItem):
self.cursor.execute(
"INSERT INTO nike (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into footshop table.
elif isinstance(item, FootShopItem):
self.cursor.execute(
"INSERT INTO footshop (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into caliroots table.
elif isinstance(item, CalirootsItem):
self.cursor.execute(
"INSERT INTO caliroots (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into afew table.
elif isinstance(item, AfewItem):
self.cursor.execute(
"INSERT INTO afew (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into einhalb table.
elif isinstance(item, EinhalbItem):
self.cursor.execute(
"INSERT INTO einhalb (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into end table.
elif isinstance(item, EndItem):
self.cursor.execute(
"INSERT INTO end (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into sns table.
elif isinstance(item, SNSItem):
self.cursor.execute(
"INSERT INTO sns (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into goodwillout table.
elif isinstance(item, GoodWillOutItem):
self.cursor.execute(
"INSERT INTO goodwillout (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into tint table.
elif isinstance(item, TintItem):
self.cursor.execute(
"INSERT INTO tint (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into overkill table.
elif isinstance(item, OverkillItem):
self.cursor.execute(
"INSERT INTO overkill (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into footdistrict table.
elif isinstance(item, FootDistrictItem):
self.cursor.execute(
"INSERT INTO footdistrict (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into size table.
elif isinstance(item, SizeItem):
self.cursor.execute(
"INSERT INTO size (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into ycmc table.
elif isinstance(item, YCMCItem):
self.cursor.execute(
"INSERT INTO ycmc (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into city table.
elif isinstance(item, CityItem):
self.cursor.execute(
"INSERT INTO city (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into nordstrom table.
elif isinstance(item, NordstromItem):
self.cursor.execute(
"INSERT INTO nordstrom (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into barneys table.
elif isinstance(item, BarneysItem):
self.cursor.execute(
"INSERT INTO barneys (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into jimmyjazz table.
elif isinstance(item, JimmyJazzItem):
self.cursor.execute(
"INSERT INTO jimmyjazz (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into jdsports table.
elif isinstance(item, JDSportsItem):
self.cursor.execute(
"INSERT INTO jdsports (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into footpatrol table.
elif isinstance(item, FootPatrolItem):
self.cursor.execute(
"INSERT INTO footpatrol (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into sneakerbaas table.
elif isinstance(item, SneakerBaasItem):
self.cursor.execute(
"INSERT INTO sneakerbaas (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into sneakerpolitics table.
elif isinstance(item, SneakerPoliticsItem):
self.cursor.execute(
"INSERT INTO sneakerpolitics (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into urbanindustry table.
elif isinstance(item, UrbanIndustryItem):
self.cursor.execute(
"INSERT INTO urbanindustry (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into urbanoutfitters table.
elif isinstance(item, UrbanOutfittersItem):
self.cursor.execute(
"INSERT INTO urbanoutfitters (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into luisa table.
elif isinstance(item, LuisaItem):
self.cursor.execute(
"INSERT INTO luisa (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into slamjam table.
elif isinstance(item, SlamJamItem):
self.cursor.execute(
"INSERT INTO slamjam (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into rise45 table.
elif isinstance(item, Rise45Item):
self.cursor.execute(
"INSERT INTO rise45 (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into undefeated table.
elif isinstance(item, UndefeatedItem):
self.cursor.execute(
"INSERT INTO undefeated (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into zappos table.
elif isinstance(item, ZapposItem):
self.cursor.execute(
"INSERT INTO zappos (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into ubiq table.
elif isinstance(item, UbiqItem):
self.cursor.execute(
"INSERT INTO ubiq (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into pointz table.
elif isinstance(item, PointzItem):
self.cursor.execute(
"INSERT INTO pointz (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into kicks table.
elif isinstance(item, KicksItem):
self.cursor.execute(
"INSERT INTO kicks (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into shoespalace table.
elif isinstance(item, ShoesPalaceItem):
self.cursor.execute(
"INSERT INTO shoespalace (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into stickabush table.
elif isinstance(item, StickABushItem):
self.cursor.execute(
"INSERT INTO stickabush (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into kong table.
elif isinstance(item, KongItem):
self.cursor.execute(
"INSERT INTO kong (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into saveoursole table.
elif isinstance(item, SaveOurSoleItem):
self.cursor.execute(
"INSERT INTO saveoursole (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into inflammable table.
elif isinstance(item, InflammableItem):
self.cursor.execute(
"INSERT INTO inflammable (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into defshop table.
elif isinstance(item, DefShopItem):
self.cursor.execute(
"INSERT INTO defshop (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into offspring table.
elif isinstance(item, OffSpringItem):
self.cursor.execute(
"INSERT INTO offspring (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into solekitchen table.
elif isinstance(item, SoleKitchenItem):
self.cursor.execute(
"INSERT INTO solekitchen (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into drome table.
elif isinstance(item, DromeItem):
self.cursor.execute(
"INSERT INTO drome (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into footasylum table.
elif isinstance(item, FootAsylumItem):
self.cursor.execute(
"INSERT INTO footasylum (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into hhv table.
elif isinstance(item, HHVItem):
self.cursor.execute(
"INSERT INTO hhv (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into concepts table.
elif isinstance(item, ConceptsItem):
self.cursor.execute(
"INSERT INTO concepts (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into socialstatus table.
elif isinstance(item, SocialStatusItem):
self.cursor.execute(
"INSERT INTO socialstatus (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into extrabutter table.
elif isinstance(item, ExtraButterItem):
self.cursor.execute(
"INSERT INTO extrabutter (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into bodega table.
elif isinstance(item, BodegaItem):
self.cursor.execute(
"INSERT INTO bodega (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into saintalfred table.
elif isinstance(item, SaintAlfredItem):
self.cursor.execute(
"INSERT INTO saintalfred (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into lapstonenhammer table.
elif isinstance(item, LapstoneNHammerItem):
self.cursor.execute(
"INSERT INTO lapstonenhammer (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into shelflife table.
elif isinstance(item, ShelfLifeItem):
self.cursor.execute(
"INSERT INTO shelflife (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into asphaltgold table.
elif isinstance(item, AsphaltGoldItem):
self.cursor.execute(
"INSERT INTO asphaltgold (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into hanon table.
elif isinstance(item, HanonItem):
self.cursor.execute(
"INSERT INTO hanon (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into solebox table.
elif isinstance(item, SoleBoxItem):
self.cursor.execute(
"INSERT INTO solebox (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into consortium table.
elif isinstance(item, ConsortiumItem):
self.cursor.execute(
"INSERT INTO consortium (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into haven table.
elif isinstance(item, HavenItem):
self.cursor.execute(
"INSERT INTO haven (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into needsupply table.
elif isinstance(item, NeedSupplyItem):
self.cursor.execute(
"INSERT INTO needsupply (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into loaded table.
elif isinstance(item, LoadedItem):
self.cursor.execute(
"INSERT INTO loaded (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into wellgosh table.
elif isinstance(item, WellGoshItem):
self.cursor.execute(
"INSERT INTO wellgosh (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into capsule table.
elif isinstance(item, CapsuleItem):
self.cursor.execute(
"INSERT INTO capsule (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into yme table.
elif isinstance(item, YMEItem):
self.cursor.execute(
"INSERT INTO yme (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into hypedc table.
elif isinstance(item, HypeDCItem):
self.cursor.execute(
"INSERT INTO hypedc (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into holypop table.
elif isinstance(item, HolyPopItem):
self.cursor.execute(
"INSERT INTO holypop (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into bstn table.
elif isinstance(item, BSTNItem):
self.cursor.execute(
"INSERT INTO bstn (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into trophyroom table.
elif isinstance(item, TrophyRoomItem):
self.cursor.execute(
"INSERT INTO trophyroom (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into sidestep table.
elif isinstance(item, SideStepItem):
self.cursor.execute(
"INSERT INTO sidestep (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into shiekh table.
elif isinstance(item, ShiekhItem):
self.cursor.execute(
"INSERT INTO shiekh (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into rezet table.
elif isinstance(item, RezetItem):
self.cursor.execute(
"INSERT INTO rezet (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into footlockereu table.
elif isinstance(item, FootLockerEUItem):
self.cursor.execute(
"INSERT INTO footlockereu (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into office table.
elif isinstance(item, OfficeItem):
self.cursor.execute(
"INSERT INTO office (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into allike table.
elif isinstance(item, ALLikeItem):
self.cursor.execute(
"INSERT INTO allike (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into sportshoes table.
elif isinstance(item, SportsShoesItem):
self.cursor.execute(
"INSERT INTO sportshoes (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into runnerspoint table.
elif isinstance(item, RunnersPointItem):
self.cursor.execute(
"INSERT INTO runnerspoint (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into graffiti table.
elif isinstance(item, GraffitiItem):
self.cursor.execute(
"INSERT INTO graffiti (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into urbanjungle table.
elif isinstance(item, UrbanJungleItem):
self.cursor.execute(
"INSERT INTO urbanjungle (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into ssense table.
elif isinstance(item, SSenseItem):
self.cursor.execute(
"INSERT INTO ssense (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into backdoor table.
elif isinstance(item, BackDoorItem):
self.cursor.execute(
"INSERT INTO backdoor (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into basket table.
elif isinstance(item, BasketItem):
self.cursor.execute(
"INSERT INTO basket (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into oneblockdown table.
elif isinstance(item, OneBlockDownItem):
self.cursor.execute(
"INSERT INTO oneblockdown (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into dopefactory table.
elif isinstance(item, DopeFactoryItem):
self.cursor.execute(
"INSERT INTO dopefactory (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into nextdoor table.
elif isinstance(item, NextDoorItem):
self.cursor.execute(
"INSERT INTO nextdoor (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into summer table.
elif isinstance(item, SummerItem):
self.cursor.execute(
"INSERT INTO summer (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into mrporter table.
elif isinstance(item, MrPorterItem):
self.cursor.execute(
"INSERT INTO mrporter (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into stormfashion table.
elif isinstance(item, StormFashionItem):
self.cursor.execute(
"INSERT INTO stormfashion (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into tresbien table.
elif isinstance(item, TresBienItem):
self.cursor.execute(
"INSERT INTO tresbien (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into packer table.
elif isinstance(item, PackerItem):
self.cursor.execute(
"INSERT INTO packer (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into addict table.
elif isinstance(item, AddictItem):
self.cursor.execute(
"INSERT INTO addict (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into aphrodite table.
elif isinstance(item, AphroditeItem):
self.cursor.execute(
"INSERT INTO aphrodite (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into bait table.
elif isinstance(item, BaitItem):
self.cursor.execute(
"INSERT INTO bait (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into blends table.
elif isinstance(item, BlendsItem):
self.cursor.execute(
"INSERT INTO blends (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into nicekicks table.
elif isinstance(item, NiceKicksItem):
self.cursor.execute(
"INSERT INTO nicekicks (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into clicks table.
elif isinstance(item, ClicksItem):
self.cursor.execute(
"INSERT INTO clicks (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into feature table.
elif isinstance(item, FeatureItem):
self.cursor.execute(
"INSERT INTO feature (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into hypebeast table.
elif isinstance(item, HypeBeastItem):
self.cursor.execute(
"INSERT INTO hypebeast (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into deadstock table.
elif isinstance(item, DeadStockItem):
self.cursor.execute(
"INSERT INTO deadstock (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into notre table.
elif isinstance(item, NotreItem):
self.cursor.execute(
"INSERT INTO notre (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into nrml table.
elif isinstance(item, NrmlItem):
self.cursor.execute(
"INSERT INTO nrml (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into oneness table.
elif isinstance(item, OnenessItem):
self.cursor.execute(
"INSERT INTO oneness (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into pufferreds table.
elif isinstance(item, PufferRedsItem):
self.cursor.execute(
"INSERT INTO pufferreds (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into renarts table.
elif isinstance(item, RenartsItem):
self.cursor.execute(
"INSERT INTO renarts (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into shoesgallery table.
elif isinstance(item, ShoesGalleryItem):
self.cursor.execute(
"INSERT INTO shoesgallery (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into proper table.
elif isinstance(item, ProperItem):
self.cursor.execute(
"INSERT INTO proper (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into solestop table.
elif isinstance(item, SoleStopItem):
self.cursor.execute(
"INSERT INTO solestop (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into titolo table.
elif isinstance(item, TitoloItem):
self.cursor.execute(
"INSERT INTO titolo (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into uptown table.
elif isinstance(item, UptownItem):
self.cursor.execute(
"INSERT INTO uptown (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into westnyc table.
elif isinstance(item, WestNYCItem):
self.cursor.execute(
"INSERT INTO westnyc (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into wishatl table.
elif isinstance(item, WishATLItem):
self.cursor.execute(
"INSERT INTO wishatl (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into xileclothing table.
elif isinstance(item, XileClothingItem):
self.cursor.execute(
"INSERT INTO xileclothing (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into solefly table.
elif isinstance(item, SoleflyItem):
self.cursor.execute(
"INSERT INTO solefly (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into patta table.
elif isinstance(item, PattaItem):
self.cursor.execute(
"INSERT INTO patta (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into svd table.
elif isinstance(item, SVDItem):
self.cursor.execute(
"INSERT INTO svd (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into dsmny table.
elif isinstance(item, DSMNYItem):
self.cursor.execute(
"INSERT INTO dsmny (link, date) VALUES (%s, %s)",
(item['link'].encode('utf-8'), DATE))
# Insert item into hubbastille table.
elif isinstance(item, HubbastilleItem):
self.cursor.execute(
"INSERT INTO hubbastille (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into shoesaddictor table.
elif isinstance(item, ShoesAddictorItem):
self.cursor.execute(
"INSERT INTO shoesaddictor (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
self.conn.commit()
# If item name contain below words. Tweet it.
keywords = [
'ultra boost', 'air jordan', 'jordan retro', 'nmd', 'boost',
'retro', 'flyknit', 'yeezy', 'ronnie', 'fieg', 'pharrel',
'atmos', 'clots', 'mars', 'yard'
]
if any(keyword in item['name'].encode('utf-8').lower()
for keyword in keywords):
# Twitter Auth - Tweet the item with date, time, item name, and link.
# To obtain Twitter CONSUMER and ACCESS keys go to https://apps.twitter.com/
CONSUMER_KEY = 'PASTE CONSUMER_KEY HERE'
CONSUMER_SECRET = 'PASTE CONSUMER_SECRET HERE'
ACCESS_TOKEN_KEY = 'PASTE ACCESS_TOKEN_KEY HERE'
ACCESS_TOKEN_SECRET = 'PASTE ACCESS_TOKEN_SECRET HERE'
API = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET,
ACCESS_TOKEN_KEY, ACCESS_TOKEN_SECRET)
TEXT_TO_SEND = DATE + " EST " + item['name'] + " " + item[
'link']
TWEET = API.request('statuses/update',
{'status': TEXT_TO_SEND})
print(Fore.RED + 'TWEET LOG SUCCESS: ' + DATE + ' EST ' +
item['name'] + ' ' + item['link'] +
Style.RESET_ALL if TWEET.status_code ==
200 else Fore.RED +
'TWEET LOG FAILURE: FAILED TO TWEET' + Style.RESET_ALL)
# WebHook for Discord. Comment/Uncomment the line below to enable/disable.
# requests.post('DISCORD WEBHOOK URL', data={'content': "**" + item['name'] + "**" + "\n" + item['link'] + "\n" + "\n" + "[ATC]: " + item['size'] + "\n" + "\n"})
# WebHook for Slack. Comment/Uncomment the line below to enable/disable.
# requests.post('SLACK WEBHOOK URL', json={'text': "*" + item['name'] + "*" + "\n" + item['link'] + "\n" + "\n" + "[ATC]: " + item['size'] + "\n" + "\n"}, headers={'Content-Type': 'application/json'})
except MySQLdb.Error, e:
# print (Fore.RED + "MYSQL ERROR %d: %s" % (e.args[0], e.args[1] + Style.RESET_ALL))
return item
class Twitter:
def __init__(self):
"""
Params:
credential_file...list of JSON objects containing the four
required tokens: consumer_key, consumer_secret, access_token, access_secret
"""
self.credentials = [json.loads(credential)]
self.credential_cycler = cycle(self.credentials)
self.reinit_api()
def reinit_api(self):
creds = next(self.credential_cycler)
sys.stderr.write('switching creds to %s\n' % creds['consumer_key'])
self.twapi = TwitterAPI(creds['consumer_key'],
creds['consumer_secret'],
creds['access_token'], creds['token_secret'])
def request(self, endpoint, params):
while True:
try:
response = self.twapi.request(endpoint, params)
if response.status_code in RATE_LIMIT_CODES:
for _ in range(len(self.credentials) - 1):
self.reinit_api()
response = self.twapi.request(endpoint, params)
if response.status_code not in RATE_LIMIT_CODES:
return response
sys.stderr.write('sleeping for 15 minutes...\n')
time.sleep(
910
) # sleep for 15 minutes # FIXME: read the required wait time.
return self.request(endpoint, params)
else:
return response
except requests.exceptions.Timeout:
# handle requests.exceptions.ConnectionError Read timed out.
print("Timeout occurred. Retrying...")
time.sleep(5)
self.reinit_api()
def _get_followers(self, identifier_field, identifier, limit=1e10):
return self._paged_request('followers/ids', {
identifier_field: identifier,
'count': 5000,
'stringify_ids': True
}, limit)
def _get_users(self, identifier_field, identifier, limit=1e10):
request = self.request('users/lookup', {identifier_field: identifier})
#request = robust_request(twitter, "users/lookup" , {'screen_name': screen_names})
users = [r for r in request]
#users = [{'id': r['id'], 'name': r['name'], 'screen_name' : r['screen_name'], 'followers_count': r['followers_count'], 'friends_count': r['friends_count'], 'statuses_count' : r['statuses_count']} for r in request]
return users
def _get_friends(self, identifier_field, identifier, limit=1e10):
return self._paged_request('friends/ids', {
identifier_field: identifier,
'count': 5000,
'stringify_ids': True
}, limit)
def add_followers(self, users):
for u in users:
temp = self._get_followers('screen_name', u['screen_name'], 5000)
u.update({'followers': temp})
def _paged_request(self, endpoint, params, limit):
results = []
cursor = -1
while len(results) <= limit:
try:
response = self.request(endpoint, params)
print(response)
if response.status_code != 200:
sys.stderr.write('Skipping bad request: %s\n' %
response.text)
return results
else:
result = json.loads(response.text)
print(result['next_cursor'])
items = [r for r in response]
if len(items) == 0:
return results
else:
sys.stderr.write(
'fetched %d more results for %s\n' %
(len(items), params['screen_name'] if 'screen_name'
in params else params['user_id']))
time.sleep(1)
results.extend(items)
params['cursor'] = result['next_cursor']
except Exception as e:
sys.stderr.write('Error: %s\nskipping...\n' % e)
sys.stderr.write(traceback.format_exc())
return results
return results
def _get_tweets(self, identifier_field, identifier, limit=1e10):
max_id = None
tweets = []
while len(tweets) < limit:
try:
params = {
identifier_field: identifier,
'count': 200,
'max_id': max_id,
'tweet_mode': 'extended',
'trim_user': 0
}
if max_id:
params['max_id'] = max_id
response = self.request('statuses/user_timeline', params)
if response.status_code == 200: # success
items = [t for t in response]
if len(items) > 0:
sys.stderr.write('fetched %d more tweets for %s\n' %
(len(items), identifier))
tweets.extend(items)
else:
return tweets
max_id = min(t['id'] for t in response) - 1
else:
sys.stderr.write('Skipping bad user: %s\n' % response.text)
return tweets
except Exception as e:
sys.stderr.write('Error: %s\nskipping...\n' % e)
sys.stderr.write(traceback.format_exc() + '\n')
return tweets
#return tweets
res = [(r['id'], r['retweet_count']) for r in tweets]
sorted_res = sorted(res, key=lambda x: -x[1])
tweets = sorted_res[:25] #top 25 tweets
return tweets
def _add_tweets(self, users):
for u in users:
temp = self._get_tweets('screen_name', u['screen_name'], 200)
u.update({'tweets': temp})
def get_retweet_ids(self, identifier_field, identifier, limit=1e10):
retweets = []
cursor = -1
while len(retweets) <= limit:
params = {identifier_field: identifier}
response = self.request('statuses/retweeters/ids', params)
if response.status_code != 200:
sys.stderr.write('Skipping bad request: %s\n' % response.text)
return results
else:
result = json.loads(response.text)
items = [r for r in response]
if len(items) == 0:
return retweets
else:
sys.stderr.write(
'fetched %d more results for %s\n' %
(len(items), params['screen_name']
if 'screen_name' in params else params['id']))
time.sleep(1)
retweets.extend(items)
params['cursor'] = result['next_cursor']
return retweets
def add_retweet_ids(self, users):
for u in users:
result = []
for tweets in u['tweets']:
temp = self.get_retweet_ids('id', tweets[0], 5)
result.extend(temp)
counter_retweet = Counter(result)
u.update({'retweet_counts': counter_retweet})
def influence_factor(self, users):
influence_factor = {}
total_tweets_author = len(users['tweets'])
#print(users['retweet_counts'])
for keys in users['retweet_counts']:
retweet_by_follower = users['retweet_counts'][keys]
influence_factor[keys] = (retweet_by_follower /
total_tweets_author)
users.update({'influence_factor': influence_factor})
def add_influence_factor(self, users):
for u in users:
temp = self.influence_factor(u)
def get_retweet_followers(self, json_filename):
with open(json_filename) as filein:
input_json = json.load(filein)
retweet_followers_list = []
for retweet_id in input_json:
for followers_id in retweet_id['retweet_counts']:
if retweet_id['retweet_counts'][followers_id] > 1:
retweet_followers_list.append(followers_id)
return retweet_followers_list
created_at=datetime.now())
print("OK")
else:
print("SKIP")
COMMANDS = {"add": add_to_story, "start": start_story}
twitter = TwitterAPI(consumer_key="",
consumer_secret="",
access_token_key="",
access_token_secret="")
tweets = twitter.request("statuses/user_timeline", {
"count": 50,
"trim_user": True,
"exclude_replies": True,
"include_rts": True
})
for tweet in reversed(list(tweets)):
print("Tweet: %s" % tweet)
is_processed = False
for command, action in COMMANDS.items():
if tweet["text"].startswith(command):
action(tweet)
is_processed = True
break
if not is_processed:
for url in tweet["entities"]["urls"]:
if url["expanded_url"].startswith("http://i.vas3k.ru"):
).get_retry_strategy()
## Put tweets in Oracle stream -----------------------------------------------------------------------------------------------
# Endless stream
message_list = []
while True:
# Get or create a stream with stream_name
stream = get_or_create_stream(stream_admin_client, compartment_id,
stream_name, partitions,
stream_admin_client_composite).data
stream_id = stream.id
# Connect to the stream using the provided message endpoint.
stream_client = oci.streaming.StreamClient(
config_oci, service_endpoint=stream.messages_endpoint)
try:
r = api.request('statuses/filter', {'track': keywords})
for item in r:
if 'text' in item:
message_list.append(
oci.streaming.models.PutMessagesDetailsEntry(
key=b64encode(str(time.time()).encode()).decode(),
value=b64encode(json.dumps(item).encode()).decode()))
elif 'disconnect' in item:
event = item['disconnect']
if event['code'] in [2, 5, 6, 7]:
raise Exception(
event['reason']
) # something needs to be fixed before re-connecting
else:
break # temporary interruption, re-try request
if len(message_list) == 50:
class TweetClient:
""" Twitter Client
Responds to mentions.
"""
def __init__(self, config, lnrpc):
self.api = TwitterAPI(config['consumer_key'],
config['consumer_secret'],
config['access_token'],
config['access_token_secret'])
self.lnrpc = lnrpc
bot_id = config['access_token'].split('-')[0]
bot_raw = self.api.request('users/lookup', {'user_id': bot_id})
try:
self.bot = bot_raw.json()[0]
logging.debug(self.bot)
except KeyError:
print("Cannot access Twitter account. Check API keys.")
raise
def _post(self, msg, reply_sid, media_id=None):
options = {
'status': msg,
'in_reply_to_status_id': reply_sid,
'auto_populate_reply_metadata': True,
'media_ids': media_id
}
tweet = self.api.request('statuses/update', options).json()
logging.info(tweet)
return tweet.get('id_str')
def _send_invoice(self, reply_sid):
memo = "ScreenshotBot #%s" % reply_sid
msg = self.lnrpc.get_invoice(memo)
sid = self._post(msg, reply_sid)
return sid
def _return_image(self, file, reply_sid):
img = open(file, "rb").read()
r = self.api.request('media/upload', None, {'media': img})
logging.info(r.json())
if r.status_code == 200:
media_id = r.json()['media_id']
self._post('Here you go:', reply_sid, media_id)
else:
logging.info("Media upload failed: %s" % file)
self._post('Error', reply_sid)
def _send_receipt(self, memo):
id_str = memo.split('#')[-1]
file = glob.glob('%s-*.png' % id_str)[0]
reply_sid = file[file.find('-') + 1:file.find('.')]
self._return_image(file, reply_sid)
def watch(self):
"""
Filter tweets based on bot's screen name
"""
while True:
try:
msgs = self.api.request('statuses/filter',
{'track': self.bot.get('screen_name')})
for m in msgs:
logging.info(m)
urls = m.get('entities').get('urls')
if len(urls) == 0:
continue
tweet_url = urls[-1]
sid = m.get('id_str')
r = self._send_invoice(sid)
logging.info(r)
screenshot(tweet_url.get('expanded_url'),
'%s-%s' % (sid, r))
continue
except TwitterAPI.TwitterError as e:
logging.warning("Twitter Error: %s" % e)
else:
logging.error("Failure while watching stream")
break
def get_invoices(self):
invoices = self.lnrpc.subscribe_invoices()
for invoice in invoices:
logging.info(invoice)
if invoice.settled:
self._send_receipt(invoice.memo)
def go(self):
watch = threading.Thread(target=self.watch)
get_invoices = threading.Thread(target=self.get_invoices)
watch.start()
get_invoices.start()
if i < LENGTH:
strip[i] = (255, 255, 255)
tree.put_pixels(strip * STRIPS)
time.sleep(0.025)
strip = [(r, g, b)] * LENGTH
tree.put_pixels(strip * STRIPS)
LAST = (r, g, b)
colour_tree(255, 0, 0)
r = twitter.request('statuses/filter', {'track': "port57tree"})
for tweet in r:
if 'text' in tweet:
changed = False
print tweet['text']
for r, g, b, colour in COLOURS:
if "%s" % colour.lower() in tweet['text'].lower():
wipe(r, g, b)
print colour
changed = True
break
if not changed:
for colour, r, g, b in COLOURS2:
if "%s" % colour.lower() in tweet['text'].lower():
wipe(r, g, b)
def tweet_counter():
global word_set_query
global word_set
global track
# Twitter authorization keys
consumer_key = '*****************'
consumer_secret = '***************'
oauth_token = '*****************'
oauth_token_secret = '******************'
api = TwitterAPI(consumer_key, consumer_secret, oauth_token, oauth_token_secret)
while True:
# DB connection to store data
conn = sqlite3.connect('twitter.db', check_same_thread=False)
cursor = conn.cursor()
track = {'track': word_set_query}
print(track)
request = request_statuses
request_options = track
msg_in_queue = False
sent_request = False
if ipc_queue.qsize() > 0:
msg_in_queue = True
if msg_in_queue is False:
r = api.request(request, request_options)
sent_request = True
else:
process_message()
if sent_request:
try:
for item in r.get_iterator() or ipc_queue.qsize() > 0:
if ipc_queue.qsize() > 0:
try:
r.close()
print("Msg in queue. Will sleep for a few secs to avoid error 420")
time.sleep(time_between_requests_secs)
process_message()
break
except:
print("Empty queue")
if 'text' in item:
print(item, flush=True)
for word in word_set:
if str(word) in item['text']:
rows = cursor.execute("SELECT * FROM tweets WHERE word like (?)", (str(word),))
non_zero_rows = rows.fetchone()
if non_zero_rows:
cursor.execute("UPDATE tweets SET {0} = {1} + 1 WHERE word LIKE \"{2}\"".
format(str(current_day), str(current_day), word))
else:
insert_row = get_insert_row_format()
cursor.execute(insert_row, (str(word), 1,))
conn.commit()
elif 'limit' in item:
skip = item['limit'].get('track')
print('*** SKIPPED %d TWEETS' % skip)
elif 'disconnect' in item:
print('[disconnect] %s' % item['disconnect'].get('reason'))
cursor.close()
conn.close()
break
except TwitterRequestError as e:
if e.status_code < 500:
print("Something critical failed")
else:
print("Temporary failure")
pass
time.sleep(time_between_requests_secs)
except TwitterConnectionError:
print("Connection error")
time.sleep(time_between_requests_secs)
cursor.close()
conn.close()
class Twitter:
def __init__(self, credential_file):
"""
Params:
credential_file...list of JSON objects containing the four
required tokens: consumer_key, consumer_secret, access_token, access_secret
"""
self.credentials = [json.loads(l) for l in open(credential_file)]
self.credential_cycler = cycle(self.credentials)
self.reinit_api()
def reinit_api(self):
creds = next(self.credential_cycler)
sys.stderr.write('switching creds to %s\n' % creds['consumer_key'])
self.twapi = TwitterAPI(creds['consumer_key'],
creds['consumer_secret'],
creds['access_token'], creds['token_secret'])
def request(self, endpoint, params):
while True:
try:
response = self.twapi.request(endpoint, params)
if response.status_code in RATE_LIMIT_CODES:
for _ in range(len(self.credentials) - 1):
self.reinit_api()
response = self.twapi.request(endpoint, params)
if response.status_code not in RATE_LIMIT_CODES:
return response
sys.stderr.write('sleeping for 15 minutes...\n')
time.sleep(
910
) # sleep for 15 minutes # FIXME: read the required wait time.
return self.request(endpoint, params)
else:
return response
except requests.exceptions.Timeout:
# handle requests.exceptions.ConnectionError Read timed out.
print("Timeout occurred. Retrying...")
time.sleep(5)
self.reinit_api()
def followers_for_id(self, theid, limit=1e10):
return self._get_followers('user_id', theid, limit)
def followers_for_screen_name(self, screen_name, limit=1e10):
return self._get_followers('screen_name', screen_name, limit)
def _get_followers(self, identifier_field, identifier, limit=1e10):
return self._paged_request('followers/ids', {
identifier_field: identifier,
'count': 5000,
'stringify_ids': True
}, limit)
def friends_for_id(self, theid, limit=1e10):
return self._get_friends('user_id', theid, limit)
def friends_for_screen_name(self, screen_name, limit=1e10):
return self._get_friends('screen_name', screen_name, limit)
def _get_friends(self, identifier_field, identifier, limit=1e10):
return self._paged_request('friends/ids', {
identifier_field: identifier,
'count': 5000,
'stringify_ids': True
}, limit)
def _paged_request(self, endpoint, params, limit):
results = []
cursor = -1
while len(results) <= limit:
try:
response = self.request(endpoint, params)
if response.status_code != 200:
sys.stderr.write('Skipping bad request: %s\n' %
response.text)
return results
else:
result = json.loads(response.text)
items = [r for r in response]
if len(items) == 0:
return results
else:
sys.stderr.write(
'fetched %d more results for %s\n' %
(len(items), params['screen_name'] if 'screen_name'
in params else params['user_id']))
time.sleep(1)
results.extend(items)
params['cursor'] = result['next_cursor']
except Exception as e:
sys.stderr.write('Error: %s\nskipping...\n' % e)
sys.stderr.write(traceback.format_exc())
return results
return results
def tweets_for_id(self, theid, limit=1e10):
return self._get_tweets('user_id', theid, limit)
def tweets_for_screen_name(self, screen_name, limit=1e10):
return self._get_tweets('screen_name', screen_name, limit)
def _get_tweets(self, identifier_field, identifier, limit=1e10):
max_id = None
tweets = []
while len(tweets) < limit:
try:
params = {
identifier_field: identifier,
'count': 200,
'max_id': max_id,
'tweet_mode': 'extended',
'trim_user': 0
}
if max_id:
params['max_id'] = max_id
response = self.request('statuses/user_timeline', params)
if response.status_code == 200: # success
items = [t for t in response]
if len(items) > 0:
sys.stderr.write('fetched %d more tweets for %s\n' %
(len(items), identifier))
tweets.extend(items)
else:
return tweets
max_id = min(t['id'] for t in response) - 1
else:
sys.stderr.write('Skipping bad user: %s\n' % response.text)
return tweets
except Exception as e:
sys.stderr.write('Error: %s\nskipping...\n' % e)
sys.stderr.write(traceback.format_exc() + '\n')
return tweets
return tweets
#L'authentification
api = TwitterAPI(twitter_credentials.CONSUMER_KEY,
twitter_credentials.CONSUMER_SECRET,
twitter_credentials.ACCESS_TOKEN,
twitter_credentials.ACCESS_TOKEN_SECRET)
#********30-day endpoint********#
QUERY = "(heart disease OR cardiovascular OR heart failure OR coronary OR ischemic) place_country:US lang:en"
PRODUCT = '30day'
#L'environement de developement de l'application
LABEL = 'PFE'
#La requete
r = api.request('tweets/search/%s/:%s' % (PRODUCT, LABEL), {
'query': QUERY,
'fromDate': '20190506000',
'toDate': '201905070000'
})
#Ouverture du fichier Json
jsonFile = open('tweets.json', 'a')
#Ouverture du fichier CSV
tweets = open('tweets.csv', 'a', newline='')
csvWriter = csv.writer(tweets)
#Parcours du resultat obtenu par la requete
for item in r:
#Teste si le tweet contient 140 caracteres ou plus
if ('extended_tweet' in item):
text = item['extended_tweet']['full_text']
else:
## Example to use twitter api and feed data into kinesis
from TwitterAPI import TwitterAPI
import boto3
import json
import twitterCreds
## twitter credentials
consumer_key = twitterCreds.consumer_key
consumer_secret = twitterCreds.consumer_secret
access_token_key = twitterCreds.access_token_key
access_token_secret = twitterCreds.access_token_secret
api = TwitterAPI(consumer_key, consumer_secret, access_token_key,
access_token_secret)
kinesis = boto3.client('kinesis')
r = api.request('statuses/filter', {'locations': '-90,-90,90,90'})
for item in r:
kinesis.put_record(StreamName="twitter",
Data=json.dumps(item),
PartitionKey="filler")
# Establish twitter info:
delay = 8 # seconds
api_details_path = '/Users/ilya/Projects/twitter_spam'
api_details = []
with open(api_details_path + '/api_details.txt', 'r') as a:
info = a.readlines()
api_details.append(info)
consumer_key, consumer_secret, access_token_key, access_token_secret = api_details[0][0].split(',')
while True:
try:
api = TwitterAPI(consumer_key, consumer_secret,
access_token_key, access_token_secret)
r = api.request('statuses/sample', {'language' : 'en'})
for item in r.get_iterator():
try:
if 'retweeted_status' not in item:
collect.insert(item)
delay = max(8, delay/2)
except:
continue
except:
print "Error"
print time.ctime()
print "Waiting " + str(delay) + " seconds"
time.sleep(delay)
delay *= 2
user_names = []
f = open("user_names.txt", "r")
for line in f:
user_names.append(line.rstrip())
count = 0
tweets = []
cls_label = []
for name in user_names:
if count < 450:
count += 1
try:
results = api.request('statuses/user_timeline', {
'screen_name': name,
'count': 200
})
# Creation of X
User_tweets = ' '
for result in results:
User_tweets = User_tweets + stopwords_removal(
result['text']) + ', '
if len(User_tweets) != 0:
tweets.append(User_tweets)
# Creation of y
male_count, female_count = identification(User_tweets)
if male_count > female_count:
cls_label.append(1) # print "male"
class TwitterNotificationService(BaseNotificationService):
"""Implementation of a notification service for the Twitter service."""
def __init__(
self,
hass,
consumer_key,
consumer_secret,
access_token_key,
access_token_secret,
username,
):
"""Initialize the service."""
self.user = username
self.hass = hass
self.api = TwitterAPI(
consumer_key, consumer_secret, access_token_key, access_token_secret
)
def send_message(self, message="", **kwargs):
"""Tweet a message, optionally with media."""
data = kwargs.get(ATTR_DATA)
media = None
if data:
media = data.get(ATTR_MEDIA)
if not self.hass.config.is_allowed_path(media):
_LOGGER.warning("'%s' is not a whitelisted directory", media)
return
callback = partial(self.send_message_callback, message)
self.upload_media_then_callback(callback, media)
def send_message_callback(self, message, media_id=None):
"""Tweet a message, optionally with media."""
if self.user:
user_resp = self.api.request("users/lookup", {"screen_name": self.user})
user_id = user_resp.json()[0]["id"]
if user_resp.status_code != HTTPStatus.OK:
self.log_error_resp(user_resp)
else:
_LOGGER.debug("Message posted: %s", user_resp.json())
event = {
"event": {
"type": "message_create",
"message_create": {
"target": {"recipient_id": user_id},
"message_data": {"text": message},
},
}
}
resp = self.api.request("direct_messages/events/new", json.dumps(event))
else:
resp = self.api.request(
"statuses/update", {"status": message, "media_ids": media_id}
)
if resp.status_code != HTTPStatus.OK:
self.log_error_resp(resp)
else:
_LOGGER.debug("Message posted: %s", resp.json())
def upload_media_then_callback(self, callback, media_path=None):
"""Upload media."""
if not media_path:
return callback()
with open(media_path, "rb") as file:
total_bytes = os.path.getsize(media_path)
(media_category, media_type) = self.media_info(media_path)
resp = self.upload_media_init(media_type, media_category, total_bytes)
if 199 > resp.status_code < 300:
self.log_error_resp(resp)
return None
media_id = resp.json()["media_id"]
media_id = self.upload_media_chunked(file, total_bytes, media_id)
resp = self.upload_media_finalize(media_id)
if 199 > resp.status_code < 300:
self.log_error_resp(resp)
return None
if resp.json().get("processing_info") is None:
return callback(media_id)
self.check_status_until_done(media_id, callback)
def media_info(self, media_path):
"""Determine mime type and Twitter media category for given media."""
(media_type, _) = mimetypes.guess_type(media_path)
media_category = self.media_category_for_type(media_type)
_LOGGER.debug(
"media %s is mime type %s and translates to %s",
media_path,
media_type,
media_category,
)
return media_category, media_type
def upload_media_init(self, media_type, media_category, total_bytes):
"""Upload media, INIT phase."""
return self.api.request(
"media/upload",
{
"command": "INIT",
"media_type": media_type,
"media_category": media_category,
"total_bytes": total_bytes,
},
)
def upload_media_chunked(self, file, total_bytes, media_id):
"""Upload media, chunked append."""
segment_id = 0
bytes_sent = 0
while bytes_sent < total_bytes:
chunk = file.read(4 * 1024 * 1024)
resp = self.upload_media_append(chunk, media_id, segment_id)
if not HTTPStatus.OK <= resp.status_code < HTTPStatus.MULTIPLE_CHOICES:
self.log_error_resp_append(resp)
return None
segment_id = segment_id + 1
bytes_sent = file.tell()
self.log_bytes_sent(bytes_sent, total_bytes)
return media_id
def upload_media_append(self, chunk, media_id, segment_id):
"""Upload media, APPEND phase."""
return self.api.request(
"media/upload",
{"command": "APPEND", "media_id": media_id, "segment_index": segment_id},
{"media": chunk},
)
def upload_media_finalize(self, media_id):
"""Upload media, FINALIZE phase."""
return self.api.request(
"media/upload", {"command": "FINALIZE", "media_id": media_id}
)
def check_status_until_done(self, media_id, callback, *args):
"""Upload media, STATUS phase."""
resp = self.api.request(
"media/upload",
{"command": "STATUS", "media_id": media_id},
method_override="GET",
)
if resp.status_code != HTTPStatus.OK:
_LOGGER.error("Media processing error: %s", resp.json())
processing_info = resp.json()["processing_info"]
_LOGGER.debug("media processing %s status: %s", media_id, processing_info)
if processing_info["state"] in {"succeeded", "failed"}:
return callback(media_id)
check_after_secs = processing_info["check_after_secs"]
_LOGGER.debug(
"media processing waiting %s seconds to check status", str(check_after_secs)
)
when = datetime.now() + timedelta(seconds=check_after_secs)
myself = partial(self.check_status_until_done, media_id, callback)
async_track_point_in_time(self.hass, myself, when)
@staticmethod
def media_category_for_type(media_type):
"""Determine Twitter media category by mime type."""
if media_type is None:
return None
if media_type.startswith("image/gif"):
return "tweet_gif"
if media_type.startswith("video/"):
return "tweet_video"
if media_type.startswith("image/"):
return "tweet_image"
return None
@staticmethod
def log_bytes_sent(bytes_sent, total_bytes):
"""Log upload progress."""
_LOGGER.debug("%s of %s bytes uploaded", str(bytes_sent), str(total_bytes))
@staticmethod
def log_error_resp(resp):
"""Log error response."""
obj = json.loads(resp.text)
if "errors" in obj:
error_message = obj["errors"]
elif "error" in obj:
error_message = obj["error"]
else:
error_message = resp.text
_LOGGER.error("Error %s: %s", resp.status_code, error_message)
@staticmethod
def log_error_resp_append(resp):
"""Log error response, during upload append phase."""
obj = json.loads(resp.text)
error_message = obj["errors"][0]["message"]
error_code = obj["errors"][0]["code"]
_LOGGER.error(
"Error %s: %s (Code %s)", resp.status_code, error_message, error_code
)
# Shaafe Khan
from TwitterAPI import TwitterAPI # importeren twitter API
consumer_key = 'NewUDIZy2opOqnsWlRq3EQutT'
consumer_secret = 'pzskqXmoSznLTPe5Qa6XjoxrgCtXT1qG305nMzVV8ZIGWg2i6H'
access_token_key = '1188970441303568385-s6sugkh2nDv7e1Y1DFHUXaPhuQI5X6'
access_secret = 'Ga40JxhyYYtp99wLvRxkL5jcsOrOwm54TipIz76vp2NAH'
# Alle noodzakelijke keys voor het posten van tweets
api = TwitterAPI(consumer_key, consumer_secret, access_token_key,
access_secret)
# variable van twitterapi functie en de keys
gebr_naam = "Rick"
gebr_tweet = "NS is heilig"
# naam en tweet te krijgen via GUI
def gebr_naam_tweet(naam, tweet):
naam_en_tweet = "\"{0}\" - {1}".format(tweet, naam)
return naam_en_tweet
# hier worden de namen en tweet samengevoegd
tweet_post = api.request('statuses/update',
{'status': gebr_naam_tweet(gebr_naam, gebr_tweet)})
# naam en tweet worden gepost
print(tweet_post.status_code)
# error code print
from TwitterAPI import TwitterAPI
CONSUMER_KEY = 'eDWLh4zSdo5SIFMNM6CxebAQ7'
CONSUMER_SECRET = 'Vom9W8K5KX1VKTqKGPzPNF5XYEvFeBuR4GprF5lTIZAQbQ0qqP'
ACCESS_TOKEN_KEY = '823063147712020481-MS9kF7Hm1fc3TjNm6NbaDNrfbIqX6O9'
ACCESS_TOKEN_SECRET = 'RMkdy5xS27Y0t0E8gkmnH8CGFu4jW5K2Dri1EBDTcASte'
b = 1
while True:
b = b + 1
api = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN_KEY,
ACCESS_TOKEN_SECRET)
file = open('/home/pi/cams/imags/' + str(b) + '.jpg', 'rb')
print('done')
data = file.read()
r = api.request('statuses/update_with_media', {'status': '#pyTweetCMR'},
{'media[]': data})
print(r.status_code)
# smtp_server = creds['smtp_server']
# phone = creds['phone_number']
# carrier = creds['carrier']
# Keywords & locations to filter the tweets by
with open('filters.json', 'r') as filters_file:
filters = json.load(filters_file)
keywords = filters["keywords"]
locations = [loc.lower() for loc in filters["locations"]]
# Initialize the API.
api = TwitterAPI(creds['consumer_key'], creds['consumer_secret'],
creds['access_token'], creds['access_token_secret'])
# Access the stream. Filter by keywords
stream = api.request('statuses/filter', {'track': ",".join(keywords)})
### Create sqlite3 database
# con = sqlite3.connect('flags_stream.db')
# con.text_factory = str
# cur = con.cursor()
# Clear out the previous result files.
with open('flagged_tweets.txt', 'w') as output_file:
output_file.write("")
with open('flagged_tweets_raw.txt', 'w') as output_file_raw:
output_file_raw.write("")
# Consume streaming tweets
for tweet in stream.get_iterator():
# Get location & possibly exact address
# python 3
sys.stdout = codecs.getwriter('utf8')(sys.stdout.buffer)
sys.stderr = codecs.getwriter('utf8')(sys.stderr.buffer)
except:
# python 2
sys.stdout = codecs.getwriter('utf8')(sys.stdout)
sys.stderr = codecs.getwriter('utf8')(sys.stderr)
# SAVE YOUR APPLICATION CREDENTIALS IN TwitterAPI/credentials.txt.
o = TwitterOAuth.read_file()
api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret)
# GET 20 TWEETS CONTAINING 'ZZZ'
api.request('search/tweets', {'q':'zzz'})
iter = api.get_iterator()
for item in iter:
sys.stdout.write('%s\n' % item['text'])
"""
# POST A TWEET
sys.stdout.write('%s\n' % api.request('statuses/update', {'status':'This is another tweet!'}))
# STREAM TWEETS FROM AROUND NYC
api.request('statuses/filter', {'locations':'-74,40,-73,41'})
iter = api.get_iterator()
for item in iter:
sys.stdout.write('%s\n' % item['text'])
# GET TWEETS FROM THE PAST WEEK OR SO CONTAINING 'LOVE'
from TwitterAPI import TwitterAPI
TWEET_TEXT = "||Shree Swami Samartha||"
CONSUMER_KEY = 'iAi9l9dLvBzKinhUdFJQ2dHaa'
CONSUMER_SECRET = 'eMBJTCjSKxd8tzHrmDLqB4yMtmleQSoXE6V1I2SP8YKAdUdP8N'
ACCESS_TOKEN_KEY = '3235720159-kYyjk5TbiB4BYjIgdRwiEmLM2NzZvqXL3UjsKuj'
ACCESS_TOKEN_SECRET = 'HtAD0ZNIJCe850y2KcUESMtfM8hrtQYSbwJZm6dI8ROGp'
api = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN_KEY,
ACCESS_TOKEN_SECRET)
r = api.request('statuses/update', {'status': TWEET_TEXT})
print('SUCCESS' if r.status_code == 200 else 'FAILURE')
