def fetch_twitter_followers(self):
twitter = Twython(APP_KEY_TWITTER, APP_SECRET_TWITTER,
OAUTH_TOKEN_TWITTER, OAUTH_TOKEN_SECRET_TWITTER)
twitter_obj = twitter.get_followers_ids()
twitter_followers = len(twitter_obj['ids'])
twitter_followers = '{:,}'.format(int(twitter_followers))
self.twitter = twitter_followers
def login():
# Go to http://twitter.com/apps/new to create an app and get these items
# See also http://dev.twitter.com/pages/oauth_single_token
server = couchdb.Server()
db = server['tweepsnigeria_followers']
db2 = server['tweetsoccupynigeria']
APP_NAME = 'traffic_ug'
TWITTER_APP_KEY = 'U31NQIxZ1HcKeKTUtTW1w' #supply the appropriate value
TWITTER_APP_KEY_SECRET = 'bZBtxjQurHUfjDvK8Pl5iJWhgvzejvC2HJHSN1E'
TWITTER_ACCESS_TOKEN = '1548279967-gDMrGBaZIb3jofVekP25SnpWZrnhxSSFra7GgRV'
TWITTER_ACCESS_TOKEN_SECRET = 'zLwSwgsMZxtH0tlB2x1UU217nq0ujlpkzP8SCXa07u8'
t = Twython(app_key=TWITTER_APP_KEY,
app_secret=TWITTER_APP_KEY_SECRET,
oauth_token=TWITTER_ACCESS_TOKEN,
oauth_token_secret=TWITTER_ACCESS_TOKEN_SECRET)
names = []
in_file = open('tweeps_no_followers.txt', 'rb')
count = 1
count2 = 1
for row in in_file:
name = row.strip()
if (count >= 608):
print count, row
profile = t.get_followers_ids(screen_name=name)
db[name] = profile
count2 += 1
count += 1
if ((count2 % 16) == 0):
time.sleep(15 * 60)
count2 = 1
in_file.close()
def login():
# Go to http://twitter.com/apps/new to create an app and get these items
# See also http://dev.twitter.com/pages/oauth_single_token
server = couchdb.Server()
db = server['tweepsnigeria_followers']
db2= server['tweetsoccupynigeria']
APP_NAME = 'traffic_ug'
TWITTER_APP_KEY = 'U31NQIxZ1HcKeKTUtTW1w' #supply the appropriate value
TWITTER_APP_KEY_SECRET = 'bZBtxjQurHUfjDvK8Pl5iJWhgvzejvC2HJHSN1E'
TWITTER_ACCESS_TOKEN = '1548279967-gDMrGBaZIb3jofVekP25SnpWZrnhxSSFra7GgRV'
TWITTER_ACCESS_TOKEN_SECRET = 'zLwSwgsMZxtH0tlB2x1UU217nq0ujlpkzP8SCXa07u8'
t = Twython(app_key=TWITTER_APP_KEY,
app_secret=TWITTER_APP_KEY_SECRET,
oauth_token=TWITTER_ACCESS_TOKEN,
oauth_token_secret=TWITTER_ACCESS_TOKEN_SECRET)
names=[]
in_file=open('tweeps_no_followers.txt','rb')
count=1
count2=1
for row in in_file:
name=row.strip()
if (count>=608):
print count, row
profile=t.get_followers_ids(screen_name=name)
db[name] = profile
count2+=1
count+=1
if((count2%16)==0):
time.sleep(15*60)
count2=1
in_file.close()
def get_followers(creds, user):
# Instantiate an object
python_tweets = Twython(creds['CONSUMER_KEY'], creds['CONSUMER_SECRET'])
followers = python_tweets.get_followers_ids(screen_name=user)
ids = [str(x) for x in followers['ids']]
subset = list(chunks(ids, 100))
screen_names = []
for c in subset:
success = False
while not success:
try:
comma_separated_string = ','.join(c)
output = python_tweets.lookup_user(
user_id=comma_separated_string)
for u in output:
screen_names.append(u['screen_name'])
success = True
except:
print('Waiting rate limit ...')
success = False
return (screen_names)
def get_followers_following(usr_id):
index = 0
print "GETTING FOLLOWERS AND FOLLOWING: " + str(usr_id)
friend_cursor = -1
follower_cursor = -1
APP_KEY = APP_KEYS[index]
APP_SECRET = APP_SECRETS[index]
OAUTH_TOKEN = OAUTH_TOKENS[index]
OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index]
while (1==1):
try:
twitter = Twython (APP_KEY, APP_SECRET)
auth = twitter.get_authentication_tokens()
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
friends = ""
while (friend_cursor != 0):
following = twitter.get_friends_ids(id = usr_id, cursor = friend_cursor)
for ID in following['ids']:
friends += str(ID) + " "
friend_cursor = following["next_cursor"]
friends = friends[:-1]
if len(friends) == 0:
friends = "null"
follow = ""
while (follower_cursor != 0):
followers = twitter.get_followers_ids(id = usr_id,cursor= follower_cursor)
for ID in followers['ids']:
follow += str(ID) + " "
follower_cursor = followers["next_cursor"]
follow= follow[:-1]
if len(follow) == 0:
follow = "null"
return (friends,follow)
except Exception as e:
print e
if "429 (Too Many Requests)" in str(e):
#global index
index += 1
if index == len(APP_KEYS):
index = 0
print "sleepy time - 15 minutes"
print datetime.datetime.now()
time.sleep(910)
return get_followers_following(usr_id)
elif "401 (Unauthorized)" in str(e):
print "401 error"
return ("null","null")
elif "404 (Not Found)" in str(e):
print "404 error"
return ("null","null")
else:
print e
return ("null","null")
def main(argv):
os.system('clear')
print 'Queue Processor v.'+VERSION+'\n'
print 'Starting the "friending" procedure for:'
for q in Queue.objects.filter(is_processed=False):
q.processing = True
q.save()
user = q.user
profile = Profile.objects.get(user=user)
target = q.requested_twitter_user_handle
processing = q.processing
is_processed = q.is_processed
if(is_processed): continue
print '\nOur target:',color.BOLD + '@' + target + color.END + '\n'
oauth_token = profile.oauth_token
oauth_token_secret = profile.oauth_secret
twitter = Twython(APP_KEY, APP_SECRET, oauth_token, oauth_token_secret)
try:
followers = twitter.get_followers_ids(screen_name = target, count=300)
except:
print 'Skipping this one...'
q.is_processed = True
q.save()
continue
count = 1
for follower_id in followers['ids']:
print count, '..befriending', color.GREEN + 'id_' + str(follower_id) + color.END
try:
twitter.create_friendship(screen_name=target, follow="true")
except TwythonError as e:
print target
print '\n', e
sleep_time = random.randint(12,90)
print '..sleeping for',sleep_time,'seconds (180 request in 15 mins = 1 request at minimum of 12 seconds)'
time.sleep(sleep_time)
count+=1
q.is_processed = 1
q.save()
class Command(BaseCommand):
def __init__(self):
self.twitter = Twython(CONSUMER_KEY, CONSUMER_SECRET)
def handle(self, *args, **options):
current_users = [user.twitter_id for user in TwitterUser.objects.all()]
# Update DB with latest followers
for user_id in self.twitter.get_followers_ids(screen_name='John_DM')["ids"]:
if user_id not in current_users:
TwitterUser.objects.create(twitter_id = user_id)
# Prioritises users who signed up most recently
# Should work through backlog if we dont get > 100 followers/day
users_to_follow = TwitterUser.objects.filter(created__lte=timezone.now() - timedelta(days=3), followed__isnull=True).order_by('-created')[:100]
for target in users_to_follow:
self.twitter.create_friendship(user_id=target.twitter_id, follow="true")
target.follow()
time.sleep(61) # Is there a better way to do rate limiting?
def getFollowers(screen_name):
consumer_key = "MLGdNZCfmzGthHTAyJU4KFvbU"
consumer_secret = "Tfp7DIZcJLbnS8BR5CWQmZklrhsbtc3fMfssKPT4SZoYsPiQKw"
access_token = "2383540880-s2C8xPgA4ITF7QnLRFnHK1es2UEbmW8qHQ87sX5"
access_token_secret = "kLYgBTPeslLgaFugCx0PoiBpPIKnyCBEVfqqJCkjsSKpP"
twitter = Twython(consumer_key, consumer_secret, access_token,
access_token_secret)
while (True):
try:
followers = twitter.get_followers_ids(screen_name=screen_name)
followers_id = followers['ids']
return followers_id
except TwythonRateLimitError:
print("Fall asleep")
sleep(300)
pass
except wythonError:
print("404 not found")
return []
except TwythonAuthError:
print("Bad Authentication")
return []
def getFollowers(screen_name):
consumer_key = "MLGdNZCfmzGthHTAyJU4KFvbU"
consumer_secret ="Tfp7DIZcJLbnS8BR5CWQmZklrhsbtc3fMfssKPT4SZoYsPiQKw"
access_token ="2383540880-s2C8xPgA4ITF7QnLRFnHK1es2UEbmW8qHQ87sX5"
access_token_secret ="kLYgBTPeslLgaFugCx0PoiBpPIKnyCBEVfqqJCkjsSKpP"
twitter = Twython(consumer_key, consumer_secret,access_token,access_token_secret )
while(True):
try:
followers = twitter.get_followers_ids(screen_name = screen_name)
followers_id = followers['ids']
return followers_id
except TwythonRateLimitError:
print("Fall asleep")
sleep(300)
pass
except wythonError:
print("404 not found")
return []
except TwythonAuthError:
print("Bad Authentication")
return []
oauth_token='2179827990-xnhyW3hxAjRco0Ff50wEMXiXdt6JrWR9qAF6oLA',
oauth_token_secret='y8iAJRPyAslCR8LPDMCnHUfB2uJOJ3lxW1oVBAFa31lqg')
cursor.execute("select id from t_gh_json")
ids = cursor.fetchall()
cursor.execute("create table ghtwdirectextra(user_id text, followerids text, friendids text)")
cursor.execute("create table ghtwdirecttimeline(user_id text, tweets text, tweetids text)")
for i in ids:
k=""
for j in i[0]:
if j != "'":
k = k + j
print k
try:
followers = t.get_followers_ids(user_id = k, count=5000)
#print followers['ids']
friends = t.get_friends_ids(user_id = k, count=5000)
#print friends['ids']
timeline = t.get_user_timeline(user_id = k, count=200, include_rts=1)
for i in timeline:
#print i['text']
#print i['id']
cursor.execute("insert into ghtwdirecttimeline(user_id, tweets, tweetids) values(%s,%s,%s)",(k, i['text'], i['id']))
cursor.execute("insert into ghtwdirectextra(user_id, followerids, friendids) values(%s,%s,%s)",(k, followers['ids'], friends['ids']))
print "inserted"
except TwythonRateLimitError as e:
print "rate limit"
time.sleep(60*15)
continue
except TwythonAuthError as e:
class TwitterHelper(object):
def __init__(self, identity):
self.identity = identity
if not self.identity.tokens:
self.identity.tokens = authorisationhelper.get_tokens(identity.screen_name)
self.twitter = Twython(
self.identity.tokens[0],
self.identity.tokens[1],
self.identity.tokens[2],
self.identity.tokens[3]
)
# self.identity.twid = self.twitter.lookup_user(screen_name=identity.screen_name)[0]["id_str"]
self.mutation = [" ,", " .", " *", " `", " -", " _"]
self.twitter_configuration = self.twitter.get_twitter_configuration()
logger.debug(self.twitter_configuration)
giphyhelper.set_photo_size_limit(self.twitter_configuration["photo_size_limit"])
me = self.twitter.lookup_user(screen_name=self.identity.screen_name)[0]
logger.debug(me)
self.identity.update(me)
self.rates = None
self.update_rate_limits()
@retry(**retry_args)
def send(self, outbox_item):
if type(outbox_item) is OutgoingTweet:
outbox_item.display()
if outbox_item.filePaths and any(outbox_item.filePaths):
for filePath in outbox_item.filePaths:
# ext = os.path.splitext(filePath)
# if ext == "mp4":
# media_id = self._upload_video(filePath)
# else:
media_id = self._upload_media(filePath)
if media_id:
outbox_item.media_ids.append(media_id)
split_tweets = tweet_splitter.split_tweet(outbox_item, self.twitter_configuration)
in_reply_to_id_str = outbox_item.in_reply_to_id_str
for split_tweet in split_tweets:
split_tweet.in_reply_to_id_str = in_reply_to_id_str
response = self.twitter.update_status(**split_tweet.get_tweet_params())
split_tweet.id_str = response["id_str"]
self.identity.conversations.outgoing(split_tweet)
self.identity.statistics.record_outgoing_tweet()
in_reply_to_id_str = split_tweet.id_str
return in_reply_to_id_str
if type(outbox_item) is OutgoingDirectMessage:
if not outbox_item.screen_name and not outbox_item.user_id:
outbox_item.screen_name = self.identity.admin_screen_name
outbox_item.display()
self.twitter.send_direct_message(
text=outbox_item.text,
screen_name=outbox_item.screen_name,
user_id=outbox_item.user_id)
self.identity.statistics.record_outgoing_direct_message()
return None
def quote_tweet(self, inbox_item, text=None, file_paths=None):
reply_as_quote_tweet = inbox_item.is_tweet
reply_as_dm = inbox_item.is_direct_message
if reply_as_quote_tweet:
logger.info("quoting to %s as quote tweet", inbox_item.text)
tweet = OutgoingTweet(
quote=inbox_item,
text=text,
file_paths=file_paths,
)
return self.send(tweet)
if reply_as_dm:
logger.info("replying to %s as DM", inbox_item.text)
dm = OutgoingDirectMessage(
reply_to=inbox_item,
text=text)
return self.send(dm)
return None
def reply_with(self, inbox_item, text=None, as_tweet=False, as_direct_message=False, file_paths=None,
in_reply_to_id_str=None):
reply_as_tweet = as_tweet or not as_direct_message and inbox_item.is_tweet
reply_as_dm = as_direct_message or not as_tweet and inbox_item.is_direct_message
if reply_as_tweet:
logger.info("replying to %s as tweet", inbox_item.text)
tweet = OutgoingTweet(
reply_to=inbox_item,
text=text,
file_paths=file_paths,
in_reply_to_id_str=in_reply_to_id_str)
return self.send(tweet)
if reply_as_dm:
logger.info("replying to %s as DM", inbox_item.text)
dm = OutgoingDirectMessage(
reply_to=inbox_item,
text=text)
return self.send(dm)
return None
def _upload_media(self, file_path):
file = None
try:
file = open(file_path, 'rb')
media = self.twitter.upload_media(media=file)
self.identity.statistics.increment("Media Uploads")
return media["media_id_string"]
finally:
if file:
file.close()
def get_streamer(self, topic=None, topic_name=None, responses=None, filter_level=None):
return Streamer(self.identity, topic, topic_name, responses, filter_level)
def _upload_video(self, file_path):
logger.info('[MyTwitter] uploading ' + file_path)
url = 'https://upload.twitter.com/1.1/media/upload.json'
file_size = os.path.getsize(file_path)
logger.info('[MyTwitter] Init')
init_params = {
"command": "INIT",
"media_type": "video/mp4",
"total_bytes": file_size
}
init_response = self.twitter.post(url, init_params)
logger.info(init_response)
media_id = init_response["media_id_string"]
logger.info('[MyTwitter] media_id ' + media_id)
segment = 0
chunk_size = 4 * 1024 * 1024
for chunk in fsh.bytes_from_file(file_path, chunk_size):
logger.info('[MyTwitter] Append ' + str(segment))
append_params = {
'command': 'APPEND',
'media_id': media_id,
'segment_index': segment
}
append_response = self.twitter.post(url, append_params, {'media': chunk})
logger.info(append_response)
segment += 1
logger.info('[MyTwitter] Finalize')
finalize_params = {
"command": "FINALIZE",
"media_id": media_id,
}
self.twitter.post(url, finalize_params)
return media_id
def get_trending_topics_for(self, woeids):
trending_topics = []
for woeid in woeids:
trends = self.twitter.get_place_trends(id=woeid)[0].get('trends', [])
for trend in trends:
trending_topics.append(trend['name'])
return trending_topics
def get_saved_searches(self):
saved_searches = []
searches = self.twitter.get_saved_searches()
for srch in searches:
saved_searches.append(srch['name'])
return saved_searches
def delete_saved_searches(self):
searches = self.twitter.get_saved_searches()
for srch in searches:
self.twitter.destroy_saved_search(id=srch["id_str"])
def search(self, text, result_type="popular"):
query = quote_plus(text)
return self.twitter.search(q=query, result_type=result_type)["statuses"]
@retry(**retry_args)
def favourite(self, id_str):
try:
self.twitter.create_favorite(id=id_str)
self.identity.statistics.record_favourite()
except TwythonError as ex:
if "You have already favourited this tweet" in str(ex):
logger.warning(ex)
else:
raise
@retry(**retry_args)
def retweet(self, id_str):
try:
self.twitter.retweet(id=id_str)
self.identity.statistics.record_retweet()
except TwythonError as ex:
if "You have already retweeted this tweet" in str(ex):
logger.warning(ex)
else:
raise
@retry(**retry_args)
def add_user_to_list(self, list_id, user_id, screen_name):
self.twitter.create_list_members(list_id=list_id, user_id=user_id, screen_name=screen_name)
@retry(**retry_args)
def block_user(self, user_id, user_screen_name=None):
self.twitter.create_block(user_id=user_id, screen_name=user_screen_name)
@retry(**retry_args)
def get_user_timeline(self, **kwargs):
return self._rate_limit("/statuses/user_timeline", self.twitter.get_user_timeline, **kwargs)
def unblock_user(self, user):
self.twitter.destroy_block(user_id=user.id, screen_name=user.screen_name)
@retry(**retry_args)
def unblock_users(self):
user_ids = self.twitter.list_block_ids(stringify_ids=True)
for user_id in user_ids["ids"]:
self.twitter.destroy_block(user_id=user_id)
@retry(**retry_args)
def show_owned_lists(self):
return self.twitter.show_owned_lists()["lists"]
@retry(**retry_args)
def get_list_members(self, list_id):
return self.twitter.get_list_members(list_id=list_id, count=5000, include_entities=False)
@blocked
@retry(**retry_args)
def create_list(self, **kwargs):
return self.twitter.create_list(**kwargs)
@retry(**retry_args)
def follow(self, user_id, screen_name):
logger.info("following user id {} @{}".format(user_id, screen_name))
self.twitter.create_friendship(user_id=user_id, screen_name=screen_name)
self.identity.statistics.increment("Follows")
@retry(**retry_args)
def unfollow(self, user_id, screen_name):
logger.info("unfollowing user id {} @{}".format(user_id, screen_name))
self.twitter.destroy_friendship(user_id=user_id, screen_name=screen_name)
self.identity.statistics.increment("Unfollows")
@retry(**retry_args)
def block(self, user_id, screen_name):
logger.info("blocking user id {} @{}".format(user_id, screen_name))
self.twitter.create_block(user_id=user_id, screen_name=screen_name)
self.identity.statistics.increment("Blocks")
@retry(**retry_args)
def report(self, user_id, screen_name):
logger.info("reporting user id {} @{}".format(user_id, screen_name))
self.twitter.report_spam(user_id=user_id, screen_name=screen_name)
self.identity.statistics.increment("Reports")
@retry(**retry_args)
def lookup_user(self, user_id):
return self._rate_limit("/users/lookup", self.twitter.lookup_user, user_id=user_id)
def sing_song(self, song, target=None, inbox_item=None, text=None, hashtag=None):
if not text:
text = random.choice(["All together now!", "Sing along!"])
text += ' ' + song["video"]
if hashtag:
text += ' ' + hashtag
in_reply_to_id_str = self._send_to(
inbox_item=inbox_item,
text=text,
target=target,
in_reply_to_id_str=None)
time.sleep(5)
lastlyrics = set([])
for lyric in song["lyrics"]:
lyric = lyric.strip()
if lyric:
if "<<screen_name>>" in lyric:
lyric = lyric.replace("<<screen_name>>", "@" + target)
if hashtag:
lyric += " " + hashtag
while lyric in lastlyrics:
lyric += random.choice(self.mutation)
lastlyrics.add(lyric)
self.identity.statistics.record_outgoing_song_lyric()
in_reply_to_id_str = self._send_to(
inbox_item,
lyric,
target,
in_reply_to_id_str)
time.sleep(5)
def _send_to(self, inbox_item, text, target, in_reply_to_id_str):
if inbox_item:
return self.reply_with(
inbox_item=inbox_item,
text=text,
in_reply_to_id_str=in_reply_to_id_str)
else:
status = ""
if target:
# noinspection PyUnresolvedReferences
if isinstance(target, basestring):
status = ".@" + target
elif isinstance(target, User.User):
status = ".@" + target.screen_name
status += " " + text
tweet = OutgoingTweet(
text=status,
in_reply_to_id_str=in_reply_to_id_str)
return self.send(tweet)
@retry(**retry_args)
def get_list_subscriptions(self):
return self.twitter.get_list_subscriptions()
@retry(**retry_args)
def subscribe_to_list(self, list_id):
return self.twitter.subscribe_to_list(list_id=list_id)
@retry(**retry_args)
def geocode(self, location):
result = self.twitter.search_geo(
query=location.full_name,
max_results=5,
lat=location.latitude,
long=location.longitude)
logger.info(result)
if result["result"]["places"]:
# for place in result["result"]["places"]:
# logger.info(place["full_name"])
place = result["result"]["places"][0]
location.place_id_twitter = place["id"]
return location
else:
return None
@retry(**retry_args)
def reverse_geocode(self, location):
result = self.twitter.reverse_geocode(
max_results=5,
lat=location.latitude,
long=location.longitude)
logger.info(result)
if result["result"]["places"]:
# for place in result["result"]["places"]:
# logger.info(place["full_name"])
place = result["result"]["places"][0]
location.place_id_twitter = place["id"]
return location
else:
return None
@retry(**retry_args)
def update_profile_image(self, file_path):
if file_path:
logger.info("updating profile image %s" % file_path)
with open(file_path, 'rb') as file:
self.twitter.update_profile_image(image=file)
@retry(**retry_args)
def update_profile_banner_image(self, file_path):
if file_path:
logger.info("updating banner image %s" % file_path)
with open(file_path, 'rb') as file:
try:
self.twitter.update_profile_banner_image(banner=file)
except TwythonError as ex:
if "Response was not valid JSON" in str(ex):
# twython issue i think
logger.warning(ex)
else:
raise
@retry(**retry_args)
def update_profile(self, **kwargs):
return self.twitter.update_profile(**kwargs)
@retry(**retry_args)
def get_list_statuses(self, **kwargs):
return self.twitter.get_list_statuses(**kwargs)
@retry(**retry_args)
def get_user_suggestions_by_slug(self, **kwargs):
return self.twitter.get_user_suggestions_by_slug(**kwargs)
@retry(**retry_args)
def get_user_suggestions(self, **kwargs):
return self.twitter.get_user_suggestions(**kwargs)
@retry(**retry_args)
def lookup_status(self, **kwargs):
return self.twitter.lookup_status(**kwargs)
@retry(**retry_args)
def get_followers(self, **kwargs):
kwargs["stringify_ids"] = True
followers = set()
cursor = -1
while cursor != "0":
kwargs["cursor"] = cursor
logger.info("getting followers")
response = self.twitter.get_followers_ids(**kwargs)
cursor = response["next_cursor_str"]
followers = followers.union(set(response["ids"]))
return followers
@retry(**retry_args)
def get_following(self, **kwargs):
kwargs["stringify_ids"] = True
following = set()
cursor = -1
while cursor != "0":
kwargs["cursor"] = cursor
logger.info("getting following")
response = self.twitter.get_friends_ids(**kwargs)
cursor = response["next_cursor_str"]
following = following.union(set(response["ids"]))
return following
@retry(**retry_args)
def update_rate_limits(self):
data = self.twitter.get_application_rate_limit_status()
self.rates = RateLimits(data)
logger.info("Updated rate limits for {}: {}".format(self.identity.screen_name, self.rates.display()))
def _rate_limit(self, limit_name, func, *args, **kwargs):
if self.rates.can(limit_name):
try:
return func(*args, **kwargs)
except Exception as ex:
logger.warning(ex)
return None
else:
logger.warning("{} limit exceeded".format(limit_name))
return None
@retry(**retry_args)
def get_statuses(self, id_strs):
id_strs_csv = ",".join(id_strs)
return self.twitter.lookup_status(id=id_strs_csv)
@retry(**retry_args)
def get_status(self, id_str):
return self.twitter.show_status(id=id_str)
class TwythonAPITestCase(unittest.TestCase):
def setUp(self):
client_args = {
'headers': {
'User-Agent': '__twython__ Test'
},
'allow_redirects': False
}
oauth2_client_args = {
'headers': {} # This is so we can hit coverage that Twython sets User-Agent for us if none is supplied
}
self.api = Twython(app_key, app_secret,
oauth_token, oauth_token_secret,
client_args=client_args)
self.oauth2_api = Twython(app_key, access_token=access_token,
client_args=oauth2_client_args)
def test_construct_api_url(self):
"""Test constructing a Twitter API url works as we expect"""
url = 'https://api.twitter.com/1.1/search/tweets.json'
constructed_url = self.api.construct_api_url(url, q='#twitter')
self.assertEqual(constructed_url, 'https://api.twitter.com/1.1/search/tweets.json?q=%23twitter')
def test_get(self):
"""Test Twython generic GET request works"""
self.api.get('account/verify_credentials')
def test_post(self):
"""Test Twython generic POST request works, with a full url and
with just an endpoint"""
update_url = 'https://api.twitter.com/1.1/statuses/update.json'
status = self.api.post(update_url, params={'status': 'I love Twython!'})
self.api.post('statuses/destroy/%s' % status['id_str'])
def test_get_lastfunction_header(self):
"""Test getting last specific header of the last API call works"""
self.api.get('statuses/home_timeline')
self.api.get_lastfunction_header('x-rate-limit-remaining')
def test_get_lastfunction_header_not_present(self):
"""Test getting specific header that does not exist from the last call returns None"""
self.api.get('statuses/home_timeline')
header = self.api.get_lastfunction_header('does-not-exist')
self.assertEqual(header, None)
def test_get_lastfunction_header_no_last_api_call(self):
"""Test attempting to get a header when no API call was made raises a TwythonError"""
self.assertRaises(TwythonError, self.api.get_lastfunction_header,
'no-api-call-was-made')
def test_search_gen(self):
"""Test looping through the generator results works, at least once that is"""
search = self.api.search_gen('twitter', count=1)
counter = 0
while counter < 2:
counter += 1
result = next(search)
new_id_str = int(result['id_str'])
if counter == 1:
prev_id_str = new_id_str
time.sleep(1) # Give time for another tweet to come into search
if counter == 2:
self.assertTrue(new_id_str > prev_id_str)
def test_encode(self):
"""Test encoding UTF-8 works"""
self.api.encode('Twython is awesome!')
def test_html_for_tweet(self):
"""Test HTML for Tweet returns what we want"""
tweet_text = self.api.html_for_tweet(test_tweet_object)
self.assertEqual(test_tweet_html, tweet_text)
def test_html_for_tweet_expanded_url(self):
"""Test using expanded url in HTML for Tweet displays full urls"""
tweet_text = self.api.html_for_tweet(test_tweet_object,
use_expanded_url=True)
# Make sure full url is in HTML
self.assertTrue('http://google.com' in tweet_text)
def test_html_for_tweet_short_url(self):
"""Test using expanded url in HTML for Tweet displays full urls"""
tweet_text = self.api.html_for_tweet(test_tweet_object, False)
# Make sure HTML doesn't contain the display OR exapanded url
self.assertTrue(not 'http://google.com' in tweet_text)
self.assertTrue(not 'google.com' in tweet_text)
def test_raise_error_on_bad_ssl_cert(self):
"""Test TwythonError is raised by a RequestException when an actual HTTP happens"""
self.assertRaises(TwythonError, self.api.get, 'https://example.com')
# Timelines
def test_get_mentions_timeline(self):
"""Test returning mentions timeline for authenticated user succeeds"""
self.api.get_mentions_timeline()
def test_get_user_timeline(self):
"""Test returning timeline for authenticated user and random user
succeeds"""
self.api.get_user_timeline() # Authenticated User Timeline
self.api.get_user_timeline(screen_name='twitter') # Random User Timeline
def test_get_protected_user_timeline_following(self):
"""Test returning a protected user timeline who you are following
succeeds"""
self.api.get_user_timeline(screen_name=protected_twitter_1)
def test_get_protected_user_timeline_not_following(self):
"""Test returning a protected user timeline who you are not following
fails and raise a TwythonAuthError"""
self.assertRaises(TwythonAuthError, self.api.get_user_timeline,
screen_name=protected_twitter_2)
def test_retweeted_of_me(self):
"""Test that getting recent tweets by authenticated user that have
been retweeted by others succeeds"""
self.api.retweeted_of_me()
def test_get_home_timeline(self):
"""Test returning home timeline for authenticated user succeeds"""
self.api.get_home_timeline()
# Tweets
def test_get_retweets(self):
"""Test getting retweets of a specific tweet succeeds"""
self.api.get_retweets(id=test_tweet_id)
def test_show_status(self):
"""Test returning a single status details succeeds"""
self.api.show_status(id=test_tweet_id)
def test_update_and_destroy_status(self):
"""Test updating and deleting a status succeeds"""
status = self.api.update_status(status='Test post just to get deleted :(')
self.api.destroy_status(id=status['id_str'])
def test_get_oembed_tweet(self):
"""Test getting info to embed tweet on Third Party site succeeds"""
self.api.get_oembed_tweet(id='99530515043983360')
def test_get_retweeters_ids(self):
"""Test getting ids for people who retweeted a tweet succeeds"""
self.api.get_retweeters_ids(id='99530515043983360')
# Search
def test_search(self):
"""Test searching tweets succeeds"""
self.api.search(q='twitter')
# Direct Messages
def test_get_direct_messages(self):
"""Test getting the authenticated users direct messages succeeds"""
self.api.get_direct_messages()
def test_get_sent_messages(self):
"""Test getting the authenticated users direct messages they've
sent succeeds"""
self.api.get_sent_messages()
def test_send_get_and_destroy_direct_message(self):
"""Test sending, getting, then destory a direct message succeeds"""
message = self.api.send_direct_message(screen_name=protected_twitter_1,
text='Hey d00d! %s' % int(time.time()))
self.api.get_direct_message(id=message['id_str'])
self.api.destroy_direct_message(id=message['id_str'])
def test_send_direct_message_to_non_follower(self):
"""Test sending a direct message to someone who doesn't follow you
fails"""
self.assertRaises(TwythonError, self.api.send_direct_message,
screen_name=protected_twitter_2, text='Yo, man!')
# Friends & Followers
def test_get_user_ids_of_blocked_retweets(self):
"""Test that collection of user_ids that the authenticated user does
not want to receive retweets from succeeds"""
self.api.get_user_ids_of_blocked_retweets(stringify_ids=True)
def test_get_friends_ids(self):
"""Test returning ids of users the authenticated user and then a random
user is following succeeds"""
self.api.get_friends_ids()
self.api.get_friends_ids(screen_name='twitter')
def test_get_followers_ids(self):
"""Test returning ids of users the authenticated user and then a random
user are followed by succeeds"""
self.api.get_followers_ids()
self.api.get_followers_ids(screen_name='twitter')
def test_lookup_friendships(self):
"""Test returning relationships of the authenticating user to the
comma-separated list of up to 100 screen_names or user_ids provided
succeeds"""
self.api.lookup_friendships(screen_name='twitter,ryanmcgrath')
def test_get_incoming_friendship_ids(self):
"""Test returning incoming friendship ids succeeds"""
self.api.get_incoming_friendship_ids()
def test_get_outgoing_friendship_ids(self):
"""Test returning outgoing friendship ids succeeds"""
self.api.get_outgoing_friendship_ids()
def test_create_friendship(self):
"""Test creating a friendship succeeds"""
self.api.create_friendship(screen_name='justinbieber')
def test_destroy_friendship(self):
"""Test destroying a friendship succeeds"""
self.api.destroy_friendship(screen_name='justinbieber')
def test_update_friendship(self):
"""Test updating friendships succeeds"""
self.api.update_friendship(screen_name=protected_twitter_1,
retweets='true')
self.api.update_friendship(screen_name=protected_twitter_1,
retweets=False)
def test_show_friendships(self):
"""Test showing specific friendship succeeds"""
self.api.show_friendship(target_screen_name=protected_twitter_1)
def test_get_friends_list(self):
"""Test getting list of users authenticated user then random user is
following succeeds"""
self.api.get_friends_list()
self.api.get_friends_list(screen_name='twitter')
def test_get_followers_list(self):
"""Test getting list of users authenticated user then random user are
followed by succeeds"""
self.api.get_followers_list()
self.api.get_followers_list(screen_name='twitter')
# Users
def test_get_account_settings(self):
"""Test getting the authenticated user account settings succeeds"""
self.api.get_account_settings()
def test_verify_credentials(self):
"""Test representation of the authenticated user call succeeds"""
self.api.verify_credentials()
def test_update_account_settings(self):
"""Test updating a user account settings succeeds"""
self.api.update_account_settings(lang='en')
def test_update_delivery_service(self):
"""Test updating delivery settings fails because we don't have
a mobile number on the account"""
self.assertRaises(TwythonError, self.api.update_delivery_service,
device='none')
def test_update_profile(self):
"""Test updating profile succeeds"""
self.api.update_profile(include_entities='true')
def test_update_profile_colors(self):
"""Test updating profile colors succeeds"""
self.api.update_profile_colors(profile_background_color='3D3D3D')
def test_list_blocks(self):
"""Test listing users who are blocked by the authenticated user
succeeds"""
self.api.list_blocks()
def test_list_block_ids(self):
"""Test listing user ids who are blocked by the authenticated user
succeeds"""
self.api.list_block_ids()
def test_create_block(self):
"""Test blocking a user succeeds"""
self.api.create_block(screen_name='justinbieber')
def test_destroy_block(self):
"""Test unblocking a user succeeds"""
self.api.destroy_block(screen_name='justinbieber')
def test_lookup_user(self):
"""Test listing a number of user objects succeeds"""
self.api.lookup_user(screen_name='twitter,justinbieber')
def test_show_user(self):
"""Test showing one user works"""
self.api.show_user(screen_name='twitter')
def test_search_users(self):
"""Test that searching for users succeeds"""
self.api.search_users(q='Twitter API')
def test_get_contributees(self):
"""Test returning list of accounts the specified user can
contribute to succeeds"""
self.api.get_contributees(screen_name='TechCrunch')
def test_get_contributors(self):
"""Test returning list of accounts that contribute to the
authenticated user fails because we are not a Contributor account"""
self.assertRaises(TwythonError, self.api.get_contributors,
screen_name=screen_name)
def test_remove_profile_banner(self):
"""Test removing profile banner succeeds"""
self.api.remove_profile_banner()
def test_get_profile_banner_sizes(self):
"""Test getting list of profile banner sizes fails because
we have not uploaded a profile banner"""
self.assertRaises(TwythonError, self.api.get_profile_banner_sizes)
# Suggested Users
def test_get_user_suggestions_by_slug(self):
"""Test getting user suggestions by slug succeeds"""
self.api.get_user_suggestions_by_slug(slug='twitter')
def test_get_user_suggestions(self):
"""Test getting user suggestions succeeds"""
self.api.get_user_suggestions()
def test_get_user_suggestions_statuses_by_slug(self):
"""Test getting status of suggested users succeeds"""
self.api.get_user_suggestions_statuses_by_slug(slug='funny')
# Favorites
def test_get_favorites(self):
"""Test getting list of favorites for the authenticated
user succeeds"""
self.api.get_favorites()
def test_create_and_destroy_favorite(self):
"""Test creating and destroying a favorite on a tweet succeeds"""
self.api.create_favorite(id=test_tweet_id)
self.api.destroy_favorite(id=test_tweet_id)
# Lists
def test_show_lists(self):
"""Test show lists for specified user"""
self.api.show_lists(screen_name='twitter')
def test_get_list_statuses(self):
"""Test timeline of tweets authored by members of the
specified list succeeds"""
self.api.get_list_statuses(list_id=test_list_id)
def test_create_update_destroy_list_add_remove_list_members(self):
"""Test create a list, adding and removing members then
deleting the list succeeds"""
the_list = self.api.create_list(name='Stuff')
list_id = the_list['id_str']
self.api.update_list(list_id=list_id, name='Stuff Renamed')
screen_names = ['johncena', 'xbox']
# Multi add/delete members
self.api.create_list_members(list_id=list_id,
screen_name=screen_names)
self.api.delete_list_members(list_id=list_id,
screen_name=screen_names)
# Single add/delete member
self.api.add_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list(list_id=list_id)
def test_get_list_subscribers(self):
"""Test list of subscribers of a specific list succeeds"""
self.api.get_list_subscribers(list_id=test_list_id)
def test_subscribe_is_subbed_and_unsubscribe_to_list(self):
"""Test subscribing, is a list sub and unsubbing to list succeeds"""
self.api.subscribe_to_list(list_id=test_list_id)
# Returns 404 if user is not a subscriber
self.api.is_list_subscriber(list_id=test_list_id,
screen_name=screen_name)
self.api.unsubscribe_from_list(list_id=test_list_id)
def test_is_list_member(self):
"""Test returning if specified user is member of a list succeeds"""
# Returns 404 if not list member
self.api.is_list_member(list_id=test_list_id, screen_name='jack')
def test_get_list_members(self):
"""Test listing members of the specified list succeeds"""
self.api.get_list_members(list_id=test_list_id)
def test_get_specific_list(self):
"""Test getting specific list succeeds"""
self.api.get_specific_list(list_id=test_list_id)
def test_get_list_subscriptions(self):
"""Test collection of the lists the specified user is
subscribed to succeeds"""
self.api.get_list_subscriptions(screen_name='twitter')
def test_show_owned_lists(self):
"""Test collection of lists the specified user owns succeeds"""
self.api.show_owned_lists(screen_name='twitter')
# Saved Searches
def test_get_saved_searches(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
self.api.get_saved_searches()
def test_create_get_destroy_saved_search(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
saved_search = self.api.create_saved_search(query='#Twitter')
saved_search_id = saved_search['id_str']
self.api.show_saved_search(id=saved_search_id)
self.api.destroy_saved_search(id=saved_search_id)
# Places & Geo
def test_get_geo_info(self):
"""Test getting info about a geo location succeeds"""
self.api.get_geo_info(place_id='df51dec6f4ee2b2c')
def test_reverse_geo_code(self):
"""Test reversing geocode succeeds"""
self.api.reverse_geocode(lat='37.76893497', long='-122.42284884')
def test_search_geo(self):
"""Test search for places that can be attached
to a statuses/update succeeds"""
self.api.search_geo(query='Toronto')
def test_get_similar_places(self):
"""Test locates places near the given coordinates which
are similar in name succeeds"""
self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ')
# Trends
def test_get_place_trends(self):
"""Test getting the top 10 trending topics for a specific
WOEID succeeds"""
self.api.get_place_trends(id=1)
def test_get_available_trends(self):
"""Test returning locations that Twitter has trending
topic information for succeeds"""
self.api.get_available_trends()
def test_get_closest_trends(self):
"""Test getting the locations that Twitter has trending topic
information for, closest to a specified location succeeds"""
self.api.get_closest_trends(lat='37', long='-122')
# Help
def test_get_application_rate_limit_status(self):
"""Test getting application rate limit status succeeds"""
self.oauth2_api.get_application_rate_limit_status()
def get_followers():
distinct = getScreenname.getDistinctScreenames("all_tweets_dict_fully_checked")
index=0
num_checked = 0
rate_ex = 0
for sname in distinct:
file = open("followers.txt", "r+")
already_checked = False
for line in file:
words = line.split('\t')
if words[0] == sname:
already_checked = True
if index >= 7:
break
APP_KEY = APP_KEYS[index]
APP_SECRET = APP_SECRETS[index]
OAUTH_TOKEN = OAUTH_TOKENS[index]
OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index]
if not already_checked:
try:
twitter = Twython (APP_KEY, APP_SECRET)
auth = twitter.get_authentication_tokens()
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
followers = twitter.get_followers_ids(screen_name=sname)
print sname
file.write (sname)
file.write ("\t")
for ID in followers['ids']:
file.write(str(ID) + " ")
file.write("\n")
num_checked += 1
except Exception as e:
if "429 (Too Many Requests)" in str(e):
print "\nchanging apps!\n"
index += 1
if "401 (Unauthorized)" in str(e):
print "401 error"
file.write(sname)
file.write("\t")
file.write("Unauthorized")
file.write("\n")
if "404 (Not Found)" in str(e):
print "404 error"
file.write(sname)
file.write("\t")
file.write("NotFound")
file.write("\n")
else:
print e
rate_ex += 1
file.close()
if (num_checked != 0 or rate_ex !=0):
print "sleepy time"
time.sleep(860)
get_followers()
def updateFollowerFollowings(dbpath, user_table = "users"):
distinct = get_D_UsrID(dbpath,sn_table )
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "select Usr_ID from %s where Following is null or Followers is null"
cursor.execute(query)
distinct = cursor.fetchall()
follower_cursor = -1
friend_cursor = -1
print "Starting @ %d"%(ind)
print datetime.datetime.now()
while ind < len(distinct):
#for i in distinct:
i = distinct[ind][0] # Usr_ID
APP_KEY = APP_KEYS[index]
APP_SECRET = APP_SECRETS[index]
OAUTH_TOKEN = OAUTH_TOKENS[index]
OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index]
try:
#twitter = Twython (APP_KEY, APP_SECRET)
#auth = twitter.get_authentication_tokens()
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
friends = ""
while (friend_cursor):
following = twitter.get_friends_ids(id = i,cursor= friend_cursor)
for ID in following['ids']:
friends += str(ID) + " "
friend_cursor = following["next_cursor"]
num_checked += 1
friends = friends[:-1]
if len(friends) == 0:
friends = "null"
follow = ""
while (follower_cursor):
followers = twitter.get_followers_ids(id = i,cursor= follower_cursor)
for ID in followers['ids']:
follow += str(ID) + " "
follower_cursor = followers["next_cursor"]
num_checked += 1
follow= follow[:-1]
if len(follow) == 0:
follow = "null"
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "UPDATE %s SET Following = '%s', Followers = '%s', NumFollowing = '%d', NumFollowers = '%d' where Usr_ID = '%s'"% (sn_table, friends,follow, len(friends.split(" ")), len(follow.split(" ")), i)
cursor.execute(query)
#print query
conn.close()
print "____________________ %dth following and friends and their corresponding counts updated ________ %d" % (ind, i)
follower_cursor = -1
friend_cursor = -1
ind+=1
except Exception as e:
print e
if "429 (Too Many Requests)" in str(e):
index += 1
if index == len(APP_KEYS):
index = 0
print "sleepy time; pausing for 15 minutes for APP Keys to renew"
print datetime.datetime.now()
#time.sleep(910)
print "App Rate Limits has been exceeded- we will wait for next runtime before running rest of followers and followings"
return
elif "401 (Unauthorized)" in str(e):
print "401 Unauthorized Error nulling %d________________Usr_ID:%d"%(ind,i)
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "UPDATE %s SET Following = 'null', Followers = 'null' where Usr_ID = '%s'"% (sn_table, i)
cursor.execute(query)
conn.commit()
conn.close()
ind+=1
elif "404 (Not Found)" in str(e):
print "404 Not Found Error nulling %d________________Usr_ID:%d"%(ind,i)
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "UPDATE %s SET Following = 'null', Followers = 'null' where Usr_ID = '%s'"% (sn_table, i)
cursor.execute(query)
conn.commit()
conn.close()
ind+=1
else:
print e
import string
import random
tweetStr = "Hello World2"
# your twitter consumer and access information goes here
# note: these are garbage strings and won't work
apiKey = ''
apiSecret = ''
accessToken = ''
accessTokenSecret = ''
twitter = Twython(apiKey,apiSecret,accessToken,accessTokenSecret)
#GET FOLLOWERS
followers = (twitter.get_followers_ids(screen_name = "@puppersplz")['ids'])
print followers
#pick a random ID from all followers
luckyid = random.choice(followers)
#lookup the handle of the lucky ID
output = twitter.lookup_user(user_id=luckyid)
# Loop through the results (Twitter screen names)
for user in output:
print user['screen_name']
username_list= (user['screen_name'])
APP_KEY = 'aylam3pBvHkhOUmycOWw'
APP_SECRET = 'uXAuZwGX8FUno0P54gdIlGnkijhkY56lVFxRwgjgI'
twitter = Twython(APP_KEY, APP_SECRET, oauth_version=2)
ACCESS_TOKEN = twitter.obtain_access_token()
twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN)
#Set root account:
root = "BeingSalmanKhan"
user_timeline = twitter.get_user_timeline(screen_name=root)
#Print tweets from @[root]:
for tweet in user_timeline:
print(tweet['text'])
follower_ids = twitter.get_followers_ids(screen_name=root, count=5000)
f = open('tweets.txt', 'w')
user_count = 0
tweet_count = 0
private_count = 0
start_time = datetime.now()
#Collect tweets for each follower of [root]:
for follower in follower_ids['ids']:
#try:
# print twitter.get_geo_info(user_id=follower)
#except:
# print 'Geo info private'
try:
class TwythonEndpointsTestCase(unittest.TestCase):
def setUp(self):
client_args = {
'headers': {
'User-Agent': '__twython__ Test'
},
'allow_redirects': False
}
# This is so we can hit coverage that Twython sets
# User-Agent for us if none is supplied
oauth2_client_args = {
'headers': {}
}
self.api = Twython(app_key, app_secret,
oauth_token, oauth_token_secret,
client_args=client_args)
self.oauth2_api = Twython(app_key, access_token=access_token,
client_args=oauth2_client_args)
# Timelines
@unittest.skip('skipping non-updated test')
def test_get_mentions_timeline(self):
"""Test returning mentions timeline for authenticated user succeeds"""
self.api.get_mentions_timeline()
@unittest.skip('skipping non-updated test')
def test_get_user_timeline(self):
"""Test returning timeline for authenticated user and random user
succeeds"""
self.api.get_user_timeline() # Authenticated User Timeline
self.api.get_user_timeline(screen_name='twitter')
# Random User Timeline
@unittest.skip('skipping non-updated test')
def test_get_protected_user_timeline_following(self):
"""Test returning a protected user timeline who you are following
succeeds"""
self.api.get_user_timeline(screen_name=protected_twitter_1)
@unittest.skip('skipping non-updated test')
def test_get_protected_user_timeline_not_following(self):
"""Test returning a protected user timeline who you are not following
fails and raise a TwythonAuthError"""
self.assertRaises(TwythonAuthError, self.api.get_user_timeline,
screen_name=protected_twitter_2)
@unittest.skip('skipping non-updated test')
def test_retweeted_of_me(self):
"""Test that getting recent tweets by authenticated user that have
been retweeted by others succeeds"""
self.api.retweeted_of_me()
@unittest.skip('skipping non-updated test')
def test_get_home_timeline(self):
"""Test returning home timeline for authenticated user succeeds"""
self.api.get_home_timeline()
# Tweets
@unittest.skip('skipping non-updated test')
def test_get_retweets(self):
"""Test getting retweets of a specific tweet succeeds"""
self.api.get_retweets(id=test_tweet_id)
@unittest.skip('skipping non-updated test')
def test_show_status(self):
"""Test returning a single status details succeeds"""
self.api.show_status(id=test_tweet_id)
@unittest.skip('skipping non-updated test')
def test_update_and_destroy_status(self):
"""Test updating and deleting a status succeeds"""
status = self.api.update_status(status='Test post just to get \
deleted :( %s' % int(time.time()))
self.api.destroy_status(id=status['id_str'])
@unittest.skip('skipping non-updated test')
def test_get_oembed_tweet(self):
"""Test getting info to embed tweet on Third Party site succeeds"""
self.api.get_oembed_tweet(id='99530515043983360')
@unittest.skip('skipping non-updated test')
def test_get_retweeters_ids(self):
"""Test getting ids for people who retweeted a tweet succeeds"""
self.api.get_retweeters_ids(id='99530515043983360')
# Search
@unittest.skip('skipping non-updated test')
def test_search(self):
"""Test searching tweets succeeds"""
self.api.search(q='twitter')
# Direct Messages
@unittest.skip('skipping non-updated test')
def test_get_direct_messages(self):
"""Test getting the authenticated users direct messages succeeds"""
self.api.get_direct_messages()
@unittest.skip('skipping non-updated test')
def test_get_sent_messages(self):
"""Test getting the authenticated users direct messages they've
sent succeeds"""
self.api.get_sent_messages()
@unittest.skip('skipping non-updated test')
def test_send_get_and_destroy_direct_message(self):
"""Test sending, getting, then destory a direct message succeeds"""
message = self.api.send_direct_message(screen_name=protected_twitter_1,
text='Hey d00d! %s\
' % int(time.time()))
self.api.get_direct_message(id=message['id_str'])
self.api.destroy_direct_message(id=message['id_str'])
@unittest.skip('skipping non-updated test')
def test_send_direct_message_to_non_follower(self):
"""Test sending a direct message to someone who doesn't follow you
fails"""
self.assertRaises(TwythonError, self.api.send_direct_message,
screen_name=protected_twitter_2, text='Yo, man! \
%s' % int(time.time()))
# Friends & Followers
@unittest.skip('skipping non-updated test')
def test_get_user_ids_of_blocked_retweets(self):
"""Test that collection of user_ids that the authenticated user does
not want to receive retweets from succeeds"""
self.api.get_user_ids_of_blocked_retweets(stringify_ids=True)
@unittest.skip('skipping non-updated test')
def test_get_friends_ids(self):
"""Test returning ids of users the authenticated user and then a random
user is following succeeds"""
self.api.get_friends_ids()
self.api.get_friends_ids(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_get_followers_ids(self):
"""Test returning ids of users the authenticated user and then a random
user are followed by succeeds"""
self.api.get_followers_ids()
self.api.get_followers_ids(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_lookup_friendships(self):
"""Test returning relationships of the authenticating user to the
comma-separated list of up to 100 screen_names or user_ids provided
succeeds"""
self.api.lookup_friendships(screen_name='twitter,ryanmcgrath')
@unittest.skip('skipping non-updated test')
def test_get_incoming_friendship_ids(self):
"""Test returning incoming friendship ids succeeds"""
self.api.get_incoming_friendship_ids()
@unittest.skip('skipping non-updated test')
def test_get_outgoing_friendship_ids(self):
"""Test returning outgoing friendship ids succeeds"""
self.api.get_outgoing_friendship_ids()
@unittest.skip('skipping non-updated test')
def test_create_friendship(self):
"""Test creating a friendship succeeds"""
self.api.create_friendship(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_destroy_friendship(self):
"""Test destroying a friendship succeeds"""
self.api.destroy_friendship(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_update_friendship(self):
"""Test updating friendships succeeds"""
self.api.update_friendship(screen_name=protected_twitter_1,
retweets='true')
self.api.update_friendship(screen_name=protected_twitter_1,
retweets=False)
@unittest.skip('skipping non-updated test')
def test_show_friendships(self):
"""Test showing specific friendship succeeds"""
self.api.show_friendship(target_screen_name=protected_twitter_1)
@unittest.skip('skipping non-updated test')
def test_get_friends_list(self):
"""Test getting list of users authenticated user then random user is
following succeeds"""
self.api.get_friends_list()
self.api.get_friends_list(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_get_followers_list(self):
"""Test getting list of users authenticated user then random user are
followed by succeeds"""
self.api.get_followers_list()
self.api.get_followers_list(screen_name='twitter')
# Users
@unittest.skip('skipping non-updated test')
def test_get_account_settings(self):
"""Test getting the authenticated user account settings succeeds"""
self.api.get_account_settings()
@unittest.skip('skipping non-updated test')
def test_verify_credentials(self):
"""Test representation of the authenticated user call succeeds"""
self.api.verify_credentials()
@unittest.skip('skipping non-updated test')
def test_update_account_settings(self):
"""Test updating a user account settings succeeds"""
self.api.update_account_settings(lang='en')
@unittest.skip('skipping non-updated test')
def test_update_delivery_service(self):
"""Test updating delivery settings fails because we don't have
a mobile number on the account"""
self.assertRaises(TwythonError, self.api.update_delivery_service,
device='none')
@unittest.skip('skipping non-updated test')
def test_update_profile(self):
"""Test updating profile succeeds"""
self.api.update_profile(include_entities='true')
@unittest.skip('skipping non-updated test')
def test_update_profile_colors(self):
"""Test updating profile colors succeeds"""
self.api.update_profile_colors(profile_background_color='3D3D3D')
@unittest.skip('skipping non-updated test')
def test_list_blocks(self):
"""Test listing users who are blocked by the authenticated user
succeeds"""
self.api.list_blocks()
@unittest.skip('skipping non-updated test')
def test_list_block_ids(self):
"""Test listing user ids who are blocked by the authenticated user
succeeds"""
self.api.list_block_ids()
@unittest.skip('skipping non-updated test')
def test_create_block(self):
"""Test blocking a user succeeds"""
self.api.create_block(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_destroy_block(self):
"""Test unblocking a user succeeds"""
self.api.destroy_block(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_lookup_user(self):
"""Test listing a number of user objects succeeds"""
self.api.lookup_user(screen_name='twitter,justinbieber')
@unittest.skip('skipping non-updated test')
def test_show_user(self):
"""Test showing one user works"""
self.api.show_user(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_search_users(self):
"""Test that searching for users succeeds"""
self.api.search_users(q='Twitter API')
@unittest.skip('skipping non-updated test')
def test_get_contributees(self):
"""Test returning list of accounts the specified user can
contribute to succeeds"""
self.api.get_contributees(screen_name='TechCrunch')
@unittest.skip('skipping non-updated test')
def test_get_contributors(self):
"""Test returning list of accounts that contribute to the
authenticated user fails because we are not a Contributor account"""
self.assertRaises(TwythonError, self.api.get_contributors,
screen_name=screen_name)
@unittest.skip('skipping non-updated test')
def test_remove_profile_banner(self):
"""Test removing profile banner succeeds"""
self.api.remove_profile_banner()
@unittest.skip('skipping non-updated test')
def test_get_profile_banner_sizes(self):
"""Test getting list of profile banner sizes fails because
we have not uploaded a profile banner"""
self.assertRaises(TwythonError, self.api.get_profile_banner_sizes)
@unittest.skip('skipping non-updated test')
def test_list_mutes(self):
"""Test listing users who are muted by the authenticated user
succeeds"""
self.api.list_mutes()
@unittest.skip('skipping non-updated test')
def test_list_mute_ids(self):
"""Test listing user ids who are muted by the authenticated user
succeeds"""
self.api.list_mute_ids()
@unittest.skip('skipping non-updated test')
def test_create_mute(self):
"""Test muting a user succeeds"""
self.api.create_mute(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_destroy_mute(self):
"""Test muting a user succeeds"""
self.api.destroy_mute(screen_name='justinbieber')
# Suggested Users
@unittest.skip('skipping non-updated test')
def test_get_user_suggestions_by_slug(self):
"""Test getting user suggestions by slug succeeds"""
self.api.get_user_suggestions_by_slug(slug='twitter')
@unittest.skip('skipping non-updated test')
def test_get_user_suggestions(self):
"""Test getting user suggestions succeeds"""
self.api.get_user_suggestions()
@unittest.skip('skipping non-updated test')
def test_get_user_suggestions_statuses_by_slug(self):
"""Test getting status of suggested users succeeds"""
self.api.get_user_suggestions_statuses_by_slug(slug='funny')
# Favorites
@unittest.skip('skipping non-updated test')
def test_get_favorites(self):
"""Test getting list of favorites for the authenticated
user succeeds"""
self.api.get_favorites()
@unittest.skip('skipping non-updated test')
def test_create_and_destroy_favorite(self):
"""Test creating and destroying a favorite on a tweet succeeds"""
self.api.create_favorite(id=test_tweet_id)
self.api.destroy_favorite(id=test_tweet_id)
# Lists
@unittest.skip('skipping non-updated test')
def test_show_lists(self):
"""Test show lists for specified user"""
self.api.show_lists(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_get_list_statuses(self):
"""Test timeline of tweets authored by members of the
specified list succeeds"""
self.api.get_list_statuses(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_create_update_destroy_list_add_remove_list_members(self):
"""Test create a list, adding and removing members then
deleting the list succeeds"""
the_list = self.api.create_list(name='Stuff %s' % int(time.time()))
list_id = the_list['id_str']
self.api.update_list(list_id=list_id, name='Stuff Renamed \
%s' % int(time.time()))
screen_names = ['johncena', 'xbox']
# Multi add/delete members
self.api.create_list_members(list_id=list_id,
screen_name=screen_names)
self.api.delete_list_members(list_id=list_id,
screen_name=screen_names)
# Single add/delete member
self.api.add_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list_member(list_id=list_id,
screen_name='justinbieber')
self.api.delete_list(list_id=list_id)
@unittest.skip('skipping non-updated test')
def test_get_list_memberships(self):
"""Test list of memberhips the authenticated user succeeds"""
self.api.get_list_memberships()
@unittest.skip('skipping non-updated test')
def test_get_list_subscribers(self):
"""Test list of subscribers of a specific list succeeds"""
self.api.get_list_subscribers(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_subscribe_is_subbed_and_unsubscribe_to_list(self):
"""Test subscribing, is a list sub and unsubbing to list succeeds"""
self.api.subscribe_to_list(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
# Returns 404 if user is not a subscriber
self.api.is_list_subscriber(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name,
screen_name=screen_name)
self.api.unsubscribe_from_list(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_is_list_member(self):
"""Test returning if specified user is member of a list succeeds"""
# Returns 404 if not list member
self.api.is_list_member(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name,
screen_name='themattharris')
@unittest.skip('skipping non-updated test')
def test_get_list_members(self):
"""Test listing members of the specified list succeeds"""
self.api.get_list_members(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_get_specific_list(self):
"""Test getting specific list succeeds"""
self.api.get_specific_list(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_get_list_subscriptions(self):
"""Test collection of the lists the specified user is
subscribed to succeeds"""
self.api.get_list_subscriptions(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_show_owned_lists(self):
"""Test collection of lists the specified user owns succeeds"""
self.api.show_owned_lists(screen_name='twitter')
# Saved Searches
@unittest.skip('skipping non-updated test')
def test_get_saved_searches(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
self.api.get_saved_searches()
@unittest.skip('skipping non-updated test')
def test_create_get_destroy_saved_search(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
saved_search = self.api.create_saved_search(query='#Twitter')
saved_search_id = saved_search['id_str']
self.api.show_saved_search(id=saved_search_id)
self.api.destroy_saved_search(id=saved_search_id)
# Places & Geo
@unittest.skip('skipping non-updated test')
def test_get_geo_info(self):
"""Test getting info about a geo location succeeds"""
self.api.get_geo_info(place_id='df51dec6f4ee2b2c')
@unittest.skip('skipping non-updated test')
def test_reverse_geo_code(self):
"""Test reversing geocode succeeds"""
self.api.reverse_geocode(lat='37.76893497', long='-122.42284884')
@unittest.skip('skipping non-updated test')
def test_search_geo(self):
"""Test search for places that can be attached
to a statuses/update succeeds"""
self.api.search_geo(query='Toronto')
@unittest.skip('skipping non-updated test')
def test_get_similar_places(self):
"""Test locates places near the given coordinates which
are similar in name succeeds"""
self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ')
# Trends
@unittest.skip('skipping non-updated test')
def test_get_place_trends(self):
"""Test getting the top 10 trending topics for a specific
WOEID succeeds"""
self.api.get_place_trends(id=1)
@unittest.skip('skipping non-updated test')
def test_get_available_trends(self):
"""Test returning locations that Twitter has trending
topic information for succeeds"""
self.api.get_available_trends()
@unittest.skip('skipping non-updated test')
def test_get_closest_trends(self):
"""Test getting the locations that Twitter has trending topic
information for, closest to a specified location succeeds"""
self.api.get_closest_trends(lat='37', long='-122')
# Help
@unittest.skip('skipping non-updated test')
def test_get_twitter_configuration(self):
"""Test getting Twitter's configuration succeeds"""
self.api.get_twitter_configuration()
@unittest.skip('skipping non-updated test')
def test_get_supported_languages(self):
"""Test getting languages supported by Twitter succeeds"""
self.api.get_supported_languages()
@unittest.skip('skipping non-updated test')
def test_privacy_policy(self):
"""Test getting Twitter's Privacy Policy succeeds"""
self.api.get_privacy_policy()
@unittest.skip('skipping non-updated test')
def test_get_tos(self):
"""Test getting the Twitter Terms of Service succeeds"""
self.api.get_tos()
@unittest.skip('skipping non-updated test')
def test_get_application_rate_limit_status(self):
"""Test getting application rate limit status succeeds"""
self.oauth2_api.get_application_rate_limit_status()
from twython import Twython, TwythonAuthError, TwythonError
app_key = 'xiHlth8bVMiFmxGt741g'
app_secret = 'pnSyIx7awxtB3BRCArdfZ5neebAAokzhmPQgzerqlU'
t = Twython(app_key, app_secret)
auth_props = t.get_authentication_tokens(callback_url='http://localhost.com/login_twitter.html')
oauth_token = '90926098-ha8yQlvDrtI92odzBcVlBehmxGpZdrOQqLOJAFs4E'
oauth_token_secret = '2n8hPIA9yF6J20G9ZjWAcDLAnKarvSYbYyjV8EDw'
oauth_verifier = ''
try:
t = Twython(app_key, app_secret, oauth_token, oauth_token_secret)
followers = t.get_followers_ids()
for follower_id in followers['ids']:
print follower_id
tweets = t.get_home_timeline()
for tweet in tweets:
print 'Name: %s \n' % (tweet['user']['name'].encode('utf-8'))
print 'Tweet: %s \n' % (tweet['text'].encode('utf-8'))
mentions = t.get_mentions_timeline()
for mention in mentions:
print 'Name: %s \n' % (mention['user']['name'].encode('utf-8'))
print 'Mention: %s \n' % (mention['text'].encode('utf-8'))
class RecolectorSeguidoresShort(Recolector):
def __init__(self, escritores):
super(RecolectorSeguidoresShort, self).__init__(escritores)
self.authorizator = GetAuthorizations(15)
self.twitter = None
self.apoyo = ApoyoTwitter()
self.tipo_id = 4
self.inicializa()
self.cursor = -1
self.lastQuery = ""
def inicializa(self):
self.authorizator.load_twitter_token(self.tipo_id)
api_key, access_token = self.authorizator.get_twython_token()
self.twitter = Twython(api_key, access_token=access_token)
def cleanCursor(self):
self.cursor = -1
def recolecta(self, query=None, id_user=-1, complete=False):
if query is None and id_user == -1:
raise Exception('Al menos debe haber un parametro usable')
if query is not None:
if query[0] == '@':
query = query[1:]
if self.lastQuery == query:
print "query es igual"
return
else:
self.lastQuery = query
self.cursor = -1
id_user = self.apoyo.getUserIDByScreenName(query)
if id_user == None:
raise Exception('El usuario debe estar en la base de datos')
else:
self.cursor = -1
retornoFinal = {"ids": []}
while True:
retorno = self.privateRealizaConsultaById(id_user)
print self.cursor
if retorno == []:
break
retornoFinal["ids"].extend(retorno["ids"])
if len(retorno["ids"]) < 5000:
break
if complete == False:
break
#se almacenan las relaciones en una tupla (usuario1, usuario2)
#esto quiere decir que el usuario1 sigue al usuario2.
lista_relaciones = []
for identificador in retornoFinal["ids"]:
lista_relaciones.append((identificador, id_user))
self.guarda(lista_relaciones)
def guarda(self, arrayDatos):
for escritor in self.escritores:
escritor.escribe(arrayDatos)
def privateRealizaConsultaById(self, identificador):
if self.authorizator.is_limit_api(self.tipo_id):
raise Exception('LIMITE')
try:
retorno = self.twitter.get_followers_ids(user_id=identificador,
cursor=self.cursor,
count='5000')
self.authorizator.add_query_to_key(self.tipo_id)
if len(retorno["ids"]) == 0:
return []
self.cursor = retorno["next_cursor"]
return retorno
except Exception, e:
self.authorizator.add_query_to_key(self.tipo_id)
if "429" in str(e):
raise Exception('LIMITE')
print e
return []
def get_active_users_around(center, start_date, end_date, threshold, num_friends, num_followers, waiting):
twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN)
user_ids = []
all_tweets = []
# get friends and followers around a central user
print "Getting users..."
while True:
try:
following = twitter.get_friends_ids(screen_name=center, count=5000)
time.sleep(0.1)
followers = twitter.get_followers_ids(screen_name=center, count=5000)
break
except TwythonAuthError as e:
print "TwythonAuthError..."
time.sleep(waiting)
continue
except TwythonRateLimitError as e:
print "TwythonRateLimitError... Waiting for 3 minutes (may take multiple waits)"
for timer in tqdm(range(3*60)):
time.sleep(1)
twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN)
time.sleep(waiting)
continue
except TwythonError as e:
print "TwythonError: " + str(e.error_code) + "..."
time.sleep(waiting)
continue
# choose out random friends and followers from the previous list, with a safety factor of 2 times as many as asked for to account for the fact that some users will not be active
following = map(lambda x: str(x), np.random.choice(following['ids'], min(int(num_friends * 2), len(following['ids'])), replace=False))
followers = map(lambda x: str(x), np.random.choice(followers['ids'], min(int(num_followers * 2), len(followers['ids'])), replace=False))
following = get_usernames(following, waiting)
followers = get_usernames(followers, waiting)
# figure out how many tweets are necessary to deem a user "active"
start = datetime.strptime(start_date, "%Y-%m-%d")
end = datetime.strptime(end_date, "%Y-%m-%d")
threshold = abs(end-start).days * threshold
# collect out as many active followed users from the central user (and their tweets) as we can, up to a limit of the number asked for
collected_friends = []
for user in following:
user_timeline = get_timeline_in_range(user, start_date, end_date, waiting)
if len(user_timeline) >= threshold:
collected_friends.append(user)
all_tweets.extend(user_timeline)
if len(collected_friends) >= num_friends:
break
# collect out as many active followers from the central user (and their tweets) as we can, up to a limit of the number asked for
collected_followers = []
for user in followers:
user_timeline = get_timeline_in_range(user, start_date, end_date, waiting)
if len(user_timeline) >= threshold:
collected_followers.append(user)
all_tweets.extend(user_timeline)
if len(collected_followers) >= num_followers:
break
# return our list of users and their tweets from the time period in question
user_ids.extend(collected_friends)
user_ids.extend(collected_followers)
return (user_ids, all_tweets)
timeout = 10 * 60
idnum = '0' # a tweet ID can be put in here to limit searches
count = 0
# your twitter consumer and access information goes here
apiKey = '****'
apiSecret = '****'
accessToken = '****'
accessTokenSecret = '****'
twitter = Twython(apiKey,apiSecret,accessToken,accessTokenSecret)
while True:
#auto follow back function
try:
followers = twitter.get_followers_ids(screen_name = "Botrack Obama") #Enter twitter handle here
for followers_ids in followers['ids']:
twitter.create_friendship(user_id=followers_ids)
except TwythonError as e:
print(e)
search_results = twitter.search(q="thanksobama", since_id = idnum, count=20)
#searchs a hashtag from after a certian tweet with a max of 20 results
count = 0
for tweet in search_results["statuses"]:
try:
rannum = random.randrange(1,5)
print(rannum)
if rannum == 2:
#Below line retweets the statuses
twitter.retweet(id = tweet["id_str"])
def get_followers_following(usr_id):
index = 0
print "GETTING FOLLOWERS AND FOLLOWING: " + str(usr_id)
friend_cursor = -1
follower_cursor = -1
APP_KEY = APP_KEYS[index]
APP_SECRET = APP_SECRETS[index]
OAUTH_TOKEN = OAUTH_TOKENS[index]
OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index]
while (1 == 1):
try:
twitter = Twython(APP_KEY, APP_SECRET)
auth = twitter.get_authentication_tokens()
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN,
OAUTH_TOKEN_SECRET)
friends = ""
while (friend_cursor != 0):
following = twitter.get_friends_ids(id=usr_id,
cursor=friend_cursor)
for ID in following['ids']:
friends += str(ID) + " "
friend_cursor = following["next_cursor"]
friends = friends[:-1]
if len(friends) == 0:
friends = "null"
follow = ""
while (follower_cursor != 0):
followers = twitter.get_followers_ids(id=usr_id,
cursor=follower_cursor)
for ID in followers['ids']:
follow += str(ID) + " "
follower_cursor = followers["next_cursor"]
follow = follow[:-1]
if len(follow) == 0:
follow = "null"
return (friends, follow)
except Exception as e:
print e
if "429 (Too Many Requests)" in str(e):
#global index
index += 1
if index == len(APP_KEYS):
index = 0
print "sleepy time - 15 minutes"
print datetime.datetime.now()
time.sleep(910)
return get_followers_following(usr_id)
elif "401 (Unauthorized)" in str(e):
print "401 error"
return ("null", "null")
elif "404 (Not Found)" in str(e):
print "404 error"
return ("null", "null")
else:
print e
return ("null", "null")
class TwitterBot():
'''
Crawls twitter and updates social network and tagged resources to repository
'''
def __init__(self):
self.service = TWITTER
APP_KEY = "w7zjbW1qHtzX2GxbrHbeNgyPr"
APP_SECRET = "qJFawkFxwYZX6U3DYNvJI3czWEjix2owU9wTXZ9BorMkhFU4Yb"
self.twapi = Twython(APP_KEY, APP_SECRET)
self.userBaseUri = 'http://twitter.com/%s' # % user screen_name
self.tagBaseUri = 'http://twitter.com/#search?q=#%s' # % trend key word
self.tweetBaseUri = 'http://twitter.com/%s/status/%s' # % (user screen_name, status id)
@staticmethod
def getname():
return self.service
@staticmethod
def getDefaultAvatar():
return defAvatar
def setStartUserId(self, start_user_id):
self.start_user_id = start_user_id
def setMaxLevel(self, max_level):
self.max_level = max_level
def setLastAdded(self, last_added):
self.last_added = last_added
def updateDatabase(self, user_metadata, user_ntw, udate=None):
raise NotImplementedError()
def fetchUser(self, username):
return (None, {'knows':[], 'fans':[]})
def getUserUri(self, username):
return self.userBaseUri % username
@staticmethod
def factory(user, params=None, depth=2, last_added=None, verbose=False):
# Twitter crawler
twcrawl = TwitterBot()
twcrawl.setStartUserId(user)
twcrawl.setMaxLevel(depth)
twcrawl.setLastAdded(last_added)
return twcrawl
def updateDatabase(self, user_metadata, user_ntw, udate=None):
username = user_metadata['username']
userUri = self.userBaseUri % username
resourceUri = None
added = 0
# add user to repository
dbaccess.User.addToDB(userUri, self.service, username,
TwitterBot.getDefaultAvatar(), udate=udate)
# add resources, tags and relations between them to repository
for resource in user_metadata['data']:
# add tweet resource to repository
resourceId = resource['id']
resourceTitle = resource['text']
resourceUri = self.tweetBaseUri % (username, resourceId)
dbaccess.Tweet.addToDB(resourceUri, resourceTitle, udate=udate)
added += 1
# add tag(trend) relations to repository
for tag in self.parseTags(resourceTitle):
tagUri = self.tagBaseUri % urllib.quote(tag.encode('utf-8'))
# add tag to repository
dbaccess.Tag.addToDB(tagUri, tag, udate=udate)
# add user, resource, tag relation to repository
dbaccess.UserResourceTag.addToDB(userUri, resourceUri, tagUri, udate=udate)
# add url relations between the tweet and urls contained
urls = map(self.getRealUrl, self.parseUrls(resourceTitle))
for url in urls:
dbaccess.Bookmark.addToDB(url, None)
dbaccess.Tweet.addReferenceToDB(resourceUri, url)
# add user social network relations to repository
# add friends
for u in user_ntw['knows']:
knownUserUri = self.userBaseUri % u
dbaccess.User.addToDB(knownUserUri, self.service, u,
TwitterBot.getDefaultAvatar(), udate=udate)
dbaccess.User.addToDBUserKnowsUser(userUri, knownUserUri, udate=udate)
# add followers
for u in user_ntw['fans']:
otherUserUri = self.userBaseUri % u
dbaccess.User.addToDB(otherUserUri, self.service, u,
TwitterBot.getDefaultAvatar(), udate=udate)
dbaccess.User.addToDBUserKnowsUser(otherUserUri, userUri, udate=udate)
# return number of added resources and last added resource uri
return added
def fetchUserMetadata(self, username):
# get user tweets (only last 20)
data = self.twapi.get_user_timeline(screen_name=username)
return { 'username': username, 'data': data}
def fetchUserNetwork(self, username):
chunk_size = 100 # max returned by bulkUserLookup
# Get Friends
fids, cursor = [], -1
fchunks = [fids[i:i + chunk_size] for i in range(0, len(fids), chunk_size)]
while True:
f = self.twapi.get_friends_ids(screen_name=username, cursor=cursor)
fids.extend(f['ids'])
if f['next_cursor'] == cursor or len(f['ids']) == 0:
break
cursor = f['next_cursor']
# get screen names from ids in chunks of 100
screen_names = []
for chunk in fchunks:
screen_names = []
screen_names.extend([userdata['screen_name']
for userdata in self.twapi.lookup_user(user_id=chunk)])
ntw_knows = screen_names
print ntw_knows
# Get Followers
fids, cursor = [], -1
while True:
f = self.twapi.get_followers_ids(screen_name=username, cursor=cursor)
fids.extend(f['ids'])
if f['next_cursor'] == cursor or len(f['ids']) == 0:
break
cursor = f['next_cursor']
fchunks = [fids[i:i + chunk_size] for i in range(0, len(fids), chunk_size)]
fan_screen_names = []
for chunk in fchunks:
# get all data for all user ids
fan_screen_names.extend([userdata['screen_name']
for userdata in self.twapi.lookup_user(user_id=chunk)])
ntw_fans = fan_screen_names
return {'knows': ntw_knows, 'fans': ntw_fans}
def fetchUser(self, username):
return (self.fetchUserMetadata(username), self.fetchUserNetwork(username))
def parseTags(self, text):
'''
Parses trends from tweet: '#trend #tre#nd ## #trend,two #tren%^& smth,#da #1dc #100 #1dc$ #100^',
finds trends: trend, tre, trend, tren, da, 1dc, 100, 1dc, 100.
Text parameter MUST be UNICODE encoded: u'some text'
'''
return re.findall('(?<=[\W^#]#)\w+', ' ' + text, re.UNICODE)
def parseUrls(self, text):
'''Returns a list of urls from the text.
'''
return re.findall('http://\S+', text)
def getRealUrl(self, url):
'''Returns the redirected url if *url* parameter is a shortened url (a redirect was produced),
or itself if not.
'''
try:
u = urllib2.urlopen(url)
realurl = u.geturl()
if realurl == None:
raise urllib2.HTTPError('Could not fetch url')
except:
realurl = url
return realurl
def crawlUserNetwork(self, user_id=None, max_level=0, start_time=None):
'''
user_id - an id to uniquely identify a user (can be a username or userid)
'''
user_id = user_id or self.start_user_id
max_level = max_level or self.max_level
queue = [user_id] # init queue
visited = {user_id: 0} # init visited nodes
added = 0 # set number of added resources
while queue:
v = queue.pop(0) # pop next user node
level = visited[v] # get their level
# get user metadata and social network
user_metadata, user_ntw = self.fetchUser(v)
# update database with user data, tagged resources and social network relations
a = self.updateDatabase(user_metadata, user_ntw, udate=start_time)
added += a
# explore following nodes
for w in user_ntw['knows']:
if w not in visited and level < max_level:
queue.append(w)
visited[w] = level + 1
return added
__copyright__ = "Copyright \u00a9 Benjamin D. McGinnes, 2013-2014"
__copyrighta__ = "Copyright (C) Benjamin D. McGinnes, 2013-2014"
__license__ = "BSD"
__version__ = "0.0.1"
__bitcoin__ = "1KvKMVnyYgLxU1HnLQmbWaMpDx3Dz15DVU"
import sys
from twython import Twython, TwythonError
from config import *
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
l = len(sys.argv)
if l >= 2:
user = sys.argv[1]
else:
user = input("Enter Twitter handle to get followers of: ")
try:
followids = twitter.get_followers_ids(screen_name=user)
except TwythonError as e:
print(e)
for x in followids["ids"]:
try:
data = twitter.show_user(user_id=x)
print(data["screen_name"])
except TwythonError as e:
print(e)
ACCESS_TOKEN = twitter.obtain_access_token()
twitter = Twython(APP_KEY[0], access_token=ACCESS_TOKEN)
ins = open("ToFetch.txt", "r" )
con = lite.connect(DATABASE)
for line in ins:
line = line.rstrip('\n')
line = line.rstrip()
USER_NAME = line
try:
followers = twitter.get_followers_ids(screen_name = USER_NAME, count = 1000)
except TwythonRateLimitError:
print "API Limit reached"
sys.exit()
try:
user = twitter.show_user(screen_name = USER_NAME)
except TwythonRateLimitError:
print "API Limit reached"
sys.exit()
for follower_id in followers['ids']:
with con:
cur = con.cursor()
cur.execute("INSERT INTO Profiles VALUES('" + user['id_str'] + "','" + str(follower_id) + "')")
# ------------------------------------------------------------------
# Initialize the database followers record for that screen_name
res = db.followers.find_one({"screen_name": screen_name})
if res is None:
db.followers.insert_one({"screen_name": screen_name, "ids": []})
# cursor is used to navigate a twitter collection
# https://dev.twitter.com/overview/api/cursoring
next_cursor = -1
follower_ids = list()
ids_count = 0
while (next_cursor != 0) and (ids_count < n_followers):
try:
print("Followers %s to %s: cursor: %s" % (ids_count, ids_count + followers_query_size, next_cursor))
result = twitter.get_followers_ids(screen_name=screen_name, count=followers_query_size, cursor=next_cursor)
follower_ids = follower_ids + result["ids"]
next_cursor = result["next_cursor"]
ids_count += len(result["ids"])
# make sure the list only has unique IDs and sort
follower_ids = list(set(follower_ids))
follower_ids.sort()
print("Retrieved %s follower IDs from twitter" % len(follower_ids))
# store what we've got so far
# insert follower_ids in the followers collection
res = db.followers.update_one({"screen_name": screen_name}, {"$set": {"ids": follower_ids}})
if res.matched_count == 0:
print("Unable to update IDs for: ", screen_name)
elif res.modified_count == 0:
print("%s IDs not modified" % screen_name)
class TwitBot(object):
def __init__(self,c_key, c_secret,
o_token, o_token_secret):
self.twitter = Twython(c_key, c_secret, o_token, o_token_secret)
#last tweet id for query "xохол"
self.id = [None]
#last tweet id for query "xахол"
self.jd = [None]
#last mentions id
self.m_id = 0
#last timelines tweet id
self.t_id = 0
#list of time_of_days, when bot should update status
self.time = [(4, 55), (17, 55), (19, 55)]
#count of replies of day
self.replies_limit = 0
self.replies_count = 35
self.flag = True
#users victims
self.users = (u"feeling_so_real", u"captein_treniki",
u"fe_city_boy", u"drunktwi",
u"Doppler_Effectt", u"koffboy")
def get_users(self, result, debug = 0):
#users = [(tweet id, user screen_name)]
users = [(int(c["id"]), c["user"]["screen_name"])
for c in result["statuses"] if
(c["user"]["screen_name"] != u"ghohol" and
u"ghohol" not in c["text"])
]
retweets = self.my_reetwets()
users = [x for x in users if x[0] not in retweets]
#sorted tweets from older to younger
users = sorted(users)
if not debug:
print "last user", users[-1]
print "#" * 40,'\n'
return users
################################## RUN SEARCH ############################################
def run_search(self, query, text):
self.query = query
result = self.twitter.search(q=query)
if result:
users = self.get_users(result)
for user in users:
if self.replies_count < self.replies_limit:
self.update_status(text,user)
else:
break
sleep(randint(180,210))
else:
sleep(randint(240, 480))
def update_status(self, text, user):
q_id = self.id if self.query == u"хохол" else self.jd
#replayed tweet or not
if user[0] > q_id[0]:
self.twitter.update_status(status= u"@{0} {1}".format(
user[1], choice(text)),
in_reply_to_status_id=user[0])
self.replies_count +=1
#save last tweet id
q_id[0] = user[0]
sleep(randint(400,560))
@wrapper(n=40)
def big_search(self):
for query in QUERYS:
self.run_search(query, replays)
sleep(70)
sleep(randint(60,120))
################################################################################
################################ HOME TIMELINE ######################################
@wrapper()
def home_timeline(self):
""" update count of replies
update timeline(random rt or fav in homeline)
"""
hour = self.data.hour
#debug
print self.replies_count, "count"
print self.replies_limit, "limit"
#end debug
def update_replies_count():
"""update flags and replies limit"""
if self.flag:
if hour in (10,17):
self.replies_limit = randint(3,12)
self.flag = False
self.replies_count = 0
def timeline_updates():
"""check time and run if time rt and fav in homeline"""
if (hour in range(0, 3) or hour in range(9, 24)):
print "CHECK TIMELINE UPDATES"
result = self.get_home_timeline(count=30,exclude_replies=1,
since_id=self.t_id)
if result:
tw = [tweet["id"] for tweet in result if (tweet['user']['screen_name'] != u"ghohol" and
not tweet['entities']['user_mentions'])]
if tw:
self.t_id = tw[0]
rt_or_fav = lambda func : map(func, filter(lambda x:x % randint(21,27) == 0, tw))
if (randint(0,7) == randint(0,7)):
map(rt_or_fav,(self.retweet, self.create_favorite))
else:
print "NO TIMELINE UPDATES"
update_replies_count()
timeline_updates()
################################### Tools #############################################
for name in ("retweet", "create_favorite", "destroy_status",
"destroy_friendship"):
exec("""def {0}(self, id):
self.twitter.{0}(id=id)""".format(name))
for name in ("get_home_timeline", "get_direct_messages", "get_friends_ids",
"get_followers_ids", "get_user_timeline", "show_status"):
exec("""def {0}(self, *args, **kwargs):
return self.twitter.{0}(*args, **kwargs)""".format(name))
@property
def data(self):
return datetime.now()
def my_reetwets(self):
"""return id of my retweets"""
retweets = self.twitter.retweeted_of_me()
retweets_id = [c["id"] for c in retweets]
return retweets_id
@wrapper(n=30)
def tw_reader(self, tw):
for c in tw:
print "Tweet from @{0} ID: {1}".format(c[1].encode('utf-8'), c[0])
print c[2].encode('utf-8'), '\n'
############################### GET REPLAYS #############################################
@wrapper(n=0)
def get_replays(self):
dirty_list = (u"хуй", u"пидор",u"пидар", u"пидр",
u"бля", u"блядь", u"сука",u"ебень",
u"гондон", u"гандон", u"шлюха",
u"чмо", u"залупа", u"eблан",
u"козел", u"казел", u"козлина",
u"тварь", u"долбоеб", u"бандеровец",
u"блять", u"уеби", u"бендеровец",
u"дебил", u"дибил", u"петух",
u"питух", u"петушок", u"ебали",
u"хуя"
)
def is_shit(tweet):
t = tweet[2].lower()
for word in dirty_list:
if word in t:
return word
return False
repl = self.twitter.get_mentions_timeline(include_rts = 0,
since_id = self.m_id)
print repl, "repl"
if repl:
print "IF REPL"
repl =[(c["id"],c["user"]["screen_name"],c["text"]) for c
in repl if u"RT" not in c["text"]]
self.m_id = repl[0][0]
repl = filter(is_shit, repl)
if repl:
print "IF REPL INNER"
print "Try send mention for dirty reply"
for tw in repl:
(self.retweet(tw[0]) if randint(0,1)
else self.update_dirty_status(tw[1],tw[0]))
#return repl
print repl, "repl"
return repl
def update_dirty_status(self, name, id):
answers = [ u"Ты еще и ругаешься",
u"Попросил бы без оскорблений",
u"Очень грубо",
u"А можно без мата?",
u"Кто так обзывается, сам так называется",
u"Сколько мата тьфу ты, бля"
]
self.update_status(status= u"@{0} {1}".format(
name, choice(answers)),
in_reply_to_status_id=id)
sleep(randint(140,280))
######################################## STEAL TWEET ################################
@wrapper(n=0)
def steal_tweets(self, d):
hour = d.hour
print hour, "FGGGGGGGGGGGGGGGGJJGJGJGJGJJJJJJJJJJJJJJJJJ"
if hour in (5, 16):
print "TIME TO STEAL TWEETS"
#get my tweets
my_tweets = self.twitter.get_user_timeline(count=200, exclude_replies=1)
my_tweets = [c["text"] for c in my_tweets]
#get victim's tweets
get_victims_timeline = self.get_victims_timeline(my_tweets)
victims_tweets = map(get_victims_timeline, self.users)
for tweet in victims_tweets:
print tweet.encode('utf-8')
if tweet is not None:
self.twitter.update_status(status=tweet)
sleep(3600)
print "STEALING TWEETS IS OVER"
else:
sleep(3600)
def get_victims_timeline(self, text):
def victims_tweets(name):
raw_tweets = self.twitter.get_user_timeline(screen_name=name, exclude_replies=1, count=60)
print "dddd"
print "raw_tweets", len(raw_tweets)
tweets = raw_tweets[-1:-len(raw_tweets):-1]
#print tweets
tweets = [c for c in tweets if (not c["entities"]['urls'] and
len(c['entities'])==4 and
not c['entities']['user_mentions']
)]
sample_tweet = [c['text'] for c in tweets if c['text'] not in text]
if sample_tweet:
sample_tweet = sample_tweet[-1]
print sample_tweet.encode('utf-8'), "sample_tweet"
return sample_tweet
else:
return None
return victims_tweets
######################################## START ########################################
@wrapper(n=0)
def start(self):
starters = ("jd", "id", "t_id", "m_id")
d = dict(zip(QUERYS,(self.jd, self.id)))
def complete():
while True:
s = yield
if s:
print "##### COMPLETE #####\n"
print "{0} = {1} {2} complete\n".format(s, getattr(self, s) ,'.' * 25)
gen = complete()
gen.next()
while not all([eval("self." + c) for c in starters]):
for query in QUERYS:
result = self.twitter.search(q=query)
d[query][0] = self.get_users(result,debug=1)[-1][0]
self.t_id = self.twitter.get_home_timeline(exclude_replies = 1)[0]["id"]
self.m_id = self.twitter.get_mentions_timeline(count=30, include_rts = 0)[0]["id"]
[gen.send(c) for c in starters]
###################################### DELETE REPLIES ##############################################
@wrapper()
def delete_replies(self):
if self.data.hour == 23:
print "START CLEAR TWEETS"
tweets = self.get_user_timeline(count=100)
tweets = (c["id"] for c in tweets if c["in_reply_to_user_id"] )
sleep(15)
map(self.destroy_status, tweets)
else:
print "NO CLEAN"
################################## UNFOLLOW ######################################
@wrapper()
def unfollow_who_not_follow_back(self):
data = self.data
#clean followers every odd day
if data.day % 2 == 0 and data.hour == 1:
# "stars" id's
stars_id = (462792965, 14774424, 344512640,
265008715, 412493190, 254655960,
179484444, 90647948, 408437933,
98422492, 81297044, 250801581)
print "UNFOLLOWING START"
#get friends id's
friends_ids = self.get_friends_ids()[u"ids"]
friends_ids = (id for id in friends_ids if id not in stars_id)
#get followers ids
followers_ids = self.twitter.get_followers_ids()[u"ids"]
#unfollowing list
destroy_list = (user_id for user_id in friends_ids
if user_id not in followers_ids)
map(self.destroy_friendship, destroy_list)
print "UNFOLLOWING END"
############################### DATE STATUS ############################################
def date_status(self, text):
current_time = self.data
print "current time = {0}".format(str(current_time))
#this part for count_replies
if current_time.hour in (9, 16):
self.flag = True
sleep(randint(1600, 2400))
## def part_of_day(time_of_day):
## """return True if current time equal time_of_day"""
## return (current_time.hour == time_of_day[0] and
## current_time.minute in range(time_of_day[1]))
## check_time = filter(part_of_day, self.time)
## if check_time:
## print "check_time"
## message = choice(text[self.time.index(check_time[0])])
## try:
## self.twitter.update_status(status= u"{0}".format(message))
## sleep(randint(1600, 2400))
## except TwythonError as err:
## print err
## sleep(480)
else:
sleep(randint (1200, 1800))
influential_user_list = pd.read_csv(#path)
influential_user_list = influential_user_list[0:5]
for index,row in influential_user_list.iterrows():
row['name'] = row['name'].replace("b'","")
row['name'] = row['name'].replace("'","")
row['name'] = row['name'].replace(" ","")
row['user_name'] = row['user_name'].replace("b'@","")
row['user_name'] = row['user_name'].replace("'","")
if not os.path.isfile(#path) :
print("---------------------------------------------------")
print('Scraping followers of:', row['name'],)
next_cursor = -1
followers_list = []
while next_cursor != 0 :
try:
followers_element = twitter_final.get_followers_ids(screen_name = row['user_name'], cursor = next_cursor,count = 5000)
followers_list.extend(followers_element['ids'])
next_cursor = followers_element['next_cursor']
print("followers scraped so far:",str(len(followers_list)))
except TwythonRateLimitError as error:
remainder = float(twitter_final.get_lastfunction_header(header='x-rate-limit-reset')) - time.time()
del twitter_final
if remainder > 0:
time.sleep(remainder)
twitter_final = Twython(API_Key, API_Secret,Access_Token, Access_Token_Secret)
else:
remainder *= -1
time.sleep(remainder)
twitter_final = Twython(API_Key, API_Secret,Access_Token, Access_Token_Secret)
continue
with open(#path) as f:
os.environ["TWITTER_TOKEN"],
os.environ["TWITTER_TOKEN_SECRET"])
s3API = S3Connection(os.environ["S3_KEY"], os.environ["S3_SECRET_KEY"])
key = s3API.get_bucket("wespooky-twittermanagement").get_key("followers.txt")
# ======================
# -=- CODE TIME -=-
# ======================
# == Load Sets ==
lastrunFollowers = set([
int(follower)
for follower in key.get_contents_as_string().decode().split("|")
])
currentFollowers = set(twitterAPI.get_followers_ids()["ids"])
# == Follow Back ==
# DESC: Follow everyone in current follower list who isn't in lastrun follower list
toFollow = currentFollowers.difference(lastrunFollowers)
for ID in toFollow:
try:
user = twitterAPI.show_user(user_id=ID)
twitterAPI.create_friendship(user_id=ID)
sys.stdout.write("Followed {} (@{}) back.\n".format(
user["name"], user["screen_name"]))
except: # Rate limit or 404 (should probably do special stuff here later)
pass
# A little script to get my followers twitter IDs
# and display them next to their screen_name (twitter handle)
from twython import Twython, TwythonError
from threading import Event, Thread, Timer
global twitter
print 'twitter client requested'
twitter = Twython('XXXXXXX', #API_KEY
'XXXXXXX', #APP_SECRET,
'XXXXXXX', #OAUTH_TOKEN,
'XXXXXXX')#OAUTH_TOKEN_SECRET
myFollowers = twitter.get_followers_ids(id = "XXXX", count=50) #last 50 followers
for follower in myFollowers["ids"]:
print follower
myfriend = twitter.show_user(id = follower)
print myfriend["screen_name"]
print ""
def updateFollowerFollowings(dbpath, user_table="users"):
distinct = get_D_UsrID(dbpath, sn_table)
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "select Usr_ID from %s where Following is null or Followers is null"
cursor.execute(query)
distinct = cursor.fetchall()
follower_cursor = -1
friend_cursor = -1
print "Starting @ %d" % (ind)
print datetime.datetime.now()
while ind < len(distinct):
#for i in distinct:
i = distinct[ind][0] # Usr_ID
APP_KEY = APP_KEYS[index]
APP_SECRET = APP_SECRETS[index]
OAUTH_TOKEN = OAUTH_TOKENS[index]
OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index]
try:
#twitter = Twython (APP_KEY, APP_SECRET)
#auth = twitter.get_authentication_tokens()
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN,
OAUTH_TOKEN_SECRET)
friends = ""
while (friend_cursor):
following = twitter.get_friends_ids(id=i, cursor=friend_cursor)
for ID in following['ids']:
friends += str(ID) + " "
friend_cursor = following["next_cursor"]
num_checked += 1
friends = friends[:-1]
if len(friends) == 0:
friends = "null"
follow = ""
while (follower_cursor):
followers = twitter.get_followers_ids(id=i,
cursor=follower_cursor)
for ID in followers['ids']:
follow += str(ID) + " "
follower_cursor = followers["next_cursor"]
num_checked += 1
follow = follow[:-1]
if len(follow) == 0:
follow = "null"
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "UPDATE %s SET Following = '%s', Followers = '%s', NumFollowing = '%d', NumFollowers = '%d' where Usr_ID = '%s'" % (
sn_table, friends, follow, len(
friends.split(" ")), len(follow.split(" ")), i)
cursor.execute(query)
#print query
conn.close()
print "____________________ %dth following and friends and their corresponding counts updated ________ %d" % (
ind, i)
follower_cursor = -1
friend_cursor = -1
ind += 1
except Exception as e:
print e
if "429 (Too Many Requests)" in str(e):
index += 1
if index == len(APP_KEYS):
index = 0
print "sleepy time; pausing for 15 minutes for APP Keys to renew"
print datetime.datetime.now()
#time.sleep(910)
print "App Rate Limits has been exceeded- we will wait for next runtime before running rest of followers and followings"
return
elif "401 (Unauthorized)" in str(e):
print "401 Unauthorized Error nulling %d________________Usr_ID:%d" % (
ind, i)
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "UPDATE %s SET Following = 'null', Followers = 'null' where Usr_ID = '%s'" % (
sn_table, i)
cursor.execute(query)
conn.commit()
conn.close()
ind += 1
elif "404 (Not Found)" in str(e):
print "404 Not Found Error nulling %d________________Usr_ID:%d" % (
ind, i)
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "UPDATE %s SET Following = 'null', Followers = 'null' where Usr_ID = '%s'" % (
sn_table, i)
cursor.execute(query)
conn.commit()
conn.close()
ind += 1
else:
print e
def get_following_to_db(dbpath, sn_table = "tweets9_users"):
distinct = get_D_UsrID(dbpath,sn_table )
index=0
num_checked = 0
rate_ex = 0
ind = 0
already_checked = False
follower_cursor = -1
friend_cursor = -1
print "Starting @ %d"%(ind)
print datetime.datetime.now()
while ind < len(distinct):
#for i in distinct:
i = distinct[ind][0]
APP_KEY = APP_KEYS[index]
APP_SECRET = APP_SECRETS[index]
OAUTH_TOKEN = OAUTH_TOKENS[index]
OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index]
try:
#twitter = Twython (APP_KEY, APP_SECRET)
#auth = twitter.get_authentication_tokens()
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
#friend_results = twitter.cursor(twitter.get_friends_ids, id = i, cursor= friend_cursor)
friends = ""
while (friend_cursor):
following = twitter.get_friends_ids(id = i,cursor= friend_cursor)
for ID in following['ids']:
friends += str(ID) + " "
friend_cursor = following["next_cursor"]
num_checked += 1
friends = friends[:-1]
if len(friends) == 0:
friends = "null"
follow = ""
while (follower_cursor):
followers = twitter.get_followers_ids(id = i,cursor= follower_cursor)
for ID in followers['ids']:
follow += str(ID) + " "
follower_cursor = followers["next_cursor"]
num_checked += 1
follow= follow[:-1]
if len(follow) == 0:
follow = "null"
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "UPDATE %s SET Following = '%s' where Usr_ID = '%s'"% (sn_table, friends, i)
cursor.execute(query)
#print query
print "____________________ %dth following updated ________ %d" % (ind, i)
conn.commit()
query = "UPDATE %s SET Followers = '%s' where Usr_ID = '%s' "% (sn_table, follow, i)
cursor.execute(query)
#print query
conn.commit()
conn.close()
print "%dth follower updated_______________________ %d" % (ind, i)
print num_checked
print
follower_cursor = -1
friend_cursor = -1
ind+=1
except Exception as e:
print e
if "429 (Too Many Requests)" in str(e):
index += 1
if index == len(APP_KEYS):
index = 0
print "sleepy time - 15 minutes"
print datetime.datetime.now()
time.sleep(910)
elif "401 (Unauthorized)" in str(e):
print "401 error"
f = open("skipped.txt", "a")
f.write("skipped %dth element, ID: %d\n"%(ind, i))
f.write("__________________________________________"+str(datetime.datetime.now()) + "\n")
f.close()
print "401 nulled %d"%ind
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "UPDATE %s SET Following = 'null', Followers = 'null' where Usr_ID = '%s'"% (sn_table, i)
cursor.execute(query)
conn.commit()
conn.close()
ind+=1
time.sleep(1)
elif "404 (Not Found)" in str(e):
print "404 error"
f = open("skipped.txt", "a")
f.write("404: skipped %dth element, ID: %d \n"%(ind, i))
f.write("__________________________________________"+str(datetime.datetime.now()) + "\n")
f.close()
print "404 nulled %d"%ind
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "UPDATE %s SET Following = 'null', Followers = 'null' where Usr_ID = '%s'"% (sn_table, i)
cursor.execute(query)
conn.commit()
conn.close()
ind+=1
time.sleep(1)
else:
print e
rate_ex += 1
print ind
print num_checked
print rate_ex
'''
ACCESS_TOKEN_SECRET = 'INSERT HERE'
twitter = Twython(API_KEY, API_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
return (twitter)
'''
Determining your Twitter followers and friends
'''
twitter = twitter_oauth_login()
friends_ids = twitter.get_friends_ids(count=5000)
friends_ids = friends_ids['ids']
followers_ids = twitter.get_followers_ids(count=5000)
followers_ids = followers_ids['ids']
print len(friends_ids), len(followers_ids)
friends_set = set(friends_ids)
followers_set = set(followers_ids)
print(
'Number of Twitter users who either are our friend or follow you (union):')
print(len(friends_set.union(followers_set)))
len(friends_set | followers_set)
len(set(friends_ids + followers_ids))
print(
def get_followers_ids(oauth_token, oauth_token_secret, user_id, count=5000):
twitter = Twython(consumer_key, consumer_secret, oauth_token, oauth_token_secret)
result = twitter.get_followers_ids(user_id=user_id, count=count)
return result
#!/usr/bin/env python3
from twython import Twython
import json
# read Twitter creds from file
creds = {}
with open("twitter_creds.json") as json_data:
creds = json.load(json_data)
twitter = Twython(creds["CONSUMER_KEY"], creds["CONSUMER_SECRET"])
twitter = Twython(
creds["CONSUMER_KEY"],
creds["CONSUMER_SECRET"],
creds["ACCESS_TOKEN"],
creds["ACCESS_SECRET"],
)
followers = twitter.get_followers_ids()
print(followers["ids"])
players_ids = {'imVkohli': 71201743,
'sachin_rt': 135421739,
'msdhoni': 92708272,
'henrygayle': 45452226,
'ImRo45': 121046433,
'BrianLara': 861083240 }
# In[5]:
# Creating the players followers list
for player, playerid in players_ids.items():
followers_list[playerid] = twitter_trilok.get_followers_ids(screen_name=player, count=40)
time.sleep(2)
# In[6]:
file2 = open('players_followers.obj', 'wb')
pickle.dump(followers_list, file2)
file2.close()
# In[7]:
# All the followers append to a master list
TWITTER_APP_KEY = 'vgDa2r4tFyNyDsR0qSbFVNQev'
TWITTER_APP_KEY_SECRET = 'VzfMzSFYLCSM4NFPaMXRIgVrtEDtGPO1BhamDGZIlYW6ePuxJE'
TWITTER_ACCESS_TOKEN = "832244289556140038-yZ5DltU262Dh4rsSEvdyHqnwLgDhu13"
TWITTER_ACCESS_TOKEN_SECRET = "5hldQV6P5hPRIirMQajVrr7N446OrTqTznpJ6fyciPYUP"
t = Twython(app_key=TWITTER_APP_KEY,
app_secret=TWITTER_APP_KEY_SECRET,
oauth_token=TWITTER_ACCESS_TOKEN,
oauth_token_secret=TWITTER_ACCESS_TOKEN_SECRET)
search = t.search(q='#hacku5', count=10000)
tweets = search['statuses']
followers = t.get_followers_ids(screen_name="DomEnterprises")['ids']
raffle = {}
# adds each user with hashtage #hacku5 to raffle list
for tweet in tweets:
raffle[tweet['user']['id']] = tweet['user']['screen_name']
eligible = {k: v for k, v in raffle.items() if k in followers}
lottery = Raffle(eligible)
def tickets_choose(request):
winner = lottery.generateRaffle()
return JsonResponse({'winner': winner})
app_key = 'xiHlth8bVMiFmxGt741g'
app_secret = 'pnSyIx7awxtB3BRCArdfZ5neebAAokzhmPQgzerqlU'
t = Twython(app_key, app_secret)
auth_props = t.get_authentication_tokens(
callback_url='http://localhost.com/login_twitter.html')
oauth_token = '90926098-ha8yQlvDrtI92odzBcVlBehmxGpZdrOQqLOJAFs4E'
oauth_token_secret = '2n8hPIA9yF6J20G9ZjWAcDLAnKarvSYbYyjV8EDw'
oauth_verifier = ''
try:
t = Twython(app_key, app_secret, oauth_token, oauth_token_secret)
followers = t.get_followers_ids()
for follower_id in followers['ids']:
print follower_id
tweets = t.get_home_timeline()
for tweet in tweets:
print 'Name: %s \n' % (tweet['user']['name'].encode('utf-8'))
print 'Tweet: %s \n' % (tweet['text'].encode('utf-8'))
mentions = t.get_mentions_timeline()
for mention in mentions:
print 'Name: %s \n' % (mention['user']['name'].encode('utf-8'))
print 'Mention: %s \n' % (mention['text'].encode('utf-8'))
class Twitter(object):
""" Class for twitter use """
query_string = '#севастополь -украина OR северная OR катер OR SevastopolBoats OR sevboats :) since:%s lang:ru'
weather_string = u'#погода в Севастополе на %s.'
timelimit = 90
max_follow = 100
def __init__(self, auth=TWITTER_OAUTH_INFO):
self.twitter = Twython(**auth)
def post_tweet(self, message):
""" Writes message into twitter stream """
return self.twitter.update_status(status=message, trim_user=True)
def delete_tweet(self, tweet_id):
""" Remove tweet from timeline """
return self.twitter.destroy_status(id=tweet_id, trim_user=True)
def post_debug(self, message):
""" Writes message into console to test """
print message
return dict(id=0, id_str='0', text=message)
def search(self, query=None, count=10):
""" Search tweet with sevboats ref """
if query is None:
since = (datetime.datetime.now() - datetime.timedelta(days=7)).date().strftime('%Y-%m-%d')
query = self.query_string % since if 'since:%s' in self.query_string else self.query_string
return self.twitter.search(q=query, count=count)
def follow(self, user_id):
""" Follow user by id """
try:
return self.twitter.create_friendship(user_id=user_id, follow=True)
except TwythonError:
return dict(status='error', id_str=None)
def unfollow(self, user_id):
""" Un Follow user by id """
try:
return self.twitter.destroy_friendship(user_id=user_id)
except TwythonError:
return dict(status='error', id_str=None)
def follow_list(self, friends_ids):
""" Follow user in search result """
friends = []
for user_id in friends_ids:
friends.append(self.follow(user_id)['id_str'])
return friends
def unfollow_list(self, friends_ids):
""" Un Follow user in ids list """
friends = []
for user_id in friends_ids:
friends.append(self.unfollow(user_id)['id_str'])
return friends
def search_to_list(self, search_list=None):
""" Follow user in search result """
if search_list is None:
search_list = self.search()
users_ids = []
for tweet in search_list['statuses']:
users_ids.append(tweet['user']['id_str'])
return users_ids
def my_followers(self):
""" Get list of my followers """
followers_ids = []
next_cursor = -1
while next_cursor != '0':
cursor = next_cursor
fids = self.twitter.get_followers_ids(cursor=cursor, stringify_ids=True, count=1000)
followers_ids += fids['ids']
next_cursor = fids['next_cursor_str']
time.sleep(self.timelimit)
return followers_ids
def follow_search(self):
""" Follow for all user in search results """
users_ids = self.search_to_list()
self.follow_list(users_ids)
return users_ids
def follow_followers(self):
""" Follow for all followers """
followers_ids = self.my_followers()
friends_ids = self.i_follow()
users_ids = []
for follower in followers_ids:
if follower not in friends_ids:
users_ids.append(follower)
self.follow_list(users_ids[:self.max_follow])
return users_ids
def i_follow(self):
""" Get list of user i follow what """
friends_ids = []
next_cursor = -1
while next_cursor != '0':
cursor = next_cursor
fids = self.twitter.get_friends_ids(cursor=cursor, stringify_ids=True, count=1000)
friends_ids += fids['ids']
next_cursor = fids['next_cursor_str']
time.sleep(self.timelimit)
return friends_ids
def post_image_weather(self):
""" Create weather image and post to twitter """
# get weather image
print 'start'
weth = Weather()
weather_img, date_title = weth.draw_img()
# create stream
print 'streem'
image_io = StringIO()
# weather_img.save(image_io, format='JPEG', quality=100, subsampling=0, optimize=True, progressive=True)
weather_img.save(image_io, format='PNG')
image_io.seek(0)
# post tweet
print 'post'
return self.twitter.update_status_with_media(media=image_io, status=self.weather_string % date_title)
#Use pip install Twython if you haven't already, this script won't work otherwise.
from twython import Twython, TwythonError
#Get your Key and oAuth tokens from apps.twitter.com/app/new
app_key = '#####'
app_secret = '#####'
oauth_token = '#####'
oauth_token_secret = '#####'
#Change the "Twitter Handler Here" to your username with NO @
#Count is set to 50 by default, Change at your own risk!
twitter = Twython(app_key, app_secret, oauth_token, oauth_token_secret)
mfriends = twitter.get_followers_ids(screen_name='Twitter Handle Here',
count=50)
try:
for unfollo in mfriends['ids']:
twitter.destroy_friendship(user_id=unfollo)
except TwythonError as e:
print(e)
#Author: ItsHex
#License: MIT
#GitHub Repsoitory: https://github.com/ItsHex/Twitter-MassUnfollow
class TwythonAPITestCase(unittest.TestCase):
def setUp(self):
client_args = {
'headers': {
'User-Agent': '__twython__ Test'
},
'allow_redirects': False
}
oauth2_client_args = {
'headers': {
} # This is so we can hit coverage that Twython sets User-Agent for us if none is supplied
}
self.api = Twython(app_key,
app_secret,
oauth_token,
oauth_token_secret,
client_args=client_args)
self.oauth2_api = Twython(app_key,
access_token=access_token,
client_args=oauth2_client_args)
def test_construct_api_url(self):
"""Test constructing a Twitter API url works as we expect"""
url = 'https://api.twitter.com/1.1/search/tweets.json'
constructed_url = self.api.construct_api_url(url, q='#twitter')
self.assertEqual(
constructed_url,
'https://api.twitter.com/1.1/search/tweets.json?q=%23twitter')
def test_get(self):
"""Test Twython generic GET request works"""
self.api.get('account/verify_credentials')
def test_post(self):
"""Test Twython generic POST request works, with a full url and
with just an endpoint"""
update_url = 'https://api.twitter.com/1.1/statuses/update.json'
status = self.api.post(
update_url,
params={'status': 'I love Twython! %s' % int(time.time())})
self.api.post('statuses/destroy/%s' % status['id_str'])
def test_get_lastfunction_header(self):
"""Test getting last specific header of the last API call works"""
self.api.get('statuses/home_timeline')
self.api.get_lastfunction_header('x-rate-limit-remaining')
def test_get_lastfunction_header_not_present(self):
"""Test getting specific header that does not exist from the last call returns None"""
self.api.get('statuses/home_timeline')
header = self.api.get_lastfunction_header('does-not-exist')
self.assertEqual(header, None)
def test_get_lastfunction_header_no_last_api_call(self):
"""Test attempting to get a header when no API call was made raises a TwythonError"""
self.assertRaises(TwythonError, self.api.get_lastfunction_header,
'no-api-call-was-made')
def test_cursor(self):
"""Test looping through the generator results works, at least once that is"""
search = self.api.cursor(self.api.search, q='twitter', count=1)
counter = 0
while counter < 2:
counter += 1
result = next(search)
new_id_str = int(result['id_str'])
if counter == 1:
prev_id_str = new_id_str
time.sleep(
1) # Give time for another tweet to come into search
if counter == 2:
self.assertTrue(new_id_str > prev_id_str)
def test_encode(self):
"""Test encoding UTF-8 works"""
self.api.encode('Twython is awesome!')
def test_html_for_tweet(self):
"""Test HTML for Tweet returns what we want"""
tweet_text = self.api.html_for_tweet(test_tweet_object)
self.assertEqual(test_tweet_html, tweet_text)
def test_html_for_tweet_expanded_url(self):
"""Test using expanded url in HTML for Tweet displays full urls"""
tweet_text = self.api.html_for_tweet(test_tweet_object,
use_expanded_url=True)
# Make sure full url is in HTML
self.assertTrue('http://google.com' in tweet_text)
def test_html_for_tweet_short_url(self):
"""Test using expanded url in HTML for Tweet displays full urls"""
tweet_text = self.api.html_for_tweet(test_tweet_object, False)
# Make sure HTML doesn't contain the display OR exapanded url
self.assertTrue(not 'http://google.com' in tweet_text)
self.assertTrue(not 'google.com' in tweet_text)
def test_raise_error_on_bad_ssl_cert(self):
"""Test TwythonError is raised by a RequestException when an actual HTTP happens"""
self.assertRaises(TwythonError, self.api.get, 'https://example.com')
# Timelines
def test_get_mentions_timeline(self):
"""Test returning mentions timeline for authenticated user succeeds"""
self.api.get_mentions_timeline()
def test_get_user_timeline(self):
"""Test returning timeline for authenticated user and random user
succeeds"""
self.api.get_user_timeline() # Authenticated User Timeline
self.api.get_user_timeline(
screen_name='twitter') # Random User Timeline
def test_get_protected_user_timeline_following(self):
"""Test returning a protected user timeline who you are following
succeeds"""
self.api.get_user_timeline(screen_name=protected_twitter_1)
def test_get_protected_user_timeline_not_following(self):
"""Test returning a protected user timeline who you are not following
fails and raise a TwythonAuthError"""
self.assertRaises(TwythonAuthError,
self.api.get_user_timeline,
screen_name=protected_twitter_2)
def test_retweeted_of_me(self):
"""Test that getting recent tweets by authenticated user that have
been retweeted by others succeeds"""
self.api.retweeted_of_me()
def test_get_home_timeline(self):
"""Test returning home timeline for authenticated user succeeds"""
self.api.get_home_timeline()
# Tweets
def test_get_retweets(self):
"""Test getting retweets of a specific tweet succeeds"""
self.api.get_retweets(id=test_tweet_id)
def test_show_status(self):
"""Test returning a single status details succeeds"""
self.api.show_status(id=test_tweet_id)
def test_update_and_destroy_status(self):
"""Test updating and deleting a status succeeds"""
status = self.api.update_status(
status='Test post just to get deleted :( %s' % int(time.time()))
self.api.destroy_status(id=status['id_str'])
def test_get_oembed_tweet(self):
"""Test getting info to embed tweet on Third Party site succeeds"""
self.api.get_oembed_tweet(id='99530515043983360')
def test_get_retweeters_ids(self):
"""Test getting ids for people who retweeted a tweet succeeds"""
self.api.get_retweeters_ids(id='99530515043983360')
# Search
def test_search(self):
"""Test searching tweets succeeds"""
self.api.search(q='twitter')
# Direct Messages
def test_get_direct_messages(self):
"""Test getting the authenticated users direct messages succeeds"""
self.api.get_direct_messages()
def test_get_sent_messages(self):
"""Test getting the authenticated users direct messages they've
sent succeeds"""
self.api.get_sent_messages()
def test_send_get_and_destroy_direct_message(self):
"""Test sending, getting, then destory a direct message succeeds"""
message = self.api.send_direct_message(screen_name=protected_twitter_1,
text='Hey d00d! %s' %
int(time.time()))
self.api.get_direct_message(id=message['id_str'])
self.api.destroy_direct_message(id=message['id_str'])
def test_send_direct_message_to_non_follower(self):
"""Test sending a direct message to someone who doesn't follow you
fails"""
self.assertRaises(TwythonError,
self.api.send_direct_message,
screen_name=protected_twitter_2,
text='Yo, man! %s' % int(time.time()))
# Friends & Followers
def test_get_user_ids_of_blocked_retweets(self):
"""Test that collection of user_ids that the authenticated user does
not want to receive retweets from succeeds"""
self.api.get_user_ids_of_blocked_retweets(stringify_ids=True)
def test_get_friends_ids(self):
"""Test returning ids of users the authenticated user and then a random
user is following succeeds"""
self.api.get_friends_ids()
self.api.get_friends_ids(screen_name='twitter')
def test_get_followers_ids(self):
"""Test returning ids of users the authenticated user and then a random
user are followed by succeeds"""
self.api.get_followers_ids()
self.api.get_followers_ids(screen_name='twitter')
def test_lookup_friendships(self):
"""Test returning relationships of the authenticating user to the
comma-separated list of up to 100 screen_names or user_ids provided
succeeds"""
self.api.lookup_friendships(screen_name='twitter,ryanmcgrath')
def test_get_incoming_friendship_ids(self):
"""Test returning incoming friendship ids succeeds"""
self.api.get_incoming_friendship_ids()
def test_get_outgoing_friendship_ids(self):
"""Test returning outgoing friendship ids succeeds"""
self.api.get_outgoing_friendship_ids()
def test_create_friendship(self):
"""Test creating a friendship succeeds"""
self.api.create_friendship(screen_name='justinbieber')
def test_destroy_friendship(self):
"""Test destroying a friendship succeeds"""
self.api.destroy_friendship(screen_name='justinbieber')
def test_update_friendship(self):
"""Test updating friendships succeeds"""
self.api.update_friendship(screen_name=protected_twitter_1,
retweets='true')
self.api.update_friendship(screen_name=protected_twitter_1,
retweets=False)
def test_show_friendships(self):
"""Test showing specific friendship succeeds"""
self.api.show_friendship(target_screen_name=protected_twitter_1)
def test_get_friends_list(self):
"""Test getting list of users authenticated user then random user is
following succeeds"""
self.api.get_friends_list()
self.api.get_friends_list(screen_name='twitter')
def test_get_followers_list(self):
"""Test getting list of users authenticated user then random user are
followed by succeeds"""
self.api.get_followers_list()
self.api.get_followers_list(screen_name='twitter')
# Users
def test_get_account_settings(self):
"""Test getting the authenticated user account settings succeeds"""
self.api.get_account_settings()
def test_verify_credentials(self):
"""Test representation of the authenticated user call succeeds"""
self.api.verify_credentials()
def test_update_account_settings(self):
"""Test updating a user account settings succeeds"""
self.api.update_account_settings(lang='en')
def test_update_delivery_service(self):
"""Test updating delivery settings fails because we don't have
a mobile number on the account"""
self.assertRaises(TwythonError,
self.api.update_delivery_service,
device='none')
def test_update_profile(self):
"""Test updating profile succeeds"""
self.api.update_profile(include_entities='true')
def test_update_profile_colors(self):
"""Test updating profile colors succeeds"""
self.api.update_profile_colors(profile_background_color='3D3D3D')
def test_list_blocks(self):
"""Test listing users who are blocked by the authenticated user
succeeds"""
self.api.list_blocks()
def test_list_block_ids(self):
"""Test listing user ids who are blocked by the authenticated user
succeeds"""
self.api.list_block_ids()
def test_create_block(self):
"""Test blocking a user succeeds"""
self.api.create_block(screen_name='justinbieber')
def test_destroy_block(self):
"""Test unblocking a user succeeds"""
self.api.destroy_block(screen_name='justinbieber')
def test_lookup_user(self):
"""Test listing a number of user objects succeeds"""
self.api.lookup_user(screen_name='twitter,justinbieber')
def test_show_user(self):
"""Test showing one user works"""
self.api.show_user(screen_name='twitter')
def test_search_users(self):
"""Test that searching for users succeeds"""
self.api.search_users(q='Twitter API')
def test_get_contributees(self):
"""Test returning list of accounts the specified user can
contribute to succeeds"""
self.api.get_contributees(screen_name='TechCrunch')
def test_get_contributors(self):
"""Test returning list of accounts that contribute to the
authenticated user fails because we are not a Contributor account"""
self.assertRaises(TwythonError,
self.api.get_contributors,
screen_name=screen_name)
def test_remove_profile_banner(self):
"""Test removing profile banner succeeds"""
self.api.remove_profile_banner()
def test_get_profile_banner_sizes(self):
"""Test getting list of profile banner sizes fails because
we have not uploaded a profile banner"""
self.assertRaises(TwythonError, self.api.get_profile_banner_sizes)
# Suggested Users
def test_get_user_suggestions_by_slug(self):
"""Test getting user suggestions by slug succeeds"""
self.api.get_user_suggestions_by_slug(slug='twitter')
def test_get_user_suggestions(self):
"""Test getting user suggestions succeeds"""
self.api.get_user_suggestions()
def test_get_user_suggestions_statuses_by_slug(self):
"""Test getting status of suggested users succeeds"""
self.api.get_user_suggestions_statuses_by_slug(slug='funny')
# Favorites
def test_get_favorites(self):
"""Test getting list of favorites for the authenticated
user succeeds"""
self.api.get_favorites()
def test_create_and_destroy_favorite(self):
"""Test creating and destroying a favorite on a tweet succeeds"""
self.api.create_favorite(id=test_tweet_id)
self.api.destroy_favorite(id=test_tweet_id)
# Lists
def test_show_lists(self):
"""Test show lists for specified user"""
self.api.show_lists(screen_name='twitter')
def test_get_list_statuses(self):
"""Test timeline of tweets authored by members of the
specified list succeeds"""
self.api.get_list_statuses(
slug=test_list_slug, owner_screen_name=test_list_owner_screen_name)
def test_create_update_destroy_list_add_remove_list_members(self):
"""Test create a list, adding and removing members then
deleting the list succeeds"""
the_list = self.api.create_list(name='Stuff %s' % int(time.time()))
list_id = the_list['id_str']
self.api.update_list(list_id=list_id,
name='Stuff Renamed %s' % int(time.time()))
screen_names = ['johncena', 'xbox']
# Multi add/delete members
self.api.create_list_members(list_id=list_id, screen_name=screen_names)
self.api.delete_list_members(list_id=list_id, screen_name=screen_names)
# Single add/delete member
self.api.add_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list_member(list_id=list_id,
screen_name='justinbieber')
self.api.delete_list(list_id=list_id)
def test_get_list_memberships(self):
"""Test list of memberhips the authenticated user succeeds"""
self.api.get_list_memberships()
def test_get_list_subscribers(self):
"""Test list of subscribers of a specific list succeeds"""
self.api.get_list_subscribers(
slug=test_list_slug, owner_screen_name=test_list_owner_screen_name)
def test_subscribe_is_subbed_and_unsubscribe_to_list(self):
"""Test subscribing, is a list sub and unsubbing to list succeeds"""
self.api.subscribe_to_list(
slug=test_list_slug, owner_screen_name=test_list_owner_screen_name)
# Returns 404 if user is not a subscriber
self.api.is_list_subscriber(
slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name,
screen_name=screen_name)
self.api.unsubscribe_from_list(
slug=test_list_slug, owner_screen_name=test_list_owner_screen_name)
def test_is_list_member(self):
"""Test returning if specified user is member of a list succeeds"""
# Returns 404 if not list member
self.api.is_list_member(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name,
screen_name='themattharris')
def test_get_list_members(self):
"""Test listing members of the specified list succeeds"""
self.api.get_list_members(
slug=test_list_slug, owner_screen_name=test_list_owner_screen_name)
def test_get_specific_list(self):
"""Test getting specific list succeeds"""
self.api.get_specific_list(
slug=test_list_slug, owner_screen_name=test_list_owner_screen_name)
def test_get_list_subscriptions(self):
"""Test collection of the lists the specified user is
subscribed to succeeds"""
self.api.get_list_subscriptions(screen_name='twitter')
def test_show_owned_lists(self):
"""Test collection of lists the specified user owns succeeds"""
self.api.show_owned_lists(screen_name='twitter')
# Saved Searches
def test_get_saved_searches(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
self.api.get_saved_searches()
def test_create_get_destroy_saved_search(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
saved_search = self.api.create_saved_search(query='#Twitter')
saved_search_id = saved_search['id_str']
self.api.show_saved_search(id=saved_search_id)
self.api.destroy_saved_search(id=saved_search_id)
# Places & Geo
def test_get_geo_info(self):
"""Test getting info about a geo location succeeds"""
self.api.get_geo_info(place_id='df51dec6f4ee2b2c')
def test_reverse_geo_code(self):
"""Test reversing geocode succeeds"""
self.api.reverse_geocode(lat='37.76893497', long='-122.42284884')
def test_search_geo(self):
"""Test search for places that can be attached
to a statuses/update succeeds"""
self.api.search_geo(query='Toronto')
def test_get_similar_places(self):
"""Test locates places near the given coordinates which
are similar in name succeeds"""
self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ')
# Trends
def test_get_place_trends(self):
"""Test getting the top 10 trending topics for a specific
WOEID succeeds"""
self.api.get_place_trends(id=1)
def test_get_available_trends(self):
"""Test returning locations that Twitter has trending
topic information for succeeds"""
self.api.get_available_trends()
def test_get_closest_trends(self):
"""Test getting the locations that Twitter has trending topic
information for, closest to a specified location succeeds"""
self.api.get_closest_trends(lat='37', long='-122')
# Help
def test_get_twitter_configuration(self):
"""Test getting Twitter's configuration succeeds"""
self.api.get_twitter_configuration()
def test_get_supported_languages(self):
"""Test getting languages supported by Twitter succeeds"""
self.api.get_supported_languages()
def test_privacy_policy(self):
"""Test getting Twitter's Privacy Policy succeeds"""
self.api.get_privacy_policy()
def test_get_tos(self):
"""Test getting the Twitter Terms of Service succeeds"""
self.api.get_tos()
def test_get_application_rate_limit_status(self):
"""Test getting application rate limit status succeeds"""
self.oauth2_api.get_application_rate_limit_status()
tw = settings['twitter']
twitter = Twython(tw['APP_KEY'], tw['APP_SECRET'], tw['OAUTH_TOKEN'], tw['OAUTH_TOKEN_SECRET'])
status = OrderedDict()
if os.path.isfile(settings['update_status_path']):
f = open(settings['update_status_path'], 'rb')
status = cPickle.load(f)
f.close()
if not status:
status = OrderedDict([(k, None) for k in settings['only_followers_of']])
try:
for account, next_cursor in status.iteritems():
while next_cursor != '0':
res = twitter.get_followers_ids(screen_name=account, stringify_ids=True, cursor=next_cursor)
for user_id in res['ids']:
c.execute("""INSERT OR IGNORE INTO followers(id) VALUES ('%s')""" % user_id)
conn.commit()
next_cursor = res['next_cursor_str']
status[account] = next_cursor
except:
pass
status = OrderedDict([item for item in status.iteritems() if item[1] != '0'])
f = open(settings['update_status_path'], 'wb')
cPickle.dump(status, f, cPickle.HIGHEST_PROTOCOL)
f.close()
conn.close()
class TwitterBot():
'''
Crawls twitter and updates social network and tagged resources to repository
'''
def __init__(self):
self.service = TWITTER
APP_KEY = "w7zjbW1qHtzX2GxbrHbeNgyPr"
APP_SECRET = "qJFawkFxwYZX6U3DYNvJI3czWEjix2owU9wTXZ9BorMkhFU4Yb"
self.twapi = Twython(APP_KEY, APP_SECRET)
self.userBaseUri = 'http://twitter.com/%s' # % user screen_name
self.tagBaseUri = 'http://twitter.com/#search?q=#%s' # % trend key word
self.tweetBaseUri = 'http://twitter.com/%s/status/%s' # % (user screen_name, status id)
@staticmethod
def getname():
return self.service
@staticmethod
def getDefaultAvatar():
return defAvatar
def setStartUserId(self, start_user_id):
self.start_user_id = start_user_id
def setMaxLevel(self, max_level):
self.max_level = max_level
def setLastAdded(self, last_added):
self.last_added = last_added
def updateDatabase(self, user_metadata, user_ntw, udate=None):
raise NotImplementedError()
def fetchUser(self, username):
return (None, {'knows': [], 'fans': []})
def getUserUri(self, username):
return self.userBaseUri % username
@staticmethod
def factory(user, params=None, depth=2, last_added=None, verbose=False):
# Twitter crawler
twcrawl = TwitterBot()
twcrawl.setStartUserId(user)
twcrawl.setMaxLevel(depth)
twcrawl.setLastAdded(last_added)
return twcrawl
def updateDatabase(self, user_metadata, user_ntw, udate=None):
username = user_metadata['username']
userUri = self.userBaseUri % username
resourceUri = None
added = 0
# add user to repository
dbaccess.User.addToDB(userUri,
self.service,
username,
TwitterBot.getDefaultAvatar(),
udate=udate)
# add resources, tags and relations between them to repository
for resource in user_metadata['data']:
# add tweet resource to repository
resourceId = resource['id']
resourceTitle = resource['text']
resourceUri = self.tweetBaseUri % (username, resourceId)
dbaccess.Tweet.addToDB(resourceUri, resourceTitle, udate=udate)
added += 1
# add tag(trend) relations to repository
for tag in self.parseTags(resourceTitle):
tagUri = self.tagBaseUri % urllib.quote(tag.encode('utf-8'))
# add tag to repository
dbaccess.Tag.addToDB(tagUri, tag, udate=udate)
# add user, resource, tag relation to repository
dbaccess.UserResourceTag.addToDB(userUri,
resourceUri,
tagUri,
udate=udate)
# add url relations between the tweet and urls contained
urls = map(self.getRealUrl, self.parseUrls(resourceTitle))
for url in urls:
dbaccess.Bookmark.addToDB(url, None)
dbaccess.Tweet.addReferenceToDB(resourceUri, url)
# add user social network relations to repository
# add friends
for u in user_ntw['knows']:
knownUserUri = self.userBaseUri % u
dbaccess.User.addToDB(knownUserUri,
self.service,
u,
TwitterBot.getDefaultAvatar(),
udate=udate)
dbaccess.User.addToDBUserKnowsUser(userUri,
knownUserUri,
udate=udate)
# add followers
for u in user_ntw['fans']:
otherUserUri = self.userBaseUri % u
dbaccess.User.addToDB(otherUserUri,
self.service,
u,
TwitterBot.getDefaultAvatar(),
udate=udate)
dbaccess.User.addToDBUserKnowsUser(otherUserUri,
userUri,
udate=udate)
# return number of added resources and last added resource uri
return added
def fetchUserMetadata(self, username):
# get user tweets (only last 20)
data = self.twapi.get_user_timeline(screen_name=username)
return {'username': username, 'data': data}
def fetchUserNetwork(self, username):
chunk_size = 100 # max returned by bulkUserLookup
# Get Friends
fids, cursor = [], -1
fchunks = [
fids[i:i + chunk_size] for i in range(0, len(fids), chunk_size)
]
while True:
f = self.twapi.get_friends_ids(screen_name=username, cursor=cursor)
fids.extend(f['ids'])
if f['next_cursor'] == cursor or len(f['ids']) == 0:
break
cursor = f['next_cursor']
# get screen names from ids in chunks of 100
screen_names = []
for chunk in fchunks:
screen_names = []
screen_names.extend([
userdata['screen_name']
for userdata in self.twapi.lookup_user(user_id=chunk)
])
ntw_knows = screen_names
print ntw_knows
# Get Followers
fids, cursor = [], -1
while True:
f = self.twapi.get_followers_ids(screen_name=username,
cursor=cursor)
fids.extend(f['ids'])
if f['next_cursor'] == cursor or len(f['ids']) == 0:
break
cursor = f['next_cursor']
fchunks = [
fids[i:i + chunk_size] for i in range(0, len(fids), chunk_size)
]
fan_screen_names = []
for chunk in fchunks:
# get all data for all user ids
fan_screen_names.extend([
userdata['screen_name']
for userdata in self.twapi.lookup_user(user_id=chunk)
])
ntw_fans = fan_screen_names
return {'knows': ntw_knows, 'fans': ntw_fans}
def fetchUser(self, username):
return (self.fetchUserMetadata(username),
self.fetchUserNetwork(username))
def parseTags(self, text):
'''
Parses trends from tweet: '#trend #tre#nd ## #trend,two #tren%^& smth,#da #1dc #100 #1dc$ #100^',
finds trends: trend, tre, trend, tren, da, 1dc, 100, 1dc, 100.
Text parameter MUST be UNICODE encoded: u'some text'
'''
return re.findall('(?<=[\W^#]#)\w+', ' ' + text, re.UNICODE)
def parseUrls(self, text):
'''Returns a list of urls from the text.
'''
return re.findall('http://\S+', text)
def getRealUrl(self, url):
'''Returns the redirected url if *url* parameter is a shortened url (a redirect was produced),
or itself if not.
'''
try:
u = urllib2.urlopen(url)
realurl = u.geturl()
if realurl == None:
raise urllib2.HTTPError('Could not fetch url')
except:
realurl = url
return realurl
def crawlUserNetwork(self, user_id=None, max_level=0, start_time=None):
'''
user_id - an id to uniquely identify a user (can be a username or userid)
'''
user_id = user_id or self.start_user_id
max_level = max_level or self.max_level
queue = [user_id] # init queue
visited = {user_id: 0} # init visited nodes
added = 0 # set number of added resources
while queue:
v = queue.pop(0) # pop next user node
level = visited[v] # get their level
# get user metadata and social network
user_metadata, user_ntw = self.fetchUser(v)
# update database with user data, tagged resources and social network relations
a = self.updateDatabase(user_metadata, user_ntw, udate=start_time)
added += a
# explore following nodes
for w in user_ntw['knows']:
if w not in visited and level < max_level:
queue.append(w)
visited[w] = level + 1
return added
TWITTER_ACCESS_TOKEN = "832244289556140038-yZ5DltU262Dh4rsSEvdyHqnwLgDhu13"
TWITTER_ACCESS_TOKEN_SECRET = "5hldQV6P5hPRIirMQajVrr7N446OrTqTznpJ6fyciPYUP"
t = Twython(app_key=TWITTER_APP_KEY,
app_secret=TWITTER_APP_KEY_SECRET,
oauth_token=TWITTER_ACCESS_TOKEN,
oauth_token_secret=TWITTER_ACCESS_TOKEN_SECRET)
#hashtag = request.GET.get("searchTxt", "")
search = t.search(q='#hacku5', count=100)
tweets = search['statuses']
raffle = {}
followers = t.get_followers_ids(screen_name="DomEnterprises",
stringify_ids=true)['ids']
# adds each user with hashtage #hacku5 to raffle list
for tweet in tweets:
raffle[tweet['id_str']] = tweet['user']['screen_name']
# users that tweeted the hashtag but are not following @DomEnterprises
waitingList = {}
# get followers of @DomEnterprises
# for each user in the raffle but not following @DomEnterprises add them to waiting list
for userID in raffle:
if userID not in followers:
waitingList[userID] = raffle[userID]
#generateResponseTweet("Follow DomEnterprises on Twitter to get a Raffle Ticket")
def createDatabase(request):
twitter = Twython(
settings.TWITTER_APP_KEY,
settings.TWITTER_APP_SECRET,
request.session['oauth_token'],
request.session['oauth_token_secret']
)
rels = {}
current_userid = request.session['oauth_token'].split('-')[0];
m = i = 5
mid = m/2
usersToQuery = [current_userid]
users = []
current_userdeets = twitter.verify_credentials()
users.append(current_userdeets)
while i > 0:
userid = usersToQuery[m-i]
print "Index ", str(i)
print "Querying userId ", str(userid)
followers = twitter.get_followers_ids(count=100, user_id=userid)['ids']
for f in followers:
if not f in rels:
rels[f] = [userid]
elif userid not in rels[f]:
rels[f].append(userid)
if len(followers) > 0:
users = users + twitter.lookup_user(user_id=",".join(map(str, followers)))
friends = twitter.get_friends_ids(count=100, user_id=userid)['ids']
for f in friends:
if not userid in rels:
rels[userid] = [f]
elif f not in rels[userid]:
rels[userid].append(f)
if len(friends) > 0:
users = users + twitter.lookup_user(user_id=",".join(map(str, friends)))
if i == m:
usersToQuery = usersToQuery + followers[0:mid] + friends[0:mid]
i = i-1
fields = ['screen_name', 'name', 'location', 'created_at', 'id', 'favourites_count', 'url', 'lang', 'followers_count', 'friends_count', 'statuses_count', 'profile_image_url']
allUsers = []
usersIndex = {}
for u in users:
if u['id'] in usersIndex:
continue
tu = {}
for f in fields:
tu[f] = u[f]
allUsers.append(tu)
usersIndex[u['id']] = 1
# print "writing.."
# save
# with io.open(os.path.dirname(__file__) + '/users.txt', 'w', encoding='utf-8') as f:
# f.write(unicode(json.dumps(allUsers, ensure_ascii=False)))
# with io.open(os.path.dirname(__file__) + '/rels.txt', 'w', encoding='utf-8') as f:
# f.write(unicode(json.dumps(rels, ensure_ascii=False)))
print 'Writing to neo4j..'
graph_db = getPy2neoService()
graph_db.clear()
batch = neo4j.WriteBatch(graph_db)
batches = {}
for u in allUsers:
batches[str(u['id'])] = batch.create(node(**u))
for r in rels:
for p in rels[r]:
if str(r) in batches and str(p) in batches:
batch.create(rel(batches[str(r)], "FOLLOWS", batches[str(p)]))
else:
print "Skipping.."
results = batch.submit()
print 'Writing done..'
return current_userdeets['screen_name']
class TwythonAPITestCase(unittest.TestCase):
def setUp(self):
self.api = Twython(app_key, app_secret,
oauth_token, oauth_token_secret)
# Timelines
def test_get_mentions_timeline(self):
'''Test returning mentions timeline for authenticated user succeeds'''
self.api.get_mentions_timeline()
def test_get_user_timeline(self):
'''Test returning timeline for authenticated user and random user
succeeds'''
self.api.get_user_timeline() # Authenticated User Timeline
self.api.get_user_timeline(screen_name='twitter') # Random User Timeline
def test_get_protected_user_timeline_following(self):
'''Test returning a protected user timeline who you are following
succeeds'''
self.api.get_user_timeline(screen_name=protected_twitter_1)
def test_get_protected_user_timeline_not_following(self):
'''Test returning a protected user timeline who you are not following
fails and raise a TwythonAuthError'''
self.assertRaises(TwythonAuthError, self.api.get_user_timeline,
screen_name=protected_twitter_2)
def test_get_home_timeline(self):
'''Test returning home timeline for authenticated user succeeds'''
self.api.get_home_timeline()
# Tweets
def test_get_retweets(self):
'''Test getting retweets of a specific tweet succeeds'''
self.api.get_retweets(id=test_tweet_id)
def test_show_status(self):
'''Test returning a single status details succeeds'''
self.api.show_status(id=test_tweet_id)
def test_update_and_destroy_status(self):
'''Test updating and deleting a status succeeds'''
status = self.api.update_status(status='Test post just to get deleted :(')
self.api.destroy_status(id=status['id_str'])
def test_retweet(self):
'''Test retweeting a status succeeds'''
retweet = self.api.retweet(id='99530515043983360')
self.api.destroy_status(id=retweet['id_str'])
def test_retweet_twice(self):
'''Test that trying to retweet a tweet twice raises a TwythonError'''
retweet = self.api.retweet(id='99530515043983360')
self.assertRaises(TwythonError, self.api.retweet,
id='99530515043983360')
# Then clean up
self.api.destroy_status(id=retweet['id_str'])
def test_get_oembed_tweet(self):
'''Test getting info to embed tweet on Third Party site succeeds'''
self.api.get_oembed_tweet(id='99530515043983360')
def test_get_retweeters_ids(self):
'''Test getting ids for people who retweeted a tweet succeeds'''
self.api.get_retweeters_ids(id='99530515043983360')
# Search
def test_search(self):
'''Test searching tweets succeeds'''
self.api.search(q='twitter')
# Direct Messages
def test_get_direct_messages(self):
'''Test getting the authenticated users direct messages succeeds'''
self.api.get_direct_messages()
def test_get_sent_messages(self):
'''Test getting the authenticated users direct messages they've
sent succeeds'''
self.api.get_sent_messages()
def test_send_get_and_destroy_direct_message(self):
'''Test sending, getting, then destory a direct message succeeds'''
message = self.api.send_direct_message(screen_name=protected_twitter_1,
text='Hey d00d!')
self.api.get_direct_message(id=message['id_str'])
self.api.destroy_direct_message(id=message['id_str'])
def test_send_direct_message_to_non_follower(self):
'''Test sending a direct message to someone who doesn't follow you
fails'''
self.assertRaises(TwythonError, self.api.send_direct_message,
screen_name=protected_twitter_2, text='Yo, man!')
# Friends & Followers
def test_get_user_ids_of_blocked_retweets(self):
'''Test that collection of user_ids that the authenticated user does
not want to receive retweets from succeeds'''
self.api.get_user_ids_of_blocked_retweets(stringify_ids='true')
def test_get_friends_ids(self):
'''Test returning ids of users the authenticated user and then a random
user is following succeeds'''
self.api.get_friends_ids()
self.api.get_friends_ids(screen_name='twitter')
def test_get_followers_ids(self):
'''Test returning ids of users the authenticated user and then a random
user are followed by succeeds'''
self.api.get_followers_ids()
self.api.get_followers_ids(screen_name='twitter')
def test_lookup_friendships(self):
'''Test returning relationships of the authenticating user to the
comma-separated list of up to 100 screen_names or user_ids provided
succeeds'''
self.api.lookup_friendships(screen_name='twitter,ryanmcgrath')
def test_get_incoming_friendship_ids(self):
'''Test returning incoming friendship ids succeeds'''
self.api.get_incoming_friendship_ids()
def test_get_outgoing_friendship_ids(self):
'''Test returning outgoing friendship ids succeeds'''
self.api.get_outgoing_friendship_ids()
def test_create_friendship(self):
'''Test creating a friendship succeeds'''
self.api.create_friendship(screen_name='justinbieber')
def test_destroy_friendship(self):
'''Test destroying a friendship succeeds'''
self.api.destroy_friendship(screen_name='justinbieber')
def test_update_friendship(self):
'''Test updating friendships succeeds'''
self.api.update_friendship(screen_name=protected_twitter_1,
retweets='true')
self.api.update_friendship(screen_name=protected_twitter_1,
retweets='false')
def test_show_friendships(self):
'''Test showing specific friendship succeeds'''
self.api.show_friendship(target_screen_name=protected_twitter_1)
def test_get_friends_list(self):
'''Test getting list of users authenticated user then random user is
following succeeds'''
self.api.get_friends_list()
self.api.get_friends_list(screen_name='twitter')
def test_get_followers_list(self):
'''Test getting list of users authenticated user then random user are
followed by succeeds'''
self.api.get_followers_list()
self.api.get_followers_list(screen_name='twitter')
# Users
def test_get_account_settings(self):
'''Test getting the authenticated user account settings succeeds'''
self.api.get_account_settings()
def test_verify_credentials(self):
'''Test representation of the authenticated user call succeeds'''
self.api.verify_credentials()
def test_update_account_settings(self):
'''Test updating a user account settings succeeds'''
self.api.update_account_settings(lang='en')
def test_update_delivery_service(self):
'''Test updating delivery settings fails because we don't have
a mobile number on the account'''
self.assertRaises(TwythonError, self.api.update_delivery_service,
device='none')
def test_update_profile(self):
'''Test updating profile succeeds'''
self.api.update_profile(include_entities='true')
def test_update_profile_colors(self):
'''Test updating profile colors succeeds'''
self.api.update_profile_colors(profile_background_color='3D3D3D')
def test_list_blocks(self):
'''Test listing users who are blocked by the authenticated user
succeeds'''
self.api.list_blocks()
def test_list_block_ids(self):
'''Test listing user ids who are blocked by the authenticated user
succeeds'''
self.api.list_block_ids()
def test_create_block(self):
'''Test blocking a user succeeds'''
self.api.create_block(screen_name='justinbieber')
def test_destroy_block(self):
'''Test unblocking a user succeeds'''
self.api.destroy_block(screen_name='justinbieber')
def test_lookup_user(self):
'''Test listing a number of user objects succeeds'''
self.api.lookup_user(screen_name='twitter,justinbieber')
def test_show_user(self):
'''Test showing one user works'''
self.api.show_user(screen_name='twitter')
def test_search_users(self):
'''Test that searching for users succeeds'''
self.api.search_users(q='Twitter API')
def test_get_contributees(self):
'''Test returning list of accounts the specified user can
contribute to succeeds'''
self.api.get_contributees(screen_name='TechCrunch')
def test_get_contributors(self):
'''Test returning list of accounts that contribute to the
authenticated user fails because we are not a Contributor account'''
self.assertRaises(TwythonError, self.api.get_contributors,
screen_name=screen_name)
def test_remove_profile_banner(self):
'''Test removing profile banner succeeds'''
self.api.remove_profile_banner()
def test_get_profile_banner_sizes(self):
'''Test getting list of profile banner sizes fails because
we have not uploaded a profile banner'''
self.assertRaises(TwythonError, self.api.get_profile_banner_sizes)
# Suggested Users
def test_get_user_suggestions_by_slug(self):
'''Test getting user suggestions by slug succeeds'''
self.api.get_user_suggestions_by_slug(slug='twitter')
def test_get_user_suggestions(self):
'''Test getting user suggestions succeeds'''
self.api.get_user_suggestions()
def test_get_user_suggestions_statuses_by_slug(self):
'''Test getting status of suggested users succeeds'''
self.api.get_user_suggestions_statuses_by_slug(slug='funny')
# Favorites
def test_get_favorites(self):
'''Test getting list of favorites for the authenticated
user succeeds'''
self.api.get_favorites()
def test_create_and_destroy_favorite(self):
'''Test creating and destroying a favorite on a tweet succeeds'''
self.api.create_favorite(id=test_tweet_id)
self.api.destroy_favorite(id=test_tweet_id)
# Lists
def test_show_lists(self):
'''Test show lists for specified user'''
self.api.show_lists(screen_name='twitter')
def test_get_list_statuses(self):
'''Test timeline of tweets authored by members of the
specified list succeeds'''
self.api.get_list_statuses(list_id=test_list_id)
def test_create_update_destroy_list_add_remove_list_members(self):
'''Test create a list, adding and removing members then
deleting the list succeeds'''
the_list = self.api.create_list(name='Stuff')
list_id = the_list['id_str']
self.api.update_list(list_id=list_id, name='Stuff Renamed')
# Multi add/delete members
self.api.create_list_members(list_id=list_id,
screen_name='johncena,xbox')
self.api.delete_list_members(list_id=list_id,
screen_name='johncena,xbox')
# Single add/delete member
self.api.add_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list(list_id=list_id)
def test_get_list_memberships(self):
'''Test list of lists the authenticated user is a member of succeeds'''
self.api.get_list_memberships()
def test_get_list_subscribers(self):
'''Test list of subscribers of a specific list succeeds'''
self.api.get_list_subscribers(list_id=test_list_id)
def test_subscribe_is_subbed_and_unsubscribe_to_list(self):
'''Test subscribing, is a list sub and unsubbing to list succeeds'''
self.api.subscribe_to_list(list_id=test_list_id)
# Returns 404 if user is not a subscriber
self.api.is_list_subscriber(list_id=test_list_id,
screen_name=screen_name)
self.api.unsubscribe_from_list(list_id=test_list_id)
def test_is_list_member(self):
'''Test returning if specified user is member of a list succeeds'''
# Returns 404 if not list member
self.api.is_list_member(list_id=test_list_id, screen_name='jack')
def test_get_list_members(self):
'''Test listing members of the specified list succeeds'''
self.api.get_list_members(list_id=test_list_id)
def test_get_specific_list(self):
'''Test getting specific list succeeds'''
self.api.get_specific_list(list_id=test_list_id)
def test_get_list_subscriptions(self):
'''Test collection of the lists the specified user is
subscribed to succeeds'''
self.api.get_list_subscriptions(screen_name='twitter')
def test_show_owned_lists(self):
'''Test collection of lists the specified user owns succeeds'''
self.api.show_owned_lists(screen_name='twitter')
# Saved Searches
def test_get_saved_searches(self):
'''Test getting list of saved searches for authenticated
user succeeds'''
self.api.get_saved_searches()
def test_create_get_destroy_saved_search(self):
'''Test getting list of saved searches for authenticated
user succeeds'''
saved_search = self.api.create_saved_search(query='#Twitter')
saved_search_id = saved_search['id_str']
self.api.show_saved_search(id=saved_search_id)
self.api.destroy_saved_search(id=saved_search_id)
# Places & Geo
def test_get_geo_info(self):
'''Test getting info about a geo location succeeds'''
self.api.get_geo_info(place_id='df51dec6f4ee2b2c')
def test_reverse_geo_code(self):
'''Test reversing geocode succeeds'''
self.api.reverse_geocode(lat='37.76893497', long='-122.42284884')
def test_search_geo(self):
'''Test search for places that can be attached
to a statuses/update succeeds'''
self.api.search_geo(query='Toronto')
def test_get_similar_places(self):
'''Test locates places near the given coordinates which
are similar in name succeeds'''
self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ')
# Trends
def test_get_place_trends(self):
'''Test getting the top 10 trending topics for a specific
WOEID succeeds'''
self.api.get_place_trends(id=1)
def test_get_available_trends(self):
'''Test returning locations that Twitter has trending
topic information for succeeds'''
self.api.get_available_trends()
def test_get_closest_trends(self):
'''Test getting the locations that Twitter has trending topic
information for, closest to a specified location succeeds'''
self.api.get_closest_trends(lat='37', long='-122')
rels = {}
current_userid = oauth_token.split('-')[0];
m = i = 5
mid = m/2
usersToQuery = [current_userid]
users = []
while i > 0:
userid = usersToQuery[m-i]
print "Index ", str(i)
print "Querying userId ", str(userid)
followers = twitter.get_followers_ids(count=100, user_id=userid)['ids']
t = time.time()
for f in followers:
if not f in rels:
rels[f] = [userid]
elif userid not in rels[f]:
rels[f].append(userid)
print time.time() - t
users = users + twitter.lookup_user(user_id=",".join(map(str, followers)))
friends = twitter.get_friends_ids(count=100, user_id=userid)['ids']
t = time.time()
for f in friends:
if not userid in rels:
def addUserToDB(sn):
APP_KEY = "UHz2Yr3qedZeNZxblgzqVbER6"
APP_SECRET = "KavH3dnHPOPybPWBWXAWD7jAqfJRxDLqEK24N1m64O2fax0vM0"
twitter = Twython(APP_KEY, APP_SECRET, oauth_version=2)
ACCESS_TOKEN = twitter.obtain_access_token()
twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN)
analyzer = SentimentIntensityAnalyzer()
totalSentimentScore = 0
tweetCount = 0
compoundText = ""
tweets = []
try:
user_timeline = twitter.get_user_timeline(screen_name=sn, count=30)
except TwythonError as e:
print(e)
return False
for tweet in user_timeline:
tweetCount += 1
sentiment_dict = analyzer.polarity_scores(tweet['text'])
totalSentimentScore += sentiment_dict['compound']
compoundText = compoundText + " " + tweet['text']
tweets.append(tweet)
syntax_dict = count_by_word(compoundText)
averageSentiment = 0
if tweetCount != 0:
averageSentiment = totalSentimentScore / tweetCount
## FOLLOWERS ##
followers = []
try:
user_followers = twitter.get_followers_ids(screen_name=sn)
followers = user_followers
except TwythonError as e:
print(e)
return False
following = []
try:
user_friends = twitter.get_friends_ids(screen_name=sn)
following = user_friends
except TwythonError as e:
print(e)
return False
lastUpdated = datetime.datetime.utcnow().isoformat()
syntax_dict = count_by_word(compoundText)
syntax_dict = list(filter(lambda x: x[1] != 1, syntax_dict))
userStore = {
"_id": sn,
"last_updated": lastUpdated,
"avg_sentiment": averageSentiment,
"syntax_dict": syntax_dict[:30],
"following": following['ids'],
"followers": followers['ids'],
"tweets": tweets
}
client = pymongo.MongoClient(
"mongodb+srv://tgabrie2:[email protected]/cs432?retryWrites=true&w=majority",
ssl_cert_reqs=ssl.CERT_NONE)
db = client.cs432
collection = db['test']
collection.insert_one(userStore)
return True
from twython import Twython
import pandas as pd
import pprint
APP_KEY = 's5zNSchQABtfEe683ZL5bgSW8'
APP_SECRET = 'dyQBARmUVVudDD56banM6OaZyOkwylkq1YO2DvbL0XnoR62sQ5'
twitter = Twython(APP_KEY, APP_SECRET, oauth_version=2)
ACCESS_TOKEN = twitter.obtain_access_token()
twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN)
D_Accounts = ('maddow','TheDemocrats','DWStweets','NancyPelosi','SenatorReid','ThePlumLineGS','dscc','dccc')
D_Twitter = pd.DataFrame({'id':00000}, index=[0])
for sn in (D_Accounts):
followers = twitter.get_followers_ids(screen_name=sn, count=1000)
followers_list = pd.DataFrame(followers['ids'])
followers_list.columns = ['id']
followers_list[sn] = True
D_Twitter = pd.merge(D_Twitter, followers_list, on='id', how='outer')
D_Twitter_indexed = D_Twitter.set_index(['id'])
D_Twitter_indexed.to_csv('Dem_Twitter1000.csv')
# Initialize the database followers record for that screen_name
res = db.followers.find_one({"screen_name": screen_name})
if res is None:
db.followers.insert_one({"screen_name": screen_name, "ids": []})
# cursor is used to navigate a twitter collection
# https://dev.twitter.com/overview/api/cursoring
next_cursor = -1
follower_ids = list()
ids_count = 0
while (next_cursor != 0) and (ids_count < n_followers):
try:
print("Followers %s to %s: cursor: %s" %
(ids_count, ids_count + followers_query_size, next_cursor))
result = twitter.get_followers_ids(screen_name=screen_name,
count=followers_query_size,
cursor=next_cursor)
follower_ids = follower_ids + result['ids']
next_cursor = result['next_cursor']
ids_count += len(result['ids'])
# make sure the list only has unique IDs and sort
follower_ids = list(set(follower_ids))
follower_ids.sort()
print("Retrieved %s follower IDs from twitter" % len(follower_ids))
# store what we've got so far
# insert follower_ids in the followers collection
res = db.followers.update_one({"screen_name": screen_name},
{'$set': {
"ids": follower_ids
}})
accessToken = '***'
accessTokenSecret = '***'
api = Twython(apiKey,apiSecret,accessToken,accessTokenSecret)
while True:
#opens a txt file containing tweets and imports it into a list
tweetStr = open("tweetsexample.txt").readlines()
listlength = len(tweetStr)
for i in range(listlength):
# Auto Following function
try:
followers = api.get_followers_ids(screen_name = "Twitter Handle goes here")
for followers_ids in followers['ids']:
api.create_friendship(user_id=followers_ids)
except TwythonError as e:
print(e)
#Trys to post the next tweet in the list, if it fails it will continue to the next one
try:
api.update_status(status=tweetStr[i])
print ("Tweeted: " + tweetStr[i])
time.sleep(timeout)
except TwythonError as e:
print ("Failed: " + tweetStr[i])
time.sleep(30)
i = 0
class TwythonEndpointsTestCase(unittest.TestCase):
def setUp(self):
client_args = {
'headers': {
'User-Agent': '__twython__ Test'
},
'allow_redirects': False
}
oauth2_client_args = {
'headers': {} # This is so we can hit coverage that Twython sets User-Agent for us if none is supplied
}
self.api = Twython(app_key, app_secret,
oauth_token, oauth_token_secret,
client_args=client_args)
self.oauth2_api = Twython(app_key, access_token=access_token,
client_args=oauth2_client_args)
# Timelines
@unittest.skip('skipping non-updated test')
def test_get_mentions_timeline(self):
"""Test returning mentions timeline for authenticated user succeeds"""
self.api.get_mentions_timeline()
@unittest.skip('skipping non-updated test')
def test_get_user_timeline(self):
"""Test returning timeline for authenticated user and random user
succeeds"""
self.api.get_user_timeline() # Authenticated User Timeline
self.api.get_user_timeline(screen_name='twitter') # Random User Timeline
@unittest.skip('skipping non-updated test')
def test_get_protected_user_timeline_following(self):
"""Test returning a protected user timeline who you are following
succeeds"""
self.api.get_user_timeline(screen_name=protected_twitter_1)
@unittest.skip('skipping non-updated test')
def test_get_protected_user_timeline_not_following(self):
"""Test returning a protected user timeline who you are not following
fails and raise a TwythonAuthError"""
self.assertRaises(TwythonAuthError, self.api.get_user_timeline,
screen_name=protected_twitter_2)
@unittest.skip('skipping non-updated test')
def test_retweeted_of_me(self):
"""Test that getting recent tweets by authenticated user that have
been retweeted by others succeeds"""
self.api.retweeted_of_me()
@unittest.skip('skipping non-updated test')
def test_get_home_timeline(self):
"""Test returning home timeline for authenticated user succeeds"""
self.api.get_home_timeline()
# Tweets
@unittest.skip('skipping non-updated test')
def test_get_retweets(self):
"""Test getting retweets of a specific tweet succeeds"""
self.api.get_retweets(id=test_tweet_id)
@unittest.skip('skipping non-updated test')
def test_show_status(self):
"""Test returning a single status details succeeds"""
self.api.show_status(id=test_tweet_id)
@unittest.skip('skipping non-updated test')
def test_update_and_destroy_status(self):
"""Test updating and deleting a status succeeds"""
status = self.api.update_status(status='Test post just to get deleted :( %s' % int(time.time()))
self.api.destroy_status(id=status['id_str'])
@unittest.skip('skipping non-updated test')
def test_get_oembed_tweet(self):
"""Test getting info to embed tweet on Third Party site succeeds"""
self.api.get_oembed_tweet(id='99530515043983360')
@unittest.skip('skipping non-updated test')
def test_get_retweeters_ids(self):
"""Test getting ids for people who retweeted a tweet succeeds"""
self.api.get_retweeters_ids(id='99530515043983360')
# Search
@unittest.skip('skipping non-updated test')
def test_search(self):
"""Test searching tweets succeeds"""
self.api.search(q='twitter')
# Direct Messages
@unittest.skip('skipping non-updated test')
def test_get_direct_messages(self):
"""Test getting the authenticated users direct messages succeeds"""
self.api.get_direct_messages()
@unittest.skip('skipping non-updated test')
def test_get_sent_messages(self):
"""Test getting the authenticated users direct messages they've
sent succeeds"""
self.api.get_sent_messages()
@unittest.skip('skipping non-updated test')
def test_send_get_and_destroy_direct_message(self):
"""Test sending, getting, then destory a direct message succeeds"""
message = self.api.send_direct_message(screen_name=protected_twitter_1,
text='Hey d00d! %s' % int(time.time()))
self.api.get_direct_message(id=message['id_str'])
self.api.destroy_direct_message(id=message['id_str'])
@unittest.skip('skipping non-updated test')
def test_send_direct_message_to_non_follower(self):
"""Test sending a direct message to someone who doesn't follow you
fails"""
self.assertRaises(TwythonError, self.api.send_direct_message,
screen_name=protected_twitter_2, text='Yo, man! %s' % int(time.time()))
# Friends & Followers
@unittest.skip('skipping non-updated test')
def test_get_user_ids_of_blocked_retweets(self):
"""Test that collection of user_ids that the authenticated user does
not want to receive retweets from succeeds"""
self.api.get_user_ids_of_blocked_retweets(stringify_ids=True)
@unittest.skip('skipping non-updated test')
def test_get_friends_ids(self):
"""Test returning ids of users the authenticated user and then a random
user is following succeeds"""
self.api.get_friends_ids()
self.api.get_friends_ids(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_get_followers_ids(self):
"""Test returning ids of users the authenticated user and then a random
user are followed by succeeds"""
self.api.get_followers_ids()
self.api.get_followers_ids(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_lookup_friendships(self):
"""Test returning relationships of the authenticating user to the
comma-separated list of up to 100 screen_names or user_ids provided
succeeds"""
self.api.lookup_friendships(screen_name='twitter,ryanmcgrath')
@unittest.skip('skipping non-updated test')
def test_get_incoming_friendship_ids(self):
"""Test returning incoming friendship ids succeeds"""
self.api.get_incoming_friendship_ids()
@unittest.skip('skipping non-updated test')
def test_get_outgoing_friendship_ids(self):
"""Test returning outgoing friendship ids succeeds"""
self.api.get_outgoing_friendship_ids()
@unittest.skip('skipping non-updated test')
def test_create_friendship(self):
"""Test creating a friendship succeeds"""
self.api.create_friendship(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_destroy_friendship(self):
"""Test destroying a friendship succeeds"""
self.api.destroy_friendship(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_update_friendship(self):
"""Test updating friendships succeeds"""
self.api.update_friendship(screen_name=protected_twitter_1,
retweets='true')
self.api.update_friendship(screen_name=protected_twitter_1,
retweets=False)
@unittest.skip('skipping non-updated test')
def test_show_friendships(self):
"""Test showing specific friendship succeeds"""
self.api.show_friendship(target_screen_name=protected_twitter_1)
@unittest.skip('skipping non-updated test')
def test_get_friends_list(self):
"""Test getting list of users authenticated user then random user is
following succeeds"""
self.api.get_friends_list()
self.api.get_friends_list(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_get_followers_list(self):
"""Test getting list of users authenticated user then random user are
followed by succeeds"""
self.api.get_followers_list()
self.api.get_followers_list(screen_name='twitter')
# Users
@unittest.skip('skipping non-updated test')
def test_get_account_settings(self):
"""Test getting the authenticated user account settings succeeds"""
self.api.get_account_settings()
@unittest.skip('skipping non-updated test')
def test_verify_credentials(self):
"""Test representation of the authenticated user call succeeds"""
self.api.verify_credentials()
@unittest.skip('skipping non-updated test')
def test_update_account_settings(self):
"""Test updating a user account settings succeeds"""
self.api.update_account_settings(lang='en')
@unittest.skip('skipping non-updated test')
def test_update_delivery_service(self):
"""Test updating delivery settings fails because we don't have
a mobile number on the account"""
self.assertRaises(TwythonError, self.api.update_delivery_service,
device='none')
@unittest.skip('skipping non-updated test')
def test_update_profile(self):
"""Test updating profile succeeds"""
self.api.update_profile(include_entities='true')
@unittest.skip('skipping non-updated test')
def test_update_profile_colors(self):
"""Test updating profile colors succeeds"""
self.api.update_profile_colors(profile_background_color='3D3D3D')
@unittest.skip('skipping non-updated test')
def test_list_blocks(self):
"""Test listing users who are blocked by the authenticated user
succeeds"""
self.api.list_blocks()
@unittest.skip('skipping non-updated test')
def test_list_block_ids(self):
"""Test listing user ids who are blocked by the authenticated user
succeeds"""
self.api.list_block_ids()
@unittest.skip('skipping non-updated test')
def test_create_block(self):
"""Test blocking a user succeeds"""
self.api.create_block(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_destroy_block(self):
"""Test unblocking a user succeeds"""
self.api.destroy_block(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_lookup_user(self):
"""Test listing a number of user objects succeeds"""
self.api.lookup_user(screen_name='twitter,justinbieber')
@unittest.skip('skipping non-updated test')
def test_show_user(self):
"""Test showing one user works"""
self.api.show_user(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_search_users(self):
"""Test that searching for users succeeds"""
self.api.search_users(q='Twitter API')
@unittest.skip('skipping non-updated test')
def test_get_contributees(self):
"""Test returning list of accounts the specified user can
contribute to succeeds"""
self.api.get_contributees(screen_name='TechCrunch')
@unittest.skip('skipping non-updated test')
def test_get_contributors(self):
"""Test returning list of accounts that contribute to the
authenticated user fails because we are not a Contributor account"""
self.assertRaises(TwythonError, self.api.get_contributors,
screen_name=screen_name)
@unittest.skip('skipping non-updated test')
def test_remove_profile_banner(self):
"""Test removing profile banner succeeds"""
self.api.remove_profile_banner()
@unittest.skip('skipping non-updated test')
def test_get_profile_banner_sizes(self):
"""Test getting list of profile banner sizes fails because
we have not uploaded a profile banner"""
self.assertRaises(TwythonError, self.api.get_profile_banner_sizes)
# Suggested Users
@unittest.skip('skipping non-updated test')
def test_get_user_suggestions_by_slug(self):
"""Test getting user suggestions by slug succeeds"""
self.api.get_user_suggestions_by_slug(slug='twitter')
@unittest.skip('skipping non-updated test')
def test_get_user_suggestions(self):
"""Test getting user suggestions succeeds"""
self.api.get_user_suggestions()
@unittest.skip('skipping non-updated test')
def test_get_user_suggestions_statuses_by_slug(self):
"""Test getting status of suggested users succeeds"""
self.api.get_user_suggestions_statuses_by_slug(slug='funny')
# Favorites
@unittest.skip('skipping non-updated test')
def test_get_favorites(self):
"""Test getting list of favorites for the authenticated
user succeeds"""
self.api.get_favorites()
@unittest.skip('skipping non-updated test')
def test_create_and_destroy_favorite(self):
"""Test creating and destroying a favorite on a tweet succeeds"""
self.api.create_favorite(id=test_tweet_id)
self.api.destroy_favorite(id=test_tweet_id)
# Lists
@unittest.skip('skipping non-updated test')
def test_show_lists(self):
"""Test show lists for specified user"""
self.api.show_lists(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_get_list_statuses(self):
"""Test timeline of tweets authored by members of the
specified list succeeds"""
self.api.get_list_statuses(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_create_update_destroy_list_add_remove_list_members(self):
"""Test create a list, adding and removing members then
deleting the list succeeds"""
the_list = self.api.create_list(name='Stuff %s' % int(time.time()))
list_id = the_list['id_str']
self.api.update_list(list_id=list_id, name='Stuff Renamed %s' % int(time.time()))
screen_names = ['johncena', 'xbox']
# Multi add/delete members
self.api.create_list_members(list_id=list_id,
screen_name=screen_names)
self.api.delete_list_members(list_id=list_id,
screen_name=screen_names)
# Single add/delete member
self.api.add_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list(list_id=list_id)
@unittest.skip('skipping non-updated test')
def test_get_list_memberships(self):
"""Test list of memberhips the authenticated user succeeds"""
self.api.get_list_memberships()
@unittest.skip('skipping non-updated test')
def test_get_list_subscribers(self):
"""Test list of subscribers of a specific list succeeds"""
self.api.get_list_subscribers(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_subscribe_is_subbed_and_unsubscribe_to_list(self):
"""Test subscribing, is a list sub and unsubbing to list succeeds"""
self.api.subscribe_to_list(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
# Returns 404 if user is not a subscriber
self.api.is_list_subscriber(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name,
screen_name=screen_name)
self.api.unsubscribe_from_list(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_is_list_member(self):
"""Test returning if specified user is member of a list succeeds"""
# Returns 404 if not list member
self.api.is_list_member(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name,
screen_name='themattharris')
@unittest.skip('skipping non-updated test')
def test_get_list_members(self):
"""Test listing members of the specified list succeeds"""
self.api.get_list_members(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_get_specific_list(self):
"""Test getting specific list succeeds"""
self.api.get_specific_list(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_get_list_subscriptions(self):
"""Test collection of the lists the specified user is
subscribed to succeeds"""
self.api.get_list_subscriptions(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_show_owned_lists(self):
"""Test collection of lists the specified user owns succeeds"""
self.api.show_owned_lists(screen_name='twitter')
# Saved Searches
@unittest.skip('skipping non-updated test')
def test_get_saved_searches(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
self.api.get_saved_searches()
@unittest.skip('skipping non-updated test')
def test_create_get_destroy_saved_search(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
saved_search = self.api.create_saved_search(query='#Twitter')
saved_search_id = saved_search['id_str']
self.api.show_saved_search(id=saved_search_id)
self.api.destroy_saved_search(id=saved_search_id)
# Places & Geo
@unittest.skip('skipping non-updated test')
def test_get_geo_info(self):
"""Test getting info about a geo location succeeds"""
self.api.get_geo_info(place_id='df51dec6f4ee2b2c')
@unittest.skip('skipping non-updated test')
def test_reverse_geo_code(self):
"""Test reversing geocode succeeds"""
self.api.reverse_geocode(lat='37.76893497', long='-122.42284884')
@unittest.skip('skipping non-updated test')
def test_search_geo(self):
"""Test search for places that can be attached
to a statuses/update succeeds"""
self.api.search_geo(query='Toronto')
@unittest.skip('skipping non-updated test')
def test_get_similar_places(self):
"""Test locates places near the given coordinates which
are similar in name succeeds"""
self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ')
# Trends
@unittest.skip('skipping non-updated test')
def test_get_place_trends(self):
"""Test getting the top 10 trending topics for a specific
WOEID succeeds"""
self.api.get_place_trends(id=1)
@unittest.skip('skipping non-updated test')
def test_get_available_trends(self):
"""Test returning locations that Twitter has trending
topic information for succeeds"""
self.api.get_available_trends()
@unittest.skip('skipping non-updated test')
def test_get_closest_trends(self):
"""Test getting the locations that Twitter has trending topic
information for, closest to a specified location succeeds"""
self.api.get_closest_trends(lat='37', long='-122')
# Help
@unittest.skip('skipping non-updated test')
def test_get_twitter_configuration(self):
"""Test getting Twitter's configuration succeeds"""
self.api.get_twitter_configuration()
@unittest.skip('skipping non-updated test')
def test_get_supported_languages(self):
"""Test getting languages supported by Twitter succeeds"""
self.api.get_supported_languages()
@unittest.skip('skipping non-updated test')
def test_privacy_policy(self):
"""Test getting Twitter's Privacy Policy succeeds"""
self.api.get_privacy_policy()
@unittest.skip('skipping non-updated test')
def test_get_tos(self):
"""Test getting the Twitter Terms of Service succeeds"""
self.api.get_tos()
@unittest.skip('skipping non-updated test')
def test_get_application_rate_limit_status(self):
"""Test getting application rate limit status succeeds"""
self.oauth2_api.get_application_rate_limit_status()
with open('auth.json') as fd:
auth = json.load(fd)
twitter = Twython(auth['API_Key'], auth['API_Secret'], auth['Access_Token'],
auth['Access_Token_Secret'])
user_id = auth['username']
with open('followers.json') as fd:
old_followers = json.load(fd)
cur_followers = []
cursor = -1
while cursor:
followers = twitter.get_followers_ids(user_id=user_id, cursor=cursor)
cursor = followers['next_cursor']
cur_followers += followers['ids']
with open('followers.json', 'w') as fd:
json.dump(cur_followers, fd)
lost_followers = list(set(old_followers) - set(cur_followers))
gained_followers = list(set(cur_followers) - set(old_followers))
if old_followers != cur_followers:
if lost_followers:
lost_follower_info = twitter.lookup_user(user_id=lost_followers)
lost_follower_names = [
user['screen_name'] for user in lost_follower_info
# ---------------------------------------------------------
# MongoDB connection
# ---------------------------------------------------------
DBNAME = config['database']['name']
client = MongoClient()
db = client[DBNAME]
screen_name = 'alexip' # The main twitter account
n_max_folwrs = 700 # The number of followers to consider
# ---------------------------------------------------------
# 1) get follower ids
# see https://dev.twitter.com/rest/reference/get/followers/ids
# ---------------------------------------------------------
flwrs = twitter.get_followers_ids(screen_name = screen_name,
count = n_max_folwrs)
flw_ids = flwrs['ids']
flw_ids.sort()
# insert follower_ids in db
db.followers.insert_one({"follower_ids": flw_ids, "user": "******"})
# ---------------------------------------------------------
# Get 200 tweets per follower
# (200 is the maximum number of tweets imposed by twitter)
# ---------------------------------------------------------
for id in flw_ids:
try:
# only retrieve tweets for user if we don't have them in store already
twt = db.tweets.find({'user_id':id})
handle_rate_limiting()
ACCESS_TOKEN_SECRET = 'INSERT HERE'
twitter = Twython(API_KEY, API_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
return(twitter)
'''
Determining your Twitter followers and friends
'''
twitter = twitter_oauth_login()
friends_ids = twitter.get_friends_ids(count=5000)
friends_ids = friends_ids['ids']
followers_ids = twitter.get_followers_ids(count=5000)
followers_ids = followers_ids['ids']
print len(friends_ids), len(followers_ids)
friends_set = set(friends_ids)
followers_set = set(followers_ids)
print('Number of Twitter users who either are our friend or follow you (union):')
print(len(friends_set.union(followers_set)))
len(friends_set | followers_set)
len(set(friends_ids+followers_ids))
print('Number of Twitter users who follow you and are your friend (intersection):')
from license import __copyright__
from license import __copyrighta__
from license import __license__
__version__ = "0.0.1"
from license import __bitcoin__
import sys
from twython import Twython, TwythonError
from config import *
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
l = len(sys.argv)
if l >= 2:
user = sys.argv[1]
else:
user = input("Enter Twitter handle to get followers of: ")
try:
followids = twitter.get_followers_ids(screen_name=user)
except TwythonError as e:
print(e)
for x in followids["ids"]:
try:
data = twitter.show_user(user_id=x)
print(data["screen_name"])
except TwythonError as e:
print(e)
def get_followers():
distinct = getScreenname.getDistinctScreenames(
"all_tweets_dict_fully_checked")
index = 0
num_checked = 0
rate_ex = 0
for sname in distinct:
file = open("followers.txt", "r+")
already_checked = False
for line in file:
words = line.split('\t')
if words[0] == sname:
already_checked = True
if index >= 7:
break
APP_KEY = APP_KEYS[index]
APP_SECRET = APP_SECRETS[index]
OAUTH_TOKEN = OAUTH_TOKENS[index]
OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index]
if not already_checked:
try:
twitter = Twython(APP_KEY, APP_SECRET)
auth = twitter.get_authentication_tokens()
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN,
OAUTH_TOKEN_SECRET)
followers = twitter.get_followers_ids(screen_name=sname)
print sname
file.write(sname)
file.write("\t")
for ID in followers['ids']:
file.write(str(ID) + " ")
file.write("\n")
num_checked += 1
except Exception as e:
if "429 (Too Many Requests)" in str(e):
print "\nchanging apps!\n"
index += 1
if "401 (Unauthorized)" in str(e):
print "401 error"
file.write(sname)
file.write("\t")
file.write("Unauthorized")
file.write("\n")
if "404 (Not Found)" in str(e):
print "404 error"
file.write(sname)
file.write("\t")
file.write("NotFound")
file.write("\n")
else:
print e
rate_ex += 1
file.close()
if (num_checked != 0 or rate_ex != 0):
print "sleepy time"
time.sleep(860)
get_followers()
