def scrape(arg):
results = {}
twitter = Twython(APP_KEY, APP_SECRET, oauth_version=2)
ACCESS_TOKEN = twitter.obtain_access_token()
twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN)
friends = twitter.get_friends_ids(screen_name=arg)
#hard cap for time
limit = 10
for follower_id in friends['ids']:
if limit > 0:
params = {'user_id': follower_id, 'count': 3}
tweets = twitter.get_user_timeline(**params)
for tweet in tweets:
encoded = tweet['text'].encode("utf8")
results = dict(Counter(results) + Counter(text_tags(encoded)))
#random.choice(news)
#f.write(str(text_tags(encoded)))
limit = limit - 1
else:
break
#print results
key = max(results.iteritems(), key=operator.itemgetter(1))[0]
n = random.choice(news)
return [key, n]
def get_queryset(self):
provider = self.request.QUERY_PARAMS.get('provider', None)
if provider == 'facebook':
# TODO: what does it look like when a user has more than one social auth for a provider? Is this a thing
# that can happen? How does it affect SocialShareMixin? The first one is the oldest--do we actually want
# the last one?
user_social_auth = self.request.user.social_auth.filter(provider='facebook').first()
graph = facebook.GraphAPI(user_social_auth.extra_data['access_token'])
facebook_friends = graph.request("v2.2/me/friends")["data"]
friends = User.objects.filter(social_auth__provider='facebook',
social_auth__uid__in=[user["id"] for user in facebook_friends])
return friends
elif provider == 'twitter':
user_social_auth = self.request.user.social_auth.filter(provider='twitter').first()
twitter = Twython(
app_key=settings.SOCIAL_AUTH_TWITTER_KEY,
app_secret=settings.SOCIAL_AUTH_TWITTER_SECRET,
oauth_token=user_social_auth.tokens['oauth_token'],
oauth_token_secret=user_social_auth.tokens['oauth_token_secret']
)
twitter_friends = twitter.get_friends_ids()["ids"]
friends = User.objects.filter(social_auth__provider='twitter',
social_auth__uid__in=twitter_friends)
return friends
else:
return Response({"errors": "{} is not a valid social provider".format(provider)},
status=status.HTTP_400_BAD_REQUEST)
def scrape(arg):
results = {}
twitter = Twython(APP_KEY, APP_SECRET, oauth_version=2)
ACCESS_TOKEN = twitter.obtain_access_token()
twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN)
friends = twitter.get_friends_ids(screen_name = arg)
#hard cap for time
limit = 10
for follower_id in friends['ids']:
if limit > 0 :
params = {'user_id':follower_id, 'count':3 }
tweets = twitter.get_user_timeline(**params)
for tweet in tweets:
encoded = tweet['text'].encode("utf8")
results = dict(Counter(results) + Counter(text_tags(encoded)))
#random.choice(news)
#f.write(str(text_tags(encoded)))
limit = limit - 1
else:
break
#print results
key = max(results.iteritems(), key=operator.itemgetter(1))[0]
n = random.choice(news)
return [key, n]
def get_queryset(self):
provider = self.request.QUERY_PARAMS.get('provider', None)
if provider == 'facebook':
# TODO: what does it look like when a user has more than one social auth for a provider? Is this a thing
# that can happen? How does it affect SocialShareMixin? The first one is the oldest--do we actually want
# the last one?
user_social_auth = self.request.user.social_auth.filter(
provider='facebook').first()
graph = facebook.GraphAPI(
user_social_auth.extra_data['access_token'])
facebook_friends = graph.request("v2.2/me/friends")["data"]
friends = User.objects.filter(
social_auth__provider='facebook',
social_auth__uid__in=[user["id"] for user in facebook_friends])
return friends
elif provider == 'twitter':
user_social_auth = self.request.user.social_auth.filter(
provider='twitter').first()
twitter = Twython(
app_key=settings.SOCIAL_AUTH_TWITTER_KEY,
app_secret=settings.SOCIAL_AUTH_TWITTER_SECRET,
oauth_token=user_social_auth.tokens['oauth_token'],
oauth_token_secret=user_social_auth.
tokens['oauth_token_secret'])
twitter_friends = twitter.get_friends_ids()["ids"]
friends = User.objects.filter(social_auth__provider='twitter',
social_auth__uid__in=twitter_friends)
return friends
else:
return Response(
{
"errors":
"{} is not a valid social provider".format(provider)
},
status=status.HTTP_400_BAD_REQUEST)
def get_followers_following(usr_id):
index = 0
print "GETTING FOLLOWERS AND FOLLOWING: " + str(usr_id)
friend_cursor = -1
follower_cursor = -1
APP_KEY = APP_KEYS[index]
APP_SECRET = APP_SECRETS[index]
OAUTH_TOKEN = OAUTH_TOKENS[index]
OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index]
while (1==1):
try:
twitter = Twython (APP_KEY, APP_SECRET)
auth = twitter.get_authentication_tokens()
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
friends = ""
while (friend_cursor != 0):
following = twitter.get_friends_ids(id = usr_id, cursor = friend_cursor)
for ID in following['ids']:
friends += str(ID) + " "
friend_cursor = following["next_cursor"]
friends = friends[:-1]
if len(friends) == 0:
friends = "null"
follow = ""
while (follower_cursor != 0):
followers = twitter.get_followers_ids(id = usr_id,cursor= follower_cursor)
for ID in followers['ids']:
follow += str(ID) + " "
follower_cursor = followers["next_cursor"]
follow= follow[:-1]
if len(follow) == 0:
follow = "null"
return (friends,follow)
except Exception as e:
print e
if "429 (Too Many Requests)" in str(e):
#global index
index += 1
if index == len(APP_KEYS):
index = 0
print "sleepy time - 15 minutes"
print datetime.datetime.now()
time.sleep(910)
return get_followers_following(usr_id)
elif "401 (Unauthorized)" in str(e):
print "401 error"
return ("null","null")
elif "404 (Not Found)" in str(e):
print "404 error"
return ("null","null")
else:
print e
return ("null","null")
def get_friends(user_social_auth):
twitter = Twython(
app_key=settings.SOCIAL_AUTH_TWITTER_KEY,
app_secret=settings.SOCIAL_AUTH_TWITTER_SECRET,
oauth_token=user_social_auth.tokens['oauth_token'],
oauth_token_secret=user_social_auth.tokens['oauth_token_secret'])
twitter_friends = twitter.get_friends_ids()["ids"]
friends = User.objects.filter(social_auth__provider='twitter',
social_auth__uid__in=twitter_friends)
return friends
def get_friends(user_social_auth):
twitter = Twython(
app_key=settings.SOCIAL_AUTH_TWITTER_KEY,
app_secret=settings.SOCIAL_AUTH_TWITTER_SECRET,
oauth_token=user_social_auth.tokens['oauth_token'],
oauth_token_secret=user_social_auth.tokens['oauth_token_secret']
)
twitter_friends = twitter.get_friends_ids()["ids"]
friends = User.objects.filter(social_auth__provider='twitter',
social_auth__uid__in=twitter_friends)
return friends
def main():
APP_KEY = 'ORbbZFlJWipi3HFblfPuQ'
APP_SECRET = 's2em5f68RBV8UDgW7iXOH5PMFArF5aVj8GG6IgsU'
twitter = Twython(APP_KEY, APP_SECRET)
auth = twitter.get_authentication_tokens()
OAUTH_TOKEN = '165916420-31NTOhsitabrC6tgo0Zd87q0f60mGUdjBZan56HQ'
OAUTH_TOKEN_SECRET = '8gpWg3g207b4QMuabv2xkw2VkkMyBPnDEImTquPgZC0vV'
main_user = 100
# Requires Authentication as of Twitter API v1.1
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
try:
user_timeline = twitter.get_friends_ids(screen_name='alex_pardo5', count=5000)
except TwythonError as e:
print e
i = 0
print user_timeline
#G=nx.Graph()
#G.add_node(main_user)
#print user_timeline
#server = xmlrpclib.Server('http://localhost:20738/RPC2')
#G = server.ubigraph
G = ubigraph.Ubigraph()
G.clear()
main_user = G.newVertex(shape="sphere", color="#ffff00", label='alex_pardo5')
red = G.newVertexStyle(shape="sphere", color="#ff0000", size="1")
for user in user_timeline['ids']:
#print twitter.show_user(user_id=user)
#tmp = twitter.show_user(user_id=user)['screen_name']
tmp = user
tmp = G.newVertex(style=red, label=str(tmp))
G.newEdge(main_user,tmp,arrow=True, width=random.randint(1, 5))
i += 1
print i
def fetch_friend_ids(self, user):
"""
fetches friend id's from twitter
Return:
collection of friend ids
"""
auth_data = self._auth_data(user)
tw = Twython(**auth_data)
cursor = -1
friend_ids = []
while True:
data = tw.get_friends_ids(cursor=cursor)
friend_ids += data.get('ids', [])
next_cursor = data.get('next_cursor', 0)
prev_cursor = data.get('prev_cursor', 0)
if not next_cursor or next_cursor == prev_cursor:
break
else:
cursor = next_cursor
return friend_ids
cursor.execute("select id from t_gh_json")
ids = cursor.fetchall()
cursor.execute("create table ghtwdirectextra(user_id text, followerids text, friendids text)")
cursor.execute("create table ghtwdirecttimeline(user_id text, tweets text, tweetids text)")
for i in ids:
k=""
for j in i[0]:
if j != "'":
k = k + j
print k
try:
followers = t.get_followers_ids(user_id = k, count=5000)
#print followers['ids']
friends = t.get_friends_ids(user_id = k, count=5000)
#print friends['ids']
timeline = t.get_user_timeline(user_id = k, count=200, include_rts=1)
for i in timeline:
#print i['text']
#print i['id']
cursor.execute("insert into ghtwdirecttimeline(user_id, tweets, tweetids) values(%s,%s,%s)",(k, i['text'], i['id']))
cursor.execute("insert into ghtwdirectextra(user_id, followerids, friendids) values(%s,%s,%s)",(k, followers['ids'], friends['ids']))
print "inserted"
except TwythonRateLimitError as e:
print "rate limit"
time.sleep(60*15)
continue
except TwythonAuthError as e:
continue
except TwythonError as e:
class TwitterHelper(object):
def __init__(self, identity):
self.identity = identity
if not self.identity.tokens:
self.identity.tokens = authorisationhelper.get_tokens(identity.screen_name)
self.twitter = Twython(
self.identity.tokens[0],
self.identity.tokens[1],
self.identity.tokens[2],
self.identity.tokens[3]
)
# self.identity.twid = self.twitter.lookup_user(screen_name=identity.screen_name)[0]["id_str"]
self.mutation = [" ,", " .", " *", " `", " -", " _"]
self.twitter_configuration = self.twitter.get_twitter_configuration()
logger.debug(self.twitter_configuration)
giphyhelper.set_photo_size_limit(self.twitter_configuration["photo_size_limit"])
me = self.twitter.lookup_user(screen_name=self.identity.screen_name)[0]
logger.debug(me)
self.identity.update(me)
self.rates = None
self.update_rate_limits()
@retry(**retry_args)
def send(self, outbox_item):
if type(outbox_item) is OutgoingTweet:
outbox_item.display()
if outbox_item.filePaths and any(outbox_item.filePaths):
for filePath in outbox_item.filePaths:
# ext = os.path.splitext(filePath)
# if ext == "mp4":
# media_id = self._upload_video(filePath)
# else:
media_id = self._upload_media(filePath)
if media_id:
outbox_item.media_ids.append(media_id)
split_tweets = tweet_splitter.split_tweet(outbox_item, self.twitter_configuration)
in_reply_to_id_str = outbox_item.in_reply_to_id_str
for split_tweet in split_tweets:
split_tweet.in_reply_to_id_str = in_reply_to_id_str
response = self.twitter.update_status(**split_tweet.get_tweet_params())
split_tweet.id_str = response["id_str"]
self.identity.conversations.outgoing(split_tweet)
self.identity.statistics.record_outgoing_tweet()
in_reply_to_id_str = split_tweet.id_str
return in_reply_to_id_str
if type(outbox_item) is OutgoingDirectMessage:
if not outbox_item.screen_name and not outbox_item.user_id:
outbox_item.screen_name = self.identity.admin_screen_name
outbox_item.display()
self.twitter.send_direct_message(
text=outbox_item.text,
screen_name=outbox_item.screen_name,
user_id=outbox_item.user_id)
self.identity.statistics.record_outgoing_direct_message()
return None
def quote_tweet(self, inbox_item, text=None, file_paths=None):
reply_as_quote_tweet = inbox_item.is_tweet
reply_as_dm = inbox_item.is_direct_message
if reply_as_quote_tweet:
logger.info("quoting to %s as quote tweet", inbox_item.text)
tweet = OutgoingTweet(
quote=inbox_item,
text=text,
file_paths=file_paths,
)
return self.send(tweet)
if reply_as_dm:
logger.info("replying to %s as DM", inbox_item.text)
dm = OutgoingDirectMessage(
reply_to=inbox_item,
text=text)
return self.send(dm)
return None
def reply_with(self, inbox_item, text=None, as_tweet=False, as_direct_message=False, file_paths=None,
in_reply_to_id_str=None):
reply_as_tweet = as_tweet or not as_direct_message and inbox_item.is_tweet
reply_as_dm = as_direct_message or not as_tweet and inbox_item.is_direct_message
if reply_as_tweet:
logger.info("replying to %s as tweet", inbox_item.text)
tweet = OutgoingTweet(
reply_to=inbox_item,
text=text,
file_paths=file_paths,
in_reply_to_id_str=in_reply_to_id_str)
return self.send(tweet)
if reply_as_dm:
logger.info("replying to %s as DM", inbox_item.text)
dm = OutgoingDirectMessage(
reply_to=inbox_item,
text=text)
return self.send(dm)
return None
def _upload_media(self, file_path):
file = None
try:
file = open(file_path, 'rb')
media = self.twitter.upload_media(media=file)
self.identity.statistics.increment("Media Uploads")
return media["media_id_string"]
finally:
if file:
file.close()
def get_streamer(self, topic=None, topic_name=None, responses=None, filter_level=None):
return Streamer(self.identity, topic, topic_name, responses, filter_level)
def _upload_video(self, file_path):
logger.info('[MyTwitter] uploading ' + file_path)
url = 'https://upload.twitter.com/1.1/media/upload.json'
file_size = os.path.getsize(file_path)
logger.info('[MyTwitter] Init')
init_params = {
"command": "INIT",
"media_type": "video/mp4",
"total_bytes": file_size
}
init_response = self.twitter.post(url, init_params)
logger.info(init_response)
media_id = init_response["media_id_string"]
logger.info('[MyTwitter] media_id ' + media_id)
segment = 0
chunk_size = 4 * 1024 * 1024
for chunk in fsh.bytes_from_file(file_path, chunk_size):
logger.info('[MyTwitter] Append ' + str(segment))
append_params = {
'command': 'APPEND',
'media_id': media_id,
'segment_index': segment
}
append_response = self.twitter.post(url, append_params, {'media': chunk})
logger.info(append_response)
segment += 1
logger.info('[MyTwitter] Finalize')
finalize_params = {
"command": "FINALIZE",
"media_id": media_id,
}
self.twitter.post(url, finalize_params)
return media_id
def get_trending_topics_for(self, woeids):
trending_topics = []
for woeid in woeids:
trends = self.twitter.get_place_trends(id=woeid)[0].get('trends', [])
for trend in trends:
trending_topics.append(trend['name'])
return trending_topics
def get_saved_searches(self):
saved_searches = []
searches = self.twitter.get_saved_searches()
for srch in searches:
saved_searches.append(srch['name'])
return saved_searches
def delete_saved_searches(self):
searches = self.twitter.get_saved_searches()
for srch in searches:
self.twitter.destroy_saved_search(id=srch["id_str"])
def search(self, text, result_type="popular"):
query = quote_plus(text)
return self.twitter.search(q=query, result_type=result_type)["statuses"]
@retry(**retry_args)
def favourite(self, id_str):
try:
self.twitter.create_favorite(id=id_str)
self.identity.statistics.record_favourite()
except TwythonError as ex:
if "You have already favourited this tweet" in str(ex):
logger.warning(ex)
else:
raise
@retry(**retry_args)
def retweet(self, id_str):
try:
self.twitter.retweet(id=id_str)
self.identity.statistics.record_retweet()
except TwythonError as ex:
if "You have already retweeted this tweet" in str(ex):
logger.warning(ex)
else:
raise
@retry(**retry_args)
def add_user_to_list(self, list_id, user_id, screen_name):
self.twitter.create_list_members(list_id=list_id, user_id=user_id, screen_name=screen_name)
@retry(**retry_args)
def block_user(self, user_id, user_screen_name=None):
self.twitter.create_block(user_id=user_id, screen_name=user_screen_name)
@retry(**retry_args)
def get_user_timeline(self, **kwargs):
return self._rate_limit("/statuses/user_timeline", self.twitter.get_user_timeline, **kwargs)
def unblock_user(self, user):
self.twitter.destroy_block(user_id=user.id, screen_name=user.screen_name)
@retry(**retry_args)
def unblock_users(self):
user_ids = self.twitter.list_block_ids(stringify_ids=True)
for user_id in user_ids["ids"]:
self.twitter.destroy_block(user_id=user_id)
@retry(**retry_args)
def show_owned_lists(self):
return self.twitter.show_owned_lists()["lists"]
@retry(**retry_args)
def get_list_members(self, list_id):
return self.twitter.get_list_members(list_id=list_id, count=5000, include_entities=False)
@blocked
@retry(**retry_args)
def create_list(self, **kwargs):
return self.twitter.create_list(**kwargs)
@retry(**retry_args)
def follow(self, user_id, screen_name):
logger.info("following user id {} @{}".format(user_id, screen_name))
self.twitter.create_friendship(user_id=user_id, screen_name=screen_name)
self.identity.statistics.increment("Follows")
@retry(**retry_args)
def unfollow(self, user_id, screen_name):
logger.info("unfollowing user id {} @{}".format(user_id, screen_name))
self.twitter.destroy_friendship(user_id=user_id, screen_name=screen_name)
self.identity.statistics.increment("Unfollows")
@retry(**retry_args)
def block(self, user_id, screen_name):
logger.info("blocking user id {} @{}".format(user_id, screen_name))
self.twitter.create_block(user_id=user_id, screen_name=screen_name)
self.identity.statistics.increment("Blocks")
@retry(**retry_args)
def report(self, user_id, screen_name):
logger.info("reporting user id {} @{}".format(user_id, screen_name))
self.twitter.report_spam(user_id=user_id, screen_name=screen_name)
self.identity.statistics.increment("Reports")
@retry(**retry_args)
def lookup_user(self, user_id):
return self._rate_limit("/users/lookup", self.twitter.lookup_user, user_id=user_id)
def sing_song(self, song, target=None, inbox_item=None, text=None, hashtag=None):
if not text:
text = random.choice(["All together now!", "Sing along!"])
text += ' ' + song["video"]
if hashtag:
text += ' ' + hashtag
in_reply_to_id_str = self._send_to(
inbox_item=inbox_item,
text=text,
target=target,
in_reply_to_id_str=None)
time.sleep(5)
lastlyrics = set([])
for lyric in song["lyrics"]:
lyric = lyric.strip()
if lyric:
if "<<screen_name>>" in lyric:
lyric = lyric.replace("<<screen_name>>", "@" + target)
if hashtag:
lyric += " " + hashtag
while lyric in lastlyrics:
lyric += random.choice(self.mutation)
lastlyrics.add(lyric)
self.identity.statistics.record_outgoing_song_lyric()
in_reply_to_id_str = self._send_to(
inbox_item,
lyric,
target,
in_reply_to_id_str)
time.sleep(5)
def _send_to(self, inbox_item, text, target, in_reply_to_id_str):
if inbox_item:
return self.reply_with(
inbox_item=inbox_item,
text=text,
in_reply_to_id_str=in_reply_to_id_str)
else:
status = ""
if target:
# noinspection PyUnresolvedReferences
if isinstance(target, basestring):
status = ".@" + target
elif isinstance(target, User.User):
status = ".@" + target.screen_name
status += " " + text
tweet = OutgoingTweet(
text=status,
in_reply_to_id_str=in_reply_to_id_str)
return self.send(tweet)
@retry(**retry_args)
def get_list_subscriptions(self):
return self.twitter.get_list_subscriptions()
@retry(**retry_args)
def subscribe_to_list(self, list_id):
return self.twitter.subscribe_to_list(list_id=list_id)
@retry(**retry_args)
def geocode(self, location):
result = self.twitter.search_geo(
query=location.full_name,
max_results=5,
lat=location.latitude,
long=location.longitude)
logger.info(result)
if result["result"]["places"]:
# for place in result["result"]["places"]:
# logger.info(place["full_name"])
place = result["result"]["places"][0]
location.place_id_twitter = place["id"]
return location
else:
return None
@retry(**retry_args)
def reverse_geocode(self, location):
result = self.twitter.reverse_geocode(
max_results=5,
lat=location.latitude,
long=location.longitude)
logger.info(result)
if result["result"]["places"]:
# for place in result["result"]["places"]:
# logger.info(place["full_name"])
place = result["result"]["places"][0]
location.place_id_twitter = place["id"]
return location
else:
return None
@retry(**retry_args)
def update_profile_image(self, file_path):
if file_path:
logger.info("updating profile image %s" % file_path)
with open(file_path, 'rb') as file:
self.twitter.update_profile_image(image=file)
@retry(**retry_args)
def update_profile_banner_image(self, file_path):
if file_path:
logger.info("updating banner image %s" % file_path)
with open(file_path, 'rb') as file:
try:
self.twitter.update_profile_banner_image(banner=file)
except TwythonError as ex:
if "Response was not valid JSON" in str(ex):
# twython issue i think
logger.warning(ex)
else:
raise
@retry(**retry_args)
def update_profile(self, **kwargs):
return self.twitter.update_profile(**kwargs)
@retry(**retry_args)
def get_list_statuses(self, **kwargs):
return self.twitter.get_list_statuses(**kwargs)
@retry(**retry_args)
def get_user_suggestions_by_slug(self, **kwargs):
return self.twitter.get_user_suggestions_by_slug(**kwargs)
@retry(**retry_args)
def get_user_suggestions(self, **kwargs):
return self.twitter.get_user_suggestions(**kwargs)
@retry(**retry_args)
def lookup_status(self, **kwargs):
return self.twitter.lookup_status(**kwargs)
@retry(**retry_args)
def get_followers(self, **kwargs):
kwargs["stringify_ids"] = True
followers = set()
cursor = -1
while cursor != "0":
kwargs["cursor"] = cursor
logger.info("getting followers")
response = self.twitter.get_followers_ids(**kwargs)
cursor = response["next_cursor_str"]
followers = followers.union(set(response["ids"]))
return followers
@retry(**retry_args)
def get_following(self, **kwargs):
kwargs["stringify_ids"] = True
following = set()
cursor = -1
while cursor != "0":
kwargs["cursor"] = cursor
logger.info("getting following")
response = self.twitter.get_friends_ids(**kwargs)
cursor = response["next_cursor_str"]
following = following.union(set(response["ids"]))
return following
@retry(**retry_args)
def update_rate_limits(self):
data = self.twitter.get_application_rate_limit_status()
self.rates = RateLimits(data)
logger.info("Updated rate limits for {}: {}".format(self.identity.screen_name, self.rates.display()))
def _rate_limit(self, limit_name, func, *args, **kwargs):
if self.rates.can(limit_name):
try:
return func(*args, **kwargs)
except Exception as ex:
logger.warning(ex)
return None
else:
logger.warning("{} limit exceeded".format(limit_name))
return None
@retry(**retry_args)
def get_statuses(self, id_strs):
id_strs_csv = ",".join(id_strs)
return self.twitter.lookup_status(id=id_strs_csv)
@retry(**retry_args)
def get_status(self, id_str):
return self.twitter.show_status(id=id_str)
def get_friends_ids(oauth_token, oauth_token_secret, user_id, count=500, cursor=-1):
twitter = Twython(consumer_key, consumer_secret, oauth_token, oauth_token_secret)
result = twitter.get_friends_ids(user_id=user_id, count=count, cursor=cursor)
return result
class TwythonAPITestCase(unittest.TestCase):
def setUp(self):
client_args = {
'headers': {
'User-Agent': '__twython__ Test'
},
'allow_redirects': False
}
oauth2_client_args = {
'headers': {} # This is so we can hit coverage that Twython sets User-Agent for us if none is supplied
}
self.api = Twython(app_key, app_secret,
oauth_token, oauth_token_secret,
client_args=client_args)
self.oauth2_api = Twython(app_key, access_token=access_token,
client_args=oauth2_client_args)
def test_construct_api_url(self):
"""Test constructing a Twitter API url works as we expect"""
url = 'https://api.twitter.com/1.1/search/tweets.json'
constructed_url = self.api.construct_api_url(url, q='#twitter')
self.assertEqual(constructed_url, 'https://api.twitter.com/1.1/search/tweets.json?q=%23twitter')
def test_get(self):
"""Test Twython generic GET request works"""
self.api.get('account/verify_credentials')
def test_post(self):
"""Test Twython generic POST request works, with a full url and
with just an endpoint"""
update_url = 'https://api.twitter.com/1.1/statuses/update.json'
status = self.api.post(update_url, params={'status': 'I love Twython!'})
self.api.post('statuses/destroy/%s' % status['id_str'])
def test_get_lastfunction_header(self):
"""Test getting last specific header of the last API call works"""
self.api.get('statuses/home_timeline')
self.api.get_lastfunction_header('x-rate-limit-remaining')
def test_get_lastfunction_header_not_present(self):
"""Test getting specific header that does not exist from the last call returns None"""
self.api.get('statuses/home_timeline')
header = self.api.get_lastfunction_header('does-not-exist')
self.assertEqual(header, None)
def test_get_lastfunction_header_no_last_api_call(self):
"""Test attempting to get a header when no API call was made raises a TwythonError"""
self.assertRaises(TwythonError, self.api.get_lastfunction_header,
'no-api-call-was-made')
def test_search_gen(self):
"""Test looping through the generator results works, at least once that is"""
search = self.api.search_gen('twitter', count=1)
counter = 0
while counter < 2:
counter += 1
result = next(search)
new_id_str = int(result['id_str'])
if counter == 1:
prev_id_str = new_id_str
time.sleep(1) # Give time for another tweet to come into search
if counter == 2:
self.assertTrue(new_id_str > prev_id_str)
def test_encode(self):
"""Test encoding UTF-8 works"""
self.api.encode('Twython is awesome!')
def test_html_for_tweet(self):
"""Test HTML for Tweet returns what we want"""
tweet_text = self.api.html_for_tweet(test_tweet_object)
self.assertEqual(test_tweet_html, tweet_text)
def test_html_for_tweet_expanded_url(self):
"""Test using expanded url in HTML for Tweet displays full urls"""
tweet_text = self.api.html_for_tweet(test_tweet_object,
use_expanded_url=True)
# Make sure full url is in HTML
self.assertTrue('http://google.com' in tweet_text)
def test_html_for_tweet_short_url(self):
"""Test using expanded url in HTML for Tweet displays full urls"""
tweet_text = self.api.html_for_tweet(test_tweet_object, False)
# Make sure HTML doesn't contain the display OR exapanded url
self.assertTrue(not 'http://google.com' in tweet_text)
self.assertTrue(not 'google.com' in tweet_text)
def test_raise_error_on_bad_ssl_cert(self):
"""Test TwythonError is raised by a RequestException when an actual HTTP happens"""
self.assertRaises(TwythonError, self.api.get, 'https://example.com')
# Timelines
def test_get_mentions_timeline(self):
"""Test returning mentions timeline for authenticated user succeeds"""
self.api.get_mentions_timeline()
def test_get_user_timeline(self):
"""Test returning timeline for authenticated user and random user
succeeds"""
self.api.get_user_timeline() # Authenticated User Timeline
self.api.get_user_timeline(screen_name='twitter') # Random User Timeline
def test_get_protected_user_timeline_following(self):
"""Test returning a protected user timeline who you are following
succeeds"""
self.api.get_user_timeline(screen_name=protected_twitter_1)
def test_get_protected_user_timeline_not_following(self):
"""Test returning a protected user timeline who you are not following
fails and raise a TwythonAuthError"""
self.assertRaises(TwythonAuthError, self.api.get_user_timeline,
screen_name=protected_twitter_2)
def test_retweeted_of_me(self):
"""Test that getting recent tweets by authenticated user that have
been retweeted by others succeeds"""
self.api.retweeted_of_me()
def test_get_home_timeline(self):
"""Test returning home timeline for authenticated user succeeds"""
self.api.get_home_timeline()
# Tweets
def test_get_retweets(self):
"""Test getting retweets of a specific tweet succeeds"""
self.api.get_retweets(id=test_tweet_id)
def test_show_status(self):
"""Test returning a single status details succeeds"""
self.api.show_status(id=test_tweet_id)
def test_update_and_destroy_status(self):
"""Test updating and deleting a status succeeds"""
status = self.api.update_status(status='Test post just to get deleted :(')
self.api.destroy_status(id=status['id_str'])
def test_get_oembed_tweet(self):
"""Test getting info to embed tweet on Third Party site succeeds"""
self.api.get_oembed_tweet(id='99530515043983360')
def test_get_retweeters_ids(self):
"""Test getting ids for people who retweeted a tweet succeeds"""
self.api.get_retweeters_ids(id='99530515043983360')
# Search
def test_search(self):
"""Test searching tweets succeeds"""
self.api.search(q='twitter')
# Direct Messages
def test_get_direct_messages(self):
"""Test getting the authenticated users direct messages succeeds"""
self.api.get_direct_messages()
def test_get_sent_messages(self):
"""Test getting the authenticated users direct messages they've
sent succeeds"""
self.api.get_sent_messages()
def test_send_get_and_destroy_direct_message(self):
"""Test sending, getting, then destory a direct message succeeds"""
message = self.api.send_direct_message(screen_name=protected_twitter_1,
text='Hey d00d! %s' % int(time.time()))
self.api.get_direct_message(id=message['id_str'])
self.api.destroy_direct_message(id=message['id_str'])
def test_send_direct_message_to_non_follower(self):
"""Test sending a direct message to someone who doesn't follow you
fails"""
self.assertRaises(TwythonError, self.api.send_direct_message,
screen_name=protected_twitter_2, text='Yo, man!')
# Friends & Followers
def test_get_user_ids_of_blocked_retweets(self):
"""Test that collection of user_ids that the authenticated user does
not want to receive retweets from succeeds"""
self.api.get_user_ids_of_blocked_retweets(stringify_ids=True)
def test_get_friends_ids(self):
"""Test returning ids of users the authenticated user and then a random
user is following succeeds"""
self.api.get_friends_ids()
self.api.get_friends_ids(screen_name='twitter')
def test_get_followers_ids(self):
"""Test returning ids of users the authenticated user and then a random
user are followed by succeeds"""
self.api.get_followers_ids()
self.api.get_followers_ids(screen_name='twitter')
def test_lookup_friendships(self):
"""Test returning relationships of the authenticating user to the
comma-separated list of up to 100 screen_names or user_ids provided
succeeds"""
self.api.lookup_friendships(screen_name='twitter,ryanmcgrath')
def test_get_incoming_friendship_ids(self):
"""Test returning incoming friendship ids succeeds"""
self.api.get_incoming_friendship_ids()
def test_get_outgoing_friendship_ids(self):
"""Test returning outgoing friendship ids succeeds"""
self.api.get_outgoing_friendship_ids()
def test_create_friendship(self):
"""Test creating a friendship succeeds"""
self.api.create_friendship(screen_name='justinbieber')
def test_destroy_friendship(self):
"""Test destroying a friendship succeeds"""
self.api.destroy_friendship(screen_name='justinbieber')
def test_update_friendship(self):
"""Test updating friendships succeeds"""
self.api.update_friendship(screen_name=protected_twitter_1,
retweets='true')
self.api.update_friendship(screen_name=protected_twitter_1,
retweets=False)
def test_show_friendships(self):
"""Test showing specific friendship succeeds"""
self.api.show_friendship(target_screen_name=protected_twitter_1)
def test_get_friends_list(self):
"""Test getting list of users authenticated user then random user is
following succeeds"""
self.api.get_friends_list()
self.api.get_friends_list(screen_name='twitter')
def test_get_followers_list(self):
"""Test getting list of users authenticated user then random user are
followed by succeeds"""
self.api.get_followers_list()
self.api.get_followers_list(screen_name='twitter')
# Users
def test_get_account_settings(self):
"""Test getting the authenticated user account settings succeeds"""
self.api.get_account_settings()
def test_verify_credentials(self):
"""Test representation of the authenticated user call succeeds"""
self.api.verify_credentials()
def test_update_account_settings(self):
"""Test updating a user account settings succeeds"""
self.api.update_account_settings(lang='en')
def test_update_delivery_service(self):
"""Test updating delivery settings fails because we don't have
a mobile number on the account"""
self.assertRaises(TwythonError, self.api.update_delivery_service,
device='none')
def test_update_profile(self):
"""Test updating profile succeeds"""
self.api.update_profile(include_entities='true')
def test_update_profile_colors(self):
"""Test updating profile colors succeeds"""
self.api.update_profile_colors(profile_background_color='3D3D3D')
def test_list_blocks(self):
"""Test listing users who are blocked by the authenticated user
succeeds"""
self.api.list_blocks()
def test_list_block_ids(self):
"""Test listing user ids who are blocked by the authenticated user
succeeds"""
self.api.list_block_ids()
def test_create_block(self):
"""Test blocking a user succeeds"""
self.api.create_block(screen_name='justinbieber')
def test_destroy_block(self):
"""Test unblocking a user succeeds"""
self.api.destroy_block(screen_name='justinbieber')
def test_lookup_user(self):
"""Test listing a number of user objects succeeds"""
self.api.lookup_user(screen_name='twitter,justinbieber')
def test_show_user(self):
"""Test showing one user works"""
self.api.show_user(screen_name='twitter')
def test_search_users(self):
"""Test that searching for users succeeds"""
self.api.search_users(q='Twitter API')
def test_get_contributees(self):
"""Test returning list of accounts the specified user can
contribute to succeeds"""
self.api.get_contributees(screen_name='TechCrunch')
def test_get_contributors(self):
"""Test returning list of accounts that contribute to the
authenticated user fails because we are not a Contributor account"""
self.assertRaises(TwythonError, self.api.get_contributors,
screen_name=screen_name)
def test_remove_profile_banner(self):
"""Test removing profile banner succeeds"""
self.api.remove_profile_banner()
def test_get_profile_banner_sizes(self):
"""Test getting list of profile banner sizes fails because
we have not uploaded a profile banner"""
self.assertRaises(TwythonError, self.api.get_profile_banner_sizes)
# Suggested Users
def test_get_user_suggestions_by_slug(self):
"""Test getting user suggestions by slug succeeds"""
self.api.get_user_suggestions_by_slug(slug='twitter')
def test_get_user_suggestions(self):
"""Test getting user suggestions succeeds"""
self.api.get_user_suggestions()
def test_get_user_suggestions_statuses_by_slug(self):
"""Test getting status of suggested users succeeds"""
self.api.get_user_suggestions_statuses_by_slug(slug='funny')
# Favorites
def test_get_favorites(self):
"""Test getting list of favorites for the authenticated
user succeeds"""
self.api.get_favorites()
def test_create_and_destroy_favorite(self):
"""Test creating and destroying a favorite on a tweet succeeds"""
self.api.create_favorite(id=test_tweet_id)
self.api.destroy_favorite(id=test_tweet_id)
# Lists
def test_show_lists(self):
"""Test show lists for specified user"""
self.api.show_lists(screen_name='twitter')
def test_get_list_statuses(self):
"""Test timeline of tweets authored by members of the
specified list succeeds"""
self.api.get_list_statuses(list_id=test_list_id)
def test_create_update_destroy_list_add_remove_list_members(self):
"""Test create a list, adding and removing members then
deleting the list succeeds"""
the_list = self.api.create_list(name='Stuff')
list_id = the_list['id_str']
self.api.update_list(list_id=list_id, name='Stuff Renamed')
screen_names = ['johncena', 'xbox']
# Multi add/delete members
self.api.create_list_members(list_id=list_id,
screen_name=screen_names)
self.api.delete_list_members(list_id=list_id,
screen_name=screen_names)
# Single add/delete member
self.api.add_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list(list_id=list_id)
def test_get_list_subscribers(self):
"""Test list of subscribers of a specific list succeeds"""
self.api.get_list_subscribers(list_id=test_list_id)
def test_subscribe_is_subbed_and_unsubscribe_to_list(self):
"""Test subscribing, is a list sub and unsubbing to list succeeds"""
self.api.subscribe_to_list(list_id=test_list_id)
# Returns 404 if user is not a subscriber
self.api.is_list_subscriber(list_id=test_list_id,
screen_name=screen_name)
self.api.unsubscribe_from_list(list_id=test_list_id)
def test_is_list_member(self):
"""Test returning if specified user is member of a list succeeds"""
# Returns 404 if not list member
self.api.is_list_member(list_id=test_list_id, screen_name='jack')
def test_get_list_members(self):
"""Test listing members of the specified list succeeds"""
self.api.get_list_members(list_id=test_list_id)
def test_get_specific_list(self):
"""Test getting specific list succeeds"""
self.api.get_specific_list(list_id=test_list_id)
def test_get_list_subscriptions(self):
"""Test collection of the lists the specified user is
subscribed to succeeds"""
self.api.get_list_subscriptions(screen_name='twitter')
def test_show_owned_lists(self):
"""Test collection of lists the specified user owns succeeds"""
self.api.show_owned_lists(screen_name='twitter')
# Saved Searches
def test_get_saved_searches(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
self.api.get_saved_searches()
def test_create_get_destroy_saved_search(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
saved_search = self.api.create_saved_search(query='#Twitter')
saved_search_id = saved_search['id_str']
self.api.show_saved_search(id=saved_search_id)
self.api.destroy_saved_search(id=saved_search_id)
# Places & Geo
def test_get_geo_info(self):
"""Test getting info about a geo location succeeds"""
self.api.get_geo_info(place_id='df51dec6f4ee2b2c')
def test_reverse_geo_code(self):
"""Test reversing geocode succeeds"""
self.api.reverse_geocode(lat='37.76893497', long='-122.42284884')
def test_search_geo(self):
"""Test search for places that can be attached
to a statuses/update succeeds"""
self.api.search_geo(query='Toronto')
def test_get_similar_places(self):
"""Test locates places near the given coordinates which
are similar in name succeeds"""
self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ')
# Trends
def test_get_place_trends(self):
"""Test getting the top 10 trending topics for a specific
WOEID succeeds"""
self.api.get_place_trends(id=1)
def test_get_available_trends(self):
"""Test returning locations that Twitter has trending
topic information for succeeds"""
self.api.get_available_trends()
def test_get_closest_trends(self):
"""Test getting the locations that Twitter has trending topic
information for, closest to a specified location succeeds"""
self.api.get_closest_trends(lat='37', long='-122')
# Help
def test_get_application_rate_limit_status(self):
"""Test getting application rate limit status succeeds"""
self.oauth2_api.get_application_rate_limit_status()
def get_following_to_db(dbpath, sn_table = "tweets9_users"):
distinct = get_D_UsrID(dbpath,sn_table )
index=0
num_checked = 0
rate_ex = 0
ind = 0
already_checked = False
follower_cursor = -1
friend_cursor = -1
print "Starting @ %d"%(ind)
print datetime.datetime.now()
while ind < len(distinct):
#for i in distinct:
i = distinct[ind][0]
APP_KEY = APP_KEYS[index]
APP_SECRET = APP_SECRETS[index]
OAUTH_TOKEN = OAUTH_TOKENS[index]
OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index]
try:
#twitter = Twython (APP_KEY, APP_SECRET)
#auth = twitter.get_authentication_tokens()
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
#friend_results = twitter.cursor(twitter.get_friends_ids, id = i, cursor= friend_cursor)
friends = ""
while (friend_cursor):
following = twitter.get_friends_ids(id = i,cursor= friend_cursor)
for ID in following['ids']:
friends += str(ID) + " "
friend_cursor = following["next_cursor"]
num_checked += 1
friends = friends[:-1]
if len(friends) == 0:
friends = "null"
follow = ""
while (follower_cursor):
followers = twitter.get_followers_ids(id = i,cursor= follower_cursor)
for ID in followers['ids']:
follow += str(ID) + " "
follower_cursor = followers["next_cursor"]
num_checked += 1
follow= follow[:-1]
if len(follow) == 0:
follow = "null"
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "UPDATE %s SET Following = '%s' where Usr_ID = '%s'"% (sn_table, friends, i)
cursor.execute(query)
#print query
print "____________________ %dth following updated ________ %d" % (ind, i)
conn.commit()
query = "UPDATE %s SET Followers = '%s' where Usr_ID = '%s' "% (sn_table, follow, i)
cursor.execute(query)
#print query
conn.commit()
conn.close()
print "%dth follower updated_______________________ %d" % (ind, i)
print num_checked
print
follower_cursor = -1
friend_cursor = -1
ind+=1
except Exception as e:
print e
if "429 (Too Many Requests)" in str(e):
index += 1
if index == len(APP_KEYS):
index = 0
print "sleepy time - 15 minutes"
print datetime.datetime.now()
time.sleep(910)
elif "401 (Unauthorized)" in str(e):
print "401 error"
f = open("skipped.txt", "a")
f.write("skipped %dth element, ID: %d\n"%(ind, i))
f.write("__________________________________________"+str(datetime.datetime.now()) + "\n")
f.close()
print "401 nulled %d"%ind
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "UPDATE %s SET Following = 'null', Followers = 'null' where Usr_ID = '%s'"% (sn_table, i)
cursor.execute(query)
conn.commit()
conn.close()
ind+=1
time.sleep(1)
elif "404 (Not Found)" in str(e):
print "404 error"
f = open("skipped.txt", "a")
f.write("404: skipped %dth element, ID: %d \n"%(ind, i))
f.write("__________________________________________"+str(datetime.datetime.now()) + "\n")
f.close()
print "404 nulled %d"%ind
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "UPDATE %s SET Following = 'null', Followers = 'null' where Usr_ID = '%s'"% (sn_table, i)
cursor.execute(query)
conn.commit()
conn.close()
ind+=1
time.sleep(1)
else:
print e
rate_ex += 1
print ind
print num_checked
print rate_ex
'''
# Get your keys from https://dev.twitter.com/ # Change the api key values marked ##### below with the twitter keys # Schedule twitterFF.py using crontab or another task scheduler to run every Friday. import time import random from twython import Twython CONSUMER_KEY = '####' CONSUMER_SECRET = '####' ACCESS_KEY = '####' ACCESS_SECRET = '####' api = Twython(CONSUMER_KEY,CONSUMER_SECRET,ACCESS_KEY,ACCESS_SECRET) friends = api.get_friends_ids(screen_name="yourtwitterhandle") new_list = [] for x in friends["ids"]: new_list.append(x) #print random id 6 times ff = '' for y in range(1,7): id_position = random.randint(0, len(new_list) -1) userid = new_list[id_position] data = api.show_user(user_id=userid) following = data["screen_name"] ff = ff + ' ' + '@' + following api.update_status(status='#FF ' + ff)
API_SECRET = 'INSERT HERE'
ACCESS_TOKEN = 'INSERT HERE'
ACCESS_TOKEN_SECRET = 'INSERT HERE'
twitter = Twython(API_KEY, API_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
return(twitter)
'''
Determining your Twitter followers and friends
'''
twitter = twitter_oauth_login()
friends_ids = twitter.get_friends_ids(count=5000)
friends_ids = friends_ids['ids']
followers_ids = twitter.get_followers_ids(count=5000)
followers_ids = followers_ids['ids']
print len(friends_ids), len(followers_ids)
friends_set = set(friends_ids)
followers_set = set(followers_ids)
print('Number of Twitter users who either are our friend or follow you (union):')
print(len(friends_set.union(followers_set)))
len(friends_set | followers_set)
def updateFollowerFollowings(dbpath, user_table = "users"):
distinct = get_D_UsrID(dbpath,sn_table )
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "select Usr_ID from %s where Following is null or Followers is null"
cursor.execute(query)
distinct = cursor.fetchall()
follower_cursor = -1
friend_cursor = -1
print "Starting @ %d"%(ind)
print datetime.datetime.now()
while ind < len(distinct):
#for i in distinct:
i = distinct[ind][0] # Usr_ID
APP_KEY = APP_KEYS[index]
APP_SECRET = APP_SECRETS[index]
OAUTH_TOKEN = OAUTH_TOKENS[index]
OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index]
try:
#twitter = Twython (APP_KEY, APP_SECRET)
#auth = twitter.get_authentication_tokens()
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
friends = ""
while (friend_cursor):
following = twitter.get_friends_ids(id = i,cursor= friend_cursor)
for ID in following['ids']:
friends += str(ID) + " "
friend_cursor = following["next_cursor"]
num_checked += 1
friends = friends[:-1]
if len(friends) == 0:
friends = "null"
follow = ""
while (follower_cursor):
followers = twitter.get_followers_ids(id = i,cursor= follower_cursor)
for ID in followers['ids']:
follow += str(ID) + " "
follower_cursor = followers["next_cursor"]
num_checked += 1
follow= follow[:-1]
if len(follow) == 0:
follow = "null"
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "UPDATE %s SET Following = '%s', Followers = '%s', NumFollowing = '%d', NumFollowers = '%d' where Usr_ID = '%s'"% (sn_table, friends,follow, len(friends.split(" ")), len(follow.split(" ")), i)
cursor.execute(query)
#print query
conn.close()
print "____________________ %dth following and friends and their corresponding counts updated ________ %d" % (ind, i)
follower_cursor = -1
friend_cursor = -1
ind+=1
except Exception as e:
print e
if "429 (Too Many Requests)" in str(e):
index += 1
if index == len(APP_KEYS):
index = 0
print "sleepy time; pausing for 15 minutes for APP Keys to renew"
print datetime.datetime.now()
#time.sleep(910)
print "App Rate Limits has been exceeded- we will wait for next runtime before running rest of followers and followings"
return
elif "401 (Unauthorized)" in str(e):
print "401 Unauthorized Error nulling %d________________Usr_ID:%d"%(ind,i)
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "UPDATE %s SET Following = 'null', Followers = 'null' where Usr_ID = '%s'"% (sn_table, i)
cursor.execute(query)
conn.commit()
conn.close()
ind+=1
elif "404 (Not Found)" in str(e):
print "404 Not Found Error nulling %d________________Usr_ID:%d"%(ind,i)
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "UPDATE %s SET Following = 'null', Followers = 'null' where Usr_ID = '%s'"% (sn_table, i)
cursor.execute(query)
conn.commit()
conn.close()
ind+=1
else:
print e
class TwythonEndpointsTestCase(unittest.TestCase):
def setUp(self):
client_args = {
'headers': {
'User-Agent': '__twython__ Test'
},
'allow_redirects': False
}
# This is so we can hit coverage that Twython sets
# User-Agent for us if none is supplied
oauth2_client_args = {
'headers': {}
}
self.api = Twython(app_key, app_secret,
oauth_token, oauth_token_secret,
client_args=client_args)
self.oauth2_api = Twython(app_key, access_token=access_token,
client_args=oauth2_client_args)
# Timelines
@unittest.skip('skipping non-updated test')
def test_get_mentions_timeline(self):
"""Test returning mentions timeline for authenticated user succeeds"""
self.api.get_mentions_timeline()
@unittest.skip('skipping non-updated test')
def test_get_user_timeline(self):
"""Test returning timeline for authenticated user and random user
succeeds"""
self.api.get_user_timeline() # Authenticated User Timeline
self.api.get_user_timeline(screen_name='twitter')
# Random User Timeline
@unittest.skip('skipping non-updated test')
def test_get_protected_user_timeline_following(self):
"""Test returning a protected user timeline who you are following
succeeds"""
self.api.get_user_timeline(screen_name=protected_twitter_1)
@unittest.skip('skipping non-updated test')
def test_get_protected_user_timeline_not_following(self):
"""Test returning a protected user timeline who you are not following
fails and raise a TwythonAuthError"""
self.assertRaises(TwythonAuthError, self.api.get_user_timeline,
screen_name=protected_twitter_2)
@unittest.skip('skipping non-updated test')
def test_retweeted_of_me(self):
"""Test that getting recent tweets by authenticated user that have
been retweeted by others succeeds"""
self.api.retweeted_of_me()
@unittest.skip('skipping non-updated test')
def test_get_home_timeline(self):
"""Test returning home timeline for authenticated user succeeds"""
self.api.get_home_timeline()
# Tweets
@unittest.skip('skipping non-updated test')
def test_get_retweets(self):
"""Test getting retweets of a specific tweet succeeds"""
self.api.get_retweets(id=test_tweet_id)
@unittest.skip('skipping non-updated test')
def test_show_status(self):
"""Test returning a single status details succeeds"""
self.api.show_status(id=test_tweet_id)
@unittest.skip('skipping non-updated test')
def test_update_and_destroy_status(self):
"""Test updating and deleting a status succeeds"""
status = self.api.update_status(status='Test post just to get \
deleted :( %s' % int(time.time()))
self.api.destroy_status(id=status['id_str'])
@unittest.skip('skipping non-updated test')
def test_get_oembed_tweet(self):
"""Test getting info to embed tweet on Third Party site succeeds"""
self.api.get_oembed_tweet(id='99530515043983360')
@unittest.skip('skipping non-updated test')
def test_get_retweeters_ids(self):
"""Test getting ids for people who retweeted a tweet succeeds"""
self.api.get_retweeters_ids(id='99530515043983360')
# Search
@unittest.skip('skipping non-updated test')
def test_search(self):
"""Test searching tweets succeeds"""
self.api.search(q='twitter')
# Direct Messages
@unittest.skip('skipping non-updated test')
def test_get_direct_messages(self):
"""Test getting the authenticated users direct messages succeeds"""
self.api.get_direct_messages()
@unittest.skip('skipping non-updated test')
def test_get_sent_messages(self):
"""Test getting the authenticated users direct messages they've
sent succeeds"""
self.api.get_sent_messages()
@unittest.skip('skipping non-updated test')
def test_send_get_and_destroy_direct_message(self):
"""Test sending, getting, then destory a direct message succeeds"""
message = self.api.send_direct_message(screen_name=protected_twitter_1,
text='Hey d00d! %s\
' % int(time.time()))
self.api.get_direct_message(id=message['id_str'])
self.api.destroy_direct_message(id=message['id_str'])
@unittest.skip('skipping non-updated test')
def test_send_direct_message_to_non_follower(self):
"""Test sending a direct message to someone who doesn't follow you
fails"""
self.assertRaises(TwythonError, self.api.send_direct_message,
screen_name=protected_twitter_2, text='Yo, man! \
%s' % int(time.time()))
# Friends & Followers
@unittest.skip('skipping non-updated test')
def test_get_user_ids_of_blocked_retweets(self):
"""Test that collection of user_ids that the authenticated user does
not want to receive retweets from succeeds"""
self.api.get_user_ids_of_blocked_retweets(stringify_ids=True)
@unittest.skip('skipping non-updated test')
def test_get_friends_ids(self):
"""Test returning ids of users the authenticated user and then a random
user is following succeeds"""
self.api.get_friends_ids()
self.api.get_friends_ids(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_get_followers_ids(self):
"""Test returning ids of users the authenticated user and then a random
user are followed by succeeds"""
self.api.get_followers_ids()
self.api.get_followers_ids(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_lookup_friendships(self):
"""Test returning relationships of the authenticating user to the
comma-separated list of up to 100 screen_names or user_ids provided
succeeds"""
self.api.lookup_friendships(screen_name='twitter,ryanmcgrath')
@unittest.skip('skipping non-updated test')
def test_get_incoming_friendship_ids(self):
"""Test returning incoming friendship ids succeeds"""
self.api.get_incoming_friendship_ids()
@unittest.skip('skipping non-updated test')
def test_get_outgoing_friendship_ids(self):
"""Test returning outgoing friendship ids succeeds"""
self.api.get_outgoing_friendship_ids()
@unittest.skip('skipping non-updated test')
def test_create_friendship(self):
"""Test creating a friendship succeeds"""
self.api.create_friendship(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_destroy_friendship(self):
"""Test destroying a friendship succeeds"""
self.api.destroy_friendship(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_update_friendship(self):
"""Test updating friendships succeeds"""
self.api.update_friendship(screen_name=protected_twitter_1,
retweets='true')
self.api.update_friendship(screen_name=protected_twitter_1,
retweets=False)
@unittest.skip('skipping non-updated test')
def test_show_friendships(self):
"""Test showing specific friendship succeeds"""
self.api.show_friendship(target_screen_name=protected_twitter_1)
@unittest.skip('skipping non-updated test')
def test_get_friends_list(self):
"""Test getting list of users authenticated user then random user is
following succeeds"""
self.api.get_friends_list()
self.api.get_friends_list(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_get_followers_list(self):
"""Test getting list of users authenticated user then random user are
followed by succeeds"""
self.api.get_followers_list()
self.api.get_followers_list(screen_name='twitter')
# Users
@unittest.skip('skipping non-updated test')
def test_get_account_settings(self):
"""Test getting the authenticated user account settings succeeds"""
self.api.get_account_settings()
@unittest.skip('skipping non-updated test')
def test_verify_credentials(self):
"""Test representation of the authenticated user call succeeds"""
self.api.verify_credentials()
@unittest.skip('skipping non-updated test')
def test_update_account_settings(self):
"""Test updating a user account settings succeeds"""
self.api.update_account_settings(lang='en')
@unittest.skip('skipping non-updated test')
def test_update_delivery_service(self):
"""Test updating delivery settings fails because we don't have
a mobile number on the account"""
self.assertRaises(TwythonError, self.api.update_delivery_service,
device='none')
@unittest.skip('skipping non-updated test')
def test_update_profile(self):
"""Test updating profile succeeds"""
self.api.update_profile(include_entities='true')
@unittest.skip('skipping non-updated test')
def test_update_profile_colors(self):
"""Test updating profile colors succeeds"""
self.api.update_profile_colors(profile_background_color='3D3D3D')
@unittest.skip('skipping non-updated test')
def test_list_blocks(self):
"""Test listing users who are blocked by the authenticated user
succeeds"""
self.api.list_blocks()
@unittest.skip('skipping non-updated test')
def test_list_block_ids(self):
"""Test listing user ids who are blocked by the authenticated user
succeeds"""
self.api.list_block_ids()
@unittest.skip('skipping non-updated test')
def test_create_block(self):
"""Test blocking a user succeeds"""
self.api.create_block(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_destroy_block(self):
"""Test unblocking a user succeeds"""
self.api.destroy_block(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_lookup_user(self):
"""Test listing a number of user objects succeeds"""
self.api.lookup_user(screen_name='twitter,justinbieber')
@unittest.skip('skipping non-updated test')
def test_show_user(self):
"""Test showing one user works"""
self.api.show_user(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_search_users(self):
"""Test that searching for users succeeds"""
self.api.search_users(q='Twitter API')
@unittest.skip('skipping non-updated test')
def test_get_contributees(self):
"""Test returning list of accounts the specified user can
contribute to succeeds"""
self.api.get_contributees(screen_name='TechCrunch')
@unittest.skip('skipping non-updated test')
def test_get_contributors(self):
"""Test returning list of accounts that contribute to the
authenticated user fails because we are not a Contributor account"""
self.assertRaises(TwythonError, self.api.get_contributors,
screen_name=screen_name)
@unittest.skip('skipping non-updated test')
def test_remove_profile_banner(self):
"""Test removing profile banner succeeds"""
self.api.remove_profile_banner()
@unittest.skip('skipping non-updated test')
def test_get_profile_banner_sizes(self):
"""Test getting list of profile banner sizes fails because
we have not uploaded a profile banner"""
self.assertRaises(TwythonError, self.api.get_profile_banner_sizes)
@unittest.skip('skipping non-updated test')
def test_list_mutes(self):
"""Test listing users who are muted by the authenticated user
succeeds"""
self.api.list_mutes()
@unittest.skip('skipping non-updated test')
def test_list_mute_ids(self):
"""Test listing user ids who are muted by the authenticated user
succeeds"""
self.api.list_mute_ids()
@unittest.skip('skipping non-updated test')
def test_create_mute(self):
"""Test muting a user succeeds"""
self.api.create_mute(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_destroy_mute(self):
"""Test muting a user succeeds"""
self.api.destroy_mute(screen_name='justinbieber')
# Suggested Users
@unittest.skip('skipping non-updated test')
def test_get_user_suggestions_by_slug(self):
"""Test getting user suggestions by slug succeeds"""
self.api.get_user_suggestions_by_slug(slug='twitter')
@unittest.skip('skipping non-updated test')
def test_get_user_suggestions(self):
"""Test getting user suggestions succeeds"""
self.api.get_user_suggestions()
@unittest.skip('skipping non-updated test')
def test_get_user_suggestions_statuses_by_slug(self):
"""Test getting status of suggested users succeeds"""
self.api.get_user_suggestions_statuses_by_slug(slug='funny')
# Favorites
@unittest.skip('skipping non-updated test')
def test_get_favorites(self):
"""Test getting list of favorites for the authenticated
user succeeds"""
self.api.get_favorites()
@unittest.skip('skipping non-updated test')
def test_create_and_destroy_favorite(self):
"""Test creating and destroying a favorite on a tweet succeeds"""
self.api.create_favorite(id=test_tweet_id)
self.api.destroy_favorite(id=test_tweet_id)
# Lists
@unittest.skip('skipping non-updated test')
def test_show_lists(self):
"""Test show lists for specified user"""
self.api.show_lists(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_get_list_statuses(self):
"""Test timeline of tweets authored by members of the
specified list succeeds"""
self.api.get_list_statuses(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_create_update_destroy_list_add_remove_list_members(self):
"""Test create a list, adding and removing members then
deleting the list succeeds"""
the_list = self.api.create_list(name='Stuff %s' % int(time.time()))
list_id = the_list['id_str']
self.api.update_list(list_id=list_id, name='Stuff Renamed \
%s' % int(time.time()))
screen_names = ['johncena', 'xbox']
# Multi add/delete members
self.api.create_list_members(list_id=list_id,
screen_name=screen_names)
self.api.delete_list_members(list_id=list_id,
screen_name=screen_names)
# Single add/delete member
self.api.add_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list_member(list_id=list_id,
screen_name='justinbieber')
self.api.delete_list(list_id=list_id)
@unittest.skip('skipping non-updated test')
def test_get_list_memberships(self):
"""Test list of memberhips the authenticated user succeeds"""
self.api.get_list_memberships()
@unittest.skip('skipping non-updated test')
def test_get_list_subscribers(self):
"""Test list of subscribers of a specific list succeeds"""
self.api.get_list_subscribers(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_subscribe_is_subbed_and_unsubscribe_to_list(self):
"""Test subscribing, is a list sub and unsubbing to list succeeds"""
self.api.subscribe_to_list(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
# Returns 404 if user is not a subscriber
self.api.is_list_subscriber(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name,
screen_name=screen_name)
self.api.unsubscribe_from_list(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_is_list_member(self):
"""Test returning if specified user is member of a list succeeds"""
# Returns 404 if not list member
self.api.is_list_member(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name,
screen_name='themattharris')
@unittest.skip('skipping non-updated test')
def test_get_list_members(self):
"""Test listing members of the specified list succeeds"""
self.api.get_list_members(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_get_specific_list(self):
"""Test getting specific list succeeds"""
self.api.get_specific_list(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_get_list_subscriptions(self):
"""Test collection of the lists the specified user is
subscribed to succeeds"""
self.api.get_list_subscriptions(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_show_owned_lists(self):
"""Test collection of lists the specified user owns succeeds"""
self.api.show_owned_lists(screen_name='twitter')
# Saved Searches
@unittest.skip('skipping non-updated test')
def test_get_saved_searches(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
self.api.get_saved_searches()
@unittest.skip('skipping non-updated test')
def test_create_get_destroy_saved_search(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
saved_search = self.api.create_saved_search(query='#Twitter')
saved_search_id = saved_search['id_str']
self.api.show_saved_search(id=saved_search_id)
self.api.destroy_saved_search(id=saved_search_id)
# Places & Geo
@unittest.skip('skipping non-updated test')
def test_get_geo_info(self):
"""Test getting info about a geo location succeeds"""
self.api.get_geo_info(place_id='df51dec6f4ee2b2c')
@unittest.skip('skipping non-updated test')
def test_reverse_geo_code(self):
"""Test reversing geocode succeeds"""
self.api.reverse_geocode(lat='37.76893497', long='-122.42284884')
@unittest.skip('skipping non-updated test')
def test_search_geo(self):
"""Test search for places that can be attached
to a statuses/update succeeds"""
self.api.search_geo(query='Toronto')
@unittest.skip('skipping non-updated test')
def test_get_similar_places(self):
"""Test locates places near the given coordinates which
are similar in name succeeds"""
self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ')
# Trends
@unittest.skip('skipping non-updated test')
def test_get_place_trends(self):
"""Test getting the top 10 trending topics for a specific
WOEID succeeds"""
self.api.get_place_trends(id=1)
@unittest.skip('skipping non-updated test')
def test_get_available_trends(self):
"""Test returning locations that Twitter has trending
topic information for succeeds"""
self.api.get_available_trends()
@unittest.skip('skipping non-updated test')
def test_get_closest_trends(self):
"""Test getting the locations that Twitter has trending topic
information for, closest to a specified location succeeds"""
self.api.get_closest_trends(lat='37', long='-122')
# Help
@unittest.skip('skipping non-updated test')
def test_get_twitter_configuration(self):
"""Test getting Twitter's configuration succeeds"""
self.api.get_twitter_configuration()
@unittest.skip('skipping non-updated test')
def test_get_supported_languages(self):
"""Test getting languages supported by Twitter succeeds"""
self.api.get_supported_languages()
@unittest.skip('skipping non-updated test')
def test_privacy_policy(self):
"""Test getting Twitter's Privacy Policy succeeds"""
self.api.get_privacy_policy()
@unittest.skip('skipping non-updated test')
def test_get_tos(self):
"""Test getting the Twitter Terms of Service succeeds"""
self.api.get_tos()
@unittest.skip('skipping non-updated test')
def test_get_application_rate_limit_status(self):
"""Test getting application rate limit status succeeds"""
self.oauth2_api.get_application_rate_limit_status()
def get_active_users_around(center, start_date, end_date, threshold, num_friends, num_followers, waiting):
twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN)
user_ids = []
all_tweets = []
# get friends and followers around a central user
print "Getting users..."
while True:
try:
following = twitter.get_friends_ids(screen_name=center, count=5000)
time.sleep(0.1)
followers = twitter.get_followers_ids(screen_name=center, count=5000)
break
except TwythonAuthError as e:
print "TwythonAuthError..."
time.sleep(waiting)
continue
except TwythonRateLimitError as e:
print "TwythonRateLimitError... Waiting for 3 minutes (may take multiple waits)"
for timer in tqdm(range(3*60)):
time.sleep(1)
twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN)
time.sleep(waiting)
continue
except TwythonError as e:
print "TwythonError: " + str(e.error_code) + "..."
time.sleep(waiting)
continue
# choose out random friends and followers from the previous list, with a safety factor of 2 times as many as asked for to account for the fact that some users will not be active
following = map(lambda x: str(x), np.random.choice(following['ids'], min(int(num_friends * 2), len(following['ids'])), replace=False))
followers = map(lambda x: str(x), np.random.choice(followers['ids'], min(int(num_followers * 2), len(followers['ids'])), replace=False))
following = get_usernames(following, waiting)
followers = get_usernames(followers, waiting)
# figure out how many tweets are necessary to deem a user "active"
start = datetime.strptime(start_date, "%Y-%m-%d")
end = datetime.strptime(end_date, "%Y-%m-%d")
threshold = abs(end-start).days * threshold
# collect out as many active followed users from the central user (and their tweets) as we can, up to a limit of the number asked for
collected_friends = []
for user in following:
user_timeline = get_timeline_in_range(user, start_date, end_date, waiting)
if len(user_timeline) >= threshold:
collected_friends.append(user)
all_tweets.extend(user_timeline)
if len(collected_friends) >= num_friends:
break
# collect out as many active followers from the central user (and their tweets) as we can, up to a limit of the number asked for
collected_followers = []
for user in followers:
user_timeline = get_timeline_in_range(user, start_date, end_date, waiting)
if len(user_timeline) >= threshold:
collected_followers.append(user)
all_tweets.extend(user_timeline)
if len(collected_followers) >= num_followers:
break
# return our list of users and their tweets from the time period in question
user_ids.extend(collected_friends)
user_ids.extend(collected_followers)
return (user_ids, all_tweets)
cur.execute("SELECT Profiles.idFollower FROM Profiles LEFT OUTER JOIN Users on Profiles.idFollower = Users.idUser WHERE Users.idUser is null GROUP BY Profiles.idFollower LIMIT 0,200")
rows = cur.fetchall()
currentAccount = 0
iRow = 1
for row in rows:
cur2 = con.cursor()
cur2.execute("SELECT Count(*) FROM Users WHERE idUser="******"idFollower"]))
numberOfRows = cur2.fetchone()[0]
if(numberOfRows==0):
try:
error = False
friends = twitter.get_friends_ids(user_id = row["idFollower"])
except TwythonAuthError:
error = True
cur4 = con.cursor()
cur4.execute("DELETE FROM Users WHERE idUser="******"idFollower"]))
cur4.execute("DELETE FROM Profiles WHERE idFollower="+str(row["idFollower"]))
except TwythonRateLimitError:
error = True
currentAccount+=1
if currentAccount == len(APP_KEY):
print 'Limit excedeed and all accounts have been used, quitting ...'
con.commit()
con.close()
execfile('/Users/cyriltrosset/Dropbox/IIT/Special project/ctrosset/remainRows.py')
sys.exit()
else:
def get_following():
distinct = getScreenname.getDistinctScreenames(
"all_tweets_dict_fully_checked")
index = 0
num_checked = 0
rate_ex = 0
for sname in distinct:
file = open("following.txt", "r+")
already_checked = False
for line in file:
words = line.split('\t')
if words[0] == sname:
already_checked = True
if index >= 8:
break
APP_KEY = APP_KEYS[index]
APP_SECRET = APP_SECRETS[index]
OAUTH_TOKEN = OAUTH_TOKENS[index]
OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index]
if not already_checked:
try:
twitter = Twython(APP_KEY, APP_SECRET)
auth = twitter.get_authentication_tokens()
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN,
OAUTH_TOKEN_SECRET)
following = twitter.get_friends_ids(screen_name=sname)
print sname
file.write(sname)
file.write("\t")
for ID in following['ids']:
file.write(str(ID) + " ")
file.write("\n")
num_checked += 1
except Exception as e:
if "429 (Too Many Requests)" in str(e):
index += 1
elif "401 (Unauthorized)" in str(e):
print "401 error"
file.write(sname)
file.write("\t")
file.write("Unauthorized")
file.write("\n")
elif "404 (Not Found)" in str(e):
print "404 error"
file.write(sname)
file.write("\t")
file.write("NotFound")
file.write("\n")
else:
print e
rate_ex += 1
file.close()
if (num_checked != 0 or rate_ex != 0):
print "sleepy time"
time.sleep(900)
get_following()
def updateFollowerFollowings(dbpath, user_table="users"):
distinct = get_D_UsrID(dbpath, sn_table)
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "select Usr_ID from %s where Following is null or Followers is null"
cursor.execute(query)
distinct = cursor.fetchall()
follower_cursor = -1
friend_cursor = -1
print "Starting @ %d" % (ind)
print datetime.datetime.now()
while ind < len(distinct):
#for i in distinct:
i = distinct[ind][0] # Usr_ID
APP_KEY = APP_KEYS[index]
APP_SECRET = APP_SECRETS[index]
OAUTH_TOKEN = OAUTH_TOKENS[index]
OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index]
try:
#twitter = Twython (APP_KEY, APP_SECRET)
#auth = twitter.get_authentication_tokens()
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN,
OAUTH_TOKEN_SECRET)
friends = ""
while (friend_cursor):
following = twitter.get_friends_ids(id=i, cursor=friend_cursor)
for ID in following['ids']:
friends += str(ID) + " "
friend_cursor = following["next_cursor"]
num_checked += 1
friends = friends[:-1]
if len(friends) == 0:
friends = "null"
follow = ""
while (follower_cursor):
followers = twitter.get_followers_ids(id=i,
cursor=follower_cursor)
for ID in followers['ids']:
follow += str(ID) + " "
follower_cursor = followers["next_cursor"]
num_checked += 1
follow = follow[:-1]
if len(follow) == 0:
follow = "null"
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "UPDATE %s SET Following = '%s', Followers = '%s', NumFollowing = '%d', NumFollowers = '%d' where Usr_ID = '%s'" % (
sn_table, friends, follow, len(
friends.split(" ")), len(follow.split(" ")), i)
cursor.execute(query)
#print query
conn.close()
print "____________________ %dth following and friends and their corresponding counts updated ________ %d" % (
ind, i)
follower_cursor = -1
friend_cursor = -1
ind += 1
except Exception as e:
print e
if "429 (Too Many Requests)" in str(e):
index += 1
if index == len(APP_KEYS):
index = 0
print "sleepy time; pausing for 15 minutes for APP Keys to renew"
print datetime.datetime.now()
#time.sleep(910)
print "App Rate Limits has been exceeded- we will wait for next runtime before running rest of followers and followings"
return
elif "401 (Unauthorized)" in str(e):
print "401 Unauthorized Error nulling %d________________Usr_ID:%d" % (
ind, i)
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "UPDATE %s SET Following = 'null', Followers = 'null' where Usr_ID = '%s'" % (
sn_table, i)
cursor.execute(query)
conn.commit()
conn.close()
ind += 1
elif "404 (Not Found)" in str(e):
print "404 Not Found Error nulling %d________________Usr_ID:%d" % (
ind, i)
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
query = "UPDATE %s SET Following = 'null', Followers = 'null' where Usr_ID = '%s'" % (
sn_table, i)
cursor.execute(query)
conn.commit()
conn.close()
ind += 1
else:
print e
class TwythonAPITestCase(unittest.TestCase):
def setUp(self):
client_args = {
'headers': {
'User-Agent': '__twython__ Test'
},
'allow_redirects': False
}
oauth2_client_args = {
'headers': {
} # This is so we can hit coverage that Twython sets User-Agent for us if none is supplied
}
self.api = Twython(app_key,
app_secret,
oauth_token,
oauth_token_secret,
client_args=client_args)
self.oauth2_api = Twython(app_key,
access_token=access_token,
client_args=oauth2_client_args)
def test_construct_api_url(self):
"""Test constructing a Twitter API url works as we expect"""
url = 'https://api.twitter.com/1.1/search/tweets.json'
constructed_url = self.api.construct_api_url(url, q='#twitter')
self.assertEqual(
constructed_url,
'https://api.twitter.com/1.1/search/tweets.json?q=%23twitter')
def test_get(self):
"""Test Twython generic GET request works"""
self.api.get('account/verify_credentials')
def test_post(self):
"""Test Twython generic POST request works, with a full url and
with just an endpoint"""
update_url = 'https://api.twitter.com/1.1/statuses/update.json'
status = self.api.post(
update_url,
params={'status': 'I love Twython! %s' % int(time.time())})
self.api.post('statuses/destroy/%s' % status['id_str'])
def test_get_lastfunction_header(self):
"""Test getting last specific header of the last API call works"""
self.api.get('statuses/home_timeline')
self.api.get_lastfunction_header('x-rate-limit-remaining')
def test_get_lastfunction_header_not_present(self):
"""Test getting specific header that does not exist from the last call returns None"""
self.api.get('statuses/home_timeline')
header = self.api.get_lastfunction_header('does-not-exist')
self.assertEqual(header, None)
def test_get_lastfunction_header_no_last_api_call(self):
"""Test attempting to get a header when no API call was made raises a TwythonError"""
self.assertRaises(TwythonError, self.api.get_lastfunction_header,
'no-api-call-was-made')
def test_cursor(self):
"""Test looping through the generator results works, at least once that is"""
search = self.api.cursor(self.api.search, q='twitter', count=1)
counter = 0
while counter < 2:
counter += 1
result = next(search)
new_id_str = int(result['id_str'])
if counter == 1:
prev_id_str = new_id_str
time.sleep(
1) # Give time for another tweet to come into search
if counter == 2:
self.assertTrue(new_id_str > prev_id_str)
def test_encode(self):
"""Test encoding UTF-8 works"""
self.api.encode('Twython is awesome!')
def test_html_for_tweet(self):
"""Test HTML for Tweet returns what we want"""
tweet_text = self.api.html_for_tweet(test_tweet_object)
self.assertEqual(test_tweet_html, tweet_text)
def test_html_for_tweet_expanded_url(self):
"""Test using expanded url in HTML for Tweet displays full urls"""
tweet_text = self.api.html_for_tweet(test_tweet_object,
use_expanded_url=True)
# Make sure full url is in HTML
self.assertTrue('http://google.com' in tweet_text)
def test_html_for_tweet_short_url(self):
"""Test using expanded url in HTML for Tweet displays full urls"""
tweet_text = self.api.html_for_tweet(test_tweet_object, False)
# Make sure HTML doesn't contain the display OR exapanded url
self.assertTrue(not 'http://google.com' in tweet_text)
self.assertTrue(not 'google.com' in tweet_text)
def test_raise_error_on_bad_ssl_cert(self):
"""Test TwythonError is raised by a RequestException when an actual HTTP happens"""
self.assertRaises(TwythonError, self.api.get, 'https://example.com')
# Timelines
def test_get_mentions_timeline(self):
"""Test returning mentions timeline for authenticated user succeeds"""
self.api.get_mentions_timeline()
def test_get_user_timeline(self):
"""Test returning timeline for authenticated user and random user
succeeds"""
self.api.get_user_timeline() # Authenticated User Timeline
self.api.get_user_timeline(
screen_name='twitter') # Random User Timeline
def test_get_protected_user_timeline_following(self):
"""Test returning a protected user timeline who you are following
succeeds"""
self.api.get_user_timeline(screen_name=protected_twitter_1)
def test_get_protected_user_timeline_not_following(self):
"""Test returning a protected user timeline who you are not following
fails and raise a TwythonAuthError"""
self.assertRaises(TwythonAuthError,
self.api.get_user_timeline,
screen_name=protected_twitter_2)
def test_retweeted_of_me(self):
"""Test that getting recent tweets by authenticated user that have
been retweeted by others succeeds"""
self.api.retweeted_of_me()
def test_get_home_timeline(self):
"""Test returning home timeline for authenticated user succeeds"""
self.api.get_home_timeline()
# Tweets
def test_get_retweets(self):
"""Test getting retweets of a specific tweet succeeds"""
self.api.get_retweets(id=test_tweet_id)
def test_show_status(self):
"""Test returning a single status details succeeds"""
self.api.show_status(id=test_tweet_id)
def test_update_and_destroy_status(self):
"""Test updating and deleting a status succeeds"""
status = self.api.update_status(
status='Test post just to get deleted :( %s' % int(time.time()))
self.api.destroy_status(id=status['id_str'])
def test_get_oembed_tweet(self):
"""Test getting info to embed tweet on Third Party site succeeds"""
self.api.get_oembed_tweet(id='99530515043983360')
def test_get_retweeters_ids(self):
"""Test getting ids for people who retweeted a tweet succeeds"""
self.api.get_retweeters_ids(id='99530515043983360')
# Search
def test_search(self):
"""Test searching tweets succeeds"""
self.api.search(q='twitter')
# Direct Messages
def test_get_direct_messages(self):
"""Test getting the authenticated users direct messages succeeds"""
self.api.get_direct_messages()
def test_get_sent_messages(self):
"""Test getting the authenticated users direct messages they've
sent succeeds"""
self.api.get_sent_messages()
def test_send_get_and_destroy_direct_message(self):
"""Test sending, getting, then destory a direct message succeeds"""
message = self.api.send_direct_message(screen_name=protected_twitter_1,
text='Hey d00d! %s' %
int(time.time()))
self.api.get_direct_message(id=message['id_str'])
self.api.destroy_direct_message(id=message['id_str'])
def test_send_direct_message_to_non_follower(self):
"""Test sending a direct message to someone who doesn't follow you
fails"""
self.assertRaises(TwythonError,
self.api.send_direct_message,
screen_name=protected_twitter_2,
text='Yo, man! %s' % int(time.time()))
# Friends & Followers
def test_get_user_ids_of_blocked_retweets(self):
"""Test that collection of user_ids that the authenticated user does
not want to receive retweets from succeeds"""
self.api.get_user_ids_of_blocked_retweets(stringify_ids=True)
def test_get_friends_ids(self):
"""Test returning ids of users the authenticated user and then a random
user is following succeeds"""
self.api.get_friends_ids()
self.api.get_friends_ids(screen_name='twitter')
def test_get_followers_ids(self):
"""Test returning ids of users the authenticated user and then a random
user are followed by succeeds"""
self.api.get_followers_ids()
self.api.get_followers_ids(screen_name='twitter')
def test_lookup_friendships(self):
"""Test returning relationships of the authenticating user to the
comma-separated list of up to 100 screen_names or user_ids provided
succeeds"""
self.api.lookup_friendships(screen_name='twitter,ryanmcgrath')
def test_get_incoming_friendship_ids(self):
"""Test returning incoming friendship ids succeeds"""
self.api.get_incoming_friendship_ids()
def test_get_outgoing_friendship_ids(self):
"""Test returning outgoing friendship ids succeeds"""
self.api.get_outgoing_friendship_ids()
def test_create_friendship(self):
"""Test creating a friendship succeeds"""
self.api.create_friendship(screen_name='justinbieber')
def test_destroy_friendship(self):
"""Test destroying a friendship succeeds"""
self.api.destroy_friendship(screen_name='justinbieber')
def test_update_friendship(self):
"""Test updating friendships succeeds"""
self.api.update_friendship(screen_name=protected_twitter_1,
retweets='true')
self.api.update_friendship(screen_name=protected_twitter_1,
retweets=False)
def test_show_friendships(self):
"""Test showing specific friendship succeeds"""
self.api.show_friendship(target_screen_name=protected_twitter_1)
def test_get_friends_list(self):
"""Test getting list of users authenticated user then random user is
following succeeds"""
self.api.get_friends_list()
self.api.get_friends_list(screen_name='twitter')
def test_get_followers_list(self):
"""Test getting list of users authenticated user then random user are
followed by succeeds"""
self.api.get_followers_list()
self.api.get_followers_list(screen_name='twitter')
# Users
def test_get_account_settings(self):
"""Test getting the authenticated user account settings succeeds"""
self.api.get_account_settings()
def test_verify_credentials(self):
"""Test representation of the authenticated user call succeeds"""
self.api.verify_credentials()
def test_update_account_settings(self):
"""Test updating a user account settings succeeds"""
self.api.update_account_settings(lang='en')
def test_update_delivery_service(self):
"""Test updating delivery settings fails because we don't have
a mobile number on the account"""
self.assertRaises(TwythonError,
self.api.update_delivery_service,
device='none')
def test_update_profile(self):
"""Test updating profile succeeds"""
self.api.update_profile(include_entities='true')
def test_update_profile_colors(self):
"""Test updating profile colors succeeds"""
self.api.update_profile_colors(profile_background_color='3D3D3D')
def test_list_blocks(self):
"""Test listing users who are blocked by the authenticated user
succeeds"""
self.api.list_blocks()
def test_list_block_ids(self):
"""Test listing user ids who are blocked by the authenticated user
succeeds"""
self.api.list_block_ids()
def test_create_block(self):
"""Test blocking a user succeeds"""
self.api.create_block(screen_name='justinbieber')
def test_destroy_block(self):
"""Test unblocking a user succeeds"""
self.api.destroy_block(screen_name='justinbieber')
def test_lookup_user(self):
"""Test listing a number of user objects succeeds"""
self.api.lookup_user(screen_name='twitter,justinbieber')
def test_show_user(self):
"""Test showing one user works"""
self.api.show_user(screen_name='twitter')
def test_search_users(self):
"""Test that searching for users succeeds"""
self.api.search_users(q='Twitter API')
def test_get_contributees(self):
"""Test returning list of accounts the specified user can
contribute to succeeds"""
self.api.get_contributees(screen_name='TechCrunch')
def test_get_contributors(self):
"""Test returning list of accounts that contribute to the
authenticated user fails because we are not a Contributor account"""
self.assertRaises(TwythonError,
self.api.get_contributors,
screen_name=screen_name)
def test_remove_profile_banner(self):
"""Test removing profile banner succeeds"""
self.api.remove_profile_banner()
def test_get_profile_banner_sizes(self):
"""Test getting list of profile banner sizes fails because
we have not uploaded a profile banner"""
self.assertRaises(TwythonError, self.api.get_profile_banner_sizes)
# Suggested Users
def test_get_user_suggestions_by_slug(self):
"""Test getting user suggestions by slug succeeds"""
self.api.get_user_suggestions_by_slug(slug='twitter')
def test_get_user_suggestions(self):
"""Test getting user suggestions succeeds"""
self.api.get_user_suggestions()
def test_get_user_suggestions_statuses_by_slug(self):
"""Test getting status of suggested users succeeds"""
self.api.get_user_suggestions_statuses_by_slug(slug='funny')
# Favorites
def test_get_favorites(self):
"""Test getting list of favorites for the authenticated
user succeeds"""
self.api.get_favorites()
def test_create_and_destroy_favorite(self):
"""Test creating and destroying a favorite on a tweet succeeds"""
self.api.create_favorite(id=test_tweet_id)
self.api.destroy_favorite(id=test_tweet_id)
# Lists
def test_show_lists(self):
"""Test show lists for specified user"""
self.api.show_lists(screen_name='twitter')
def test_get_list_statuses(self):
"""Test timeline of tweets authored by members of the
specified list succeeds"""
self.api.get_list_statuses(
slug=test_list_slug, owner_screen_name=test_list_owner_screen_name)
def test_create_update_destroy_list_add_remove_list_members(self):
"""Test create a list, adding and removing members then
deleting the list succeeds"""
the_list = self.api.create_list(name='Stuff %s' % int(time.time()))
list_id = the_list['id_str']
self.api.update_list(list_id=list_id,
name='Stuff Renamed %s' % int(time.time()))
screen_names = ['johncena', 'xbox']
# Multi add/delete members
self.api.create_list_members(list_id=list_id, screen_name=screen_names)
self.api.delete_list_members(list_id=list_id, screen_name=screen_names)
# Single add/delete member
self.api.add_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list_member(list_id=list_id,
screen_name='justinbieber')
self.api.delete_list(list_id=list_id)
def test_get_list_memberships(self):
"""Test list of memberhips the authenticated user succeeds"""
self.api.get_list_memberships()
def test_get_list_subscribers(self):
"""Test list of subscribers of a specific list succeeds"""
self.api.get_list_subscribers(
slug=test_list_slug, owner_screen_name=test_list_owner_screen_name)
def test_subscribe_is_subbed_and_unsubscribe_to_list(self):
"""Test subscribing, is a list sub and unsubbing to list succeeds"""
self.api.subscribe_to_list(
slug=test_list_slug, owner_screen_name=test_list_owner_screen_name)
# Returns 404 if user is not a subscriber
self.api.is_list_subscriber(
slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name,
screen_name=screen_name)
self.api.unsubscribe_from_list(
slug=test_list_slug, owner_screen_name=test_list_owner_screen_name)
def test_is_list_member(self):
"""Test returning if specified user is member of a list succeeds"""
# Returns 404 if not list member
self.api.is_list_member(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name,
screen_name='themattharris')
def test_get_list_members(self):
"""Test listing members of the specified list succeeds"""
self.api.get_list_members(
slug=test_list_slug, owner_screen_name=test_list_owner_screen_name)
def test_get_specific_list(self):
"""Test getting specific list succeeds"""
self.api.get_specific_list(
slug=test_list_slug, owner_screen_name=test_list_owner_screen_name)
def test_get_list_subscriptions(self):
"""Test collection of the lists the specified user is
subscribed to succeeds"""
self.api.get_list_subscriptions(screen_name='twitter')
def test_show_owned_lists(self):
"""Test collection of lists the specified user owns succeeds"""
self.api.show_owned_lists(screen_name='twitter')
# Saved Searches
def test_get_saved_searches(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
self.api.get_saved_searches()
def test_create_get_destroy_saved_search(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
saved_search = self.api.create_saved_search(query='#Twitter')
saved_search_id = saved_search['id_str']
self.api.show_saved_search(id=saved_search_id)
self.api.destroy_saved_search(id=saved_search_id)
# Places & Geo
def test_get_geo_info(self):
"""Test getting info about a geo location succeeds"""
self.api.get_geo_info(place_id='df51dec6f4ee2b2c')
def test_reverse_geo_code(self):
"""Test reversing geocode succeeds"""
self.api.reverse_geocode(lat='37.76893497', long='-122.42284884')
def test_search_geo(self):
"""Test search for places that can be attached
to a statuses/update succeeds"""
self.api.search_geo(query='Toronto')
def test_get_similar_places(self):
"""Test locates places near the given coordinates which
are similar in name succeeds"""
self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ')
# Trends
def test_get_place_trends(self):
"""Test getting the top 10 trending topics for a specific
WOEID succeeds"""
self.api.get_place_trends(id=1)
def test_get_available_trends(self):
"""Test returning locations that Twitter has trending
topic information for succeeds"""
self.api.get_available_trends()
def test_get_closest_trends(self):
"""Test getting the locations that Twitter has trending topic
information for, closest to a specified location succeeds"""
self.api.get_closest_trends(lat='37', long='-122')
# Help
def test_get_twitter_configuration(self):
"""Test getting Twitter's configuration succeeds"""
self.api.get_twitter_configuration()
def test_get_supported_languages(self):
"""Test getting languages supported by Twitter succeeds"""
self.api.get_supported_languages()
def test_privacy_policy(self):
"""Test getting Twitter's Privacy Policy succeeds"""
self.api.get_privacy_policy()
def test_get_tos(self):
"""Test getting the Twitter Terms of Service succeeds"""
self.api.get_tos()
def test_get_application_rate_limit_status(self):
"""Test getting application rate limit status succeeds"""
self.oauth2_api.get_application_rate_limit_status()
class TwythonEndpointsTestCase(unittest.TestCase):
def setUp(self):
client_args = {
'headers': {
'User-Agent': '__twython__ Test'
},
'allow_redirects': False
}
oauth2_client_args = {
'headers': {} # This is so we can hit coverage that Twython sets User-Agent for us if none is supplied
}
self.api = Twython(app_key, app_secret,
oauth_token, oauth_token_secret,
client_args=client_args)
self.oauth2_api = Twython(app_key, access_token=access_token,
client_args=oauth2_client_args)
# Timelines
@unittest.skip('skipping non-updated test')
def test_get_mentions_timeline(self):
"""Test returning mentions timeline for authenticated user succeeds"""
self.api.get_mentions_timeline()
@unittest.skip('skipping non-updated test')
def test_get_user_timeline(self):
"""Test returning timeline for authenticated user and random user
succeeds"""
self.api.get_user_timeline() # Authenticated User Timeline
self.api.get_user_timeline(screen_name='twitter') # Random User Timeline
@unittest.skip('skipping non-updated test')
def test_get_protected_user_timeline_following(self):
"""Test returning a protected user timeline who you are following
succeeds"""
self.api.get_user_timeline(screen_name=protected_twitter_1)
@unittest.skip('skipping non-updated test')
def test_get_protected_user_timeline_not_following(self):
"""Test returning a protected user timeline who you are not following
fails and raise a TwythonAuthError"""
self.assertRaises(TwythonAuthError, self.api.get_user_timeline,
screen_name=protected_twitter_2)
@unittest.skip('skipping non-updated test')
def test_retweeted_of_me(self):
"""Test that getting recent tweets by authenticated user that have
been retweeted by others succeeds"""
self.api.retweeted_of_me()
@unittest.skip('skipping non-updated test')
def test_get_home_timeline(self):
"""Test returning home timeline for authenticated user succeeds"""
self.api.get_home_timeline()
# Tweets
@unittest.skip('skipping non-updated test')
def test_get_retweets(self):
"""Test getting retweets of a specific tweet succeeds"""
self.api.get_retweets(id=test_tweet_id)
@unittest.skip('skipping non-updated test')
def test_show_status(self):
"""Test returning a single status details succeeds"""
self.api.show_status(id=test_tweet_id)
@unittest.skip('skipping non-updated test')
def test_update_and_destroy_status(self):
"""Test updating and deleting a status succeeds"""
status = self.api.update_status(status='Test post just to get deleted :( %s' % int(time.time()))
self.api.destroy_status(id=status['id_str'])
@unittest.skip('skipping non-updated test')
def test_get_oembed_tweet(self):
"""Test getting info to embed tweet on Third Party site succeeds"""
self.api.get_oembed_tweet(id='99530515043983360')
@unittest.skip('skipping non-updated test')
def test_get_retweeters_ids(self):
"""Test getting ids for people who retweeted a tweet succeeds"""
self.api.get_retweeters_ids(id='99530515043983360')
# Search
@unittest.skip('skipping non-updated test')
def test_search(self):
"""Test searching tweets succeeds"""
self.api.search(q='twitter')
# Direct Messages
@unittest.skip('skipping non-updated test')
def test_get_direct_messages(self):
"""Test getting the authenticated users direct messages succeeds"""
self.api.get_direct_messages()
@unittest.skip('skipping non-updated test')
def test_get_sent_messages(self):
"""Test getting the authenticated users direct messages they've
sent succeeds"""
self.api.get_sent_messages()
@unittest.skip('skipping non-updated test')
def test_send_get_and_destroy_direct_message(self):
"""Test sending, getting, then destory a direct message succeeds"""
message = self.api.send_direct_message(screen_name=protected_twitter_1,
text='Hey d00d! %s' % int(time.time()))
self.api.get_direct_message(id=message['id_str'])
self.api.destroy_direct_message(id=message['id_str'])
@unittest.skip('skipping non-updated test')
def test_send_direct_message_to_non_follower(self):
"""Test sending a direct message to someone who doesn't follow you
fails"""
self.assertRaises(TwythonError, self.api.send_direct_message,
screen_name=protected_twitter_2, text='Yo, man! %s' % int(time.time()))
# Friends & Followers
@unittest.skip('skipping non-updated test')
def test_get_user_ids_of_blocked_retweets(self):
"""Test that collection of user_ids that the authenticated user does
not want to receive retweets from succeeds"""
self.api.get_user_ids_of_blocked_retweets(stringify_ids=True)
@unittest.skip('skipping non-updated test')
def test_get_friends_ids(self):
"""Test returning ids of users the authenticated user and then a random
user is following succeeds"""
self.api.get_friends_ids()
self.api.get_friends_ids(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_get_followers_ids(self):
"""Test returning ids of users the authenticated user and then a random
user are followed by succeeds"""
self.api.get_followers_ids()
self.api.get_followers_ids(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_lookup_friendships(self):
"""Test returning relationships of the authenticating user to the
comma-separated list of up to 100 screen_names or user_ids provided
succeeds"""
self.api.lookup_friendships(screen_name='twitter,ryanmcgrath')
@unittest.skip('skipping non-updated test')
def test_get_incoming_friendship_ids(self):
"""Test returning incoming friendship ids succeeds"""
self.api.get_incoming_friendship_ids()
@unittest.skip('skipping non-updated test')
def test_get_outgoing_friendship_ids(self):
"""Test returning outgoing friendship ids succeeds"""
self.api.get_outgoing_friendship_ids()
@unittest.skip('skipping non-updated test')
def test_create_friendship(self):
"""Test creating a friendship succeeds"""
self.api.create_friendship(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_destroy_friendship(self):
"""Test destroying a friendship succeeds"""
self.api.destroy_friendship(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_update_friendship(self):
"""Test updating friendships succeeds"""
self.api.update_friendship(screen_name=protected_twitter_1,
retweets='true')
self.api.update_friendship(screen_name=protected_twitter_1,
retweets=False)
@unittest.skip('skipping non-updated test')
def test_show_friendships(self):
"""Test showing specific friendship succeeds"""
self.api.show_friendship(target_screen_name=protected_twitter_1)
@unittest.skip('skipping non-updated test')
def test_get_friends_list(self):
"""Test getting list of users authenticated user then random user is
following succeeds"""
self.api.get_friends_list()
self.api.get_friends_list(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_get_followers_list(self):
"""Test getting list of users authenticated user then random user are
followed by succeeds"""
self.api.get_followers_list()
self.api.get_followers_list(screen_name='twitter')
# Users
@unittest.skip('skipping non-updated test')
def test_get_account_settings(self):
"""Test getting the authenticated user account settings succeeds"""
self.api.get_account_settings()
@unittest.skip('skipping non-updated test')
def test_verify_credentials(self):
"""Test representation of the authenticated user call succeeds"""
self.api.verify_credentials()
@unittest.skip('skipping non-updated test')
def test_update_account_settings(self):
"""Test updating a user account settings succeeds"""
self.api.update_account_settings(lang='en')
@unittest.skip('skipping non-updated test')
def test_update_delivery_service(self):
"""Test updating delivery settings fails because we don't have
a mobile number on the account"""
self.assertRaises(TwythonError, self.api.update_delivery_service,
device='none')
@unittest.skip('skipping non-updated test')
def test_update_profile(self):
"""Test updating profile succeeds"""
self.api.update_profile(include_entities='true')
@unittest.skip('skipping non-updated test')
def test_update_profile_colors(self):
"""Test updating profile colors succeeds"""
self.api.update_profile_colors(profile_background_color='3D3D3D')
@unittest.skip('skipping non-updated test')
def test_list_blocks(self):
"""Test listing users who are blocked by the authenticated user
succeeds"""
self.api.list_blocks()
@unittest.skip('skipping non-updated test')
def test_list_block_ids(self):
"""Test listing user ids who are blocked by the authenticated user
succeeds"""
self.api.list_block_ids()
@unittest.skip('skipping non-updated test')
def test_create_block(self):
"""Test blocking a user succeeds"""
self.api.create_block(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_destroy_block(self):
"""Test unblocking a user succeeds"""
self.api.destroy_block(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_lookup_user(self):
"""Test listing a number of user objects succeeds"""
self.api.lookup_user(screen_name='twitter,justinbieber')
@unittest.skip('skipping non-updated test')
def test_show_user(self):
"""Test showing one user works"""
self.api.show_user(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_search_users(self):
"""Test that searching for users succeeds"""
self.api.search_users(q='Twitter API')
@unittest.skip('skipping non-updated test')
def test_get_contributees(self):
"""Test returning list of accounts the specified user can
contribute to succeeds"""
self.api.get_contributees(screen_name='TechCrunch')
@unittest.skip('skipping non-updated test')
def test_get_contributors(self):
"""Test returning list of accounts that contribute to the
authenticated user fails because we are not a Contributor account"""
self.assertRaises(TwythonError, self.api.get_contributors,
screen_name=screen_name)
@unittest.skip('skipping non-updated test')
def test_remove_profile_banner(self):
"""Test removing profile banner succeeds"""
self.api.remove_profile_banner()
@unittest.skip('skipping non-updated test')
def test_get_profile_banner_sizes(self):
"""Test getting list of profile banner sizes fails because
we have not uploaded a profile banner"""
self.assertRaises(TwythonError, self.api.get_profile_banner_sizes)
# Suggested Users
@unittest.skip('skipping non-updated test')
def test_get_user_suggestions_by_slug(self):
"""Test getting user suggestions by slug succeeds"""
self.api.get_user_suggestions_by_slug(slug='twitter')
@unittest.skip('skipping non-updated test')
def test_get_user_suggestions(self):
"""Test getting user suggestions succeeds"""
self.api.get_user_suggestions()
@unittest.skip('skipping non-updated test')
def test_get_user_suggestions_statuses_by_slug(self):
"""Test getting status of suggested users succeeds"""
self.api.get_user_suggestions_statuses_by_slug(slug='funny')
# Favorites
@unittest.skip('skipping non-updated test')
def test_get_favorites(self):
"""Test getting list of favorites for the authenticated
user succeeds"""
self.api.get_favorites()
@unittest.skip('skipping non-updated test')
def test_create_and_destroy_favorite(self):
"""Test creating and destroying a favorite on a tweet succeeds"""
self.api.create_favorite(id=test_tweet_id)
self.api.destroy_favorite(id=test_tweet_id)
# Lists
@unittest.skip('skipping non-updated test')
def test_show_lists(self):
"""Test show lists for specified user"""
self.api.show_lists(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_get_list_statuses(self):
"""Test timeline of tweets authored by members of the
specified list succeeds"""
self.api.get_list_statuses(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_create_update_destroy_list_add_remove_list_members(self):
"""Test create a list, adding and removing members then
deleting the list succeeds"""
the_list = self.api.create_list(name='Stuff %s' % int(time.time()))
list_id = the_list['id_str']
self.api.update_list(list_id=list_id, name='Stuff Renamed %s' % int(time.time()))
screen_names = ['johncena', 'xbox']
# Multi add/delete members
self.api.create_list_members(list_id=list_id,
screen_name=screen_names)
self.api.delete_list_members(list_id=list_id,
screen_name=screen_names)
# Single add/delete member
self.api.add_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list(list_id=list_id)
@unittest.skip('skipping non-updated test')
def test_get_list_memberships(self):
"""Test list of memberhips the authenticated user succeeds"""
self.api.get_list_memberships()
@unittest.skip('skipping non-updated test')
def test_get_list_subscribers(self):
"""Test list of subscribers of a specific list succeeds"""
self.api.get_list_subscribers(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_subscribe_is_subbed_and_unsubscribe_to_list(self):
"""Test subscribing, is a list sub and unsubbing to list succeeds"""
self.api.subscribe_to_list(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
# Returns 404 if user is not a subscriber
self.api.is_list_subscriber(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name,
screen_name=screen_name)
self.api.unsubscribe_from_list(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_is_list_member(self):
"""Test returning if specified user is member of a list succeeds"""
# Returns 404 if not list member
self.api.is_list_member(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name,
screen_name='themattharris')
@unittest.skip('skipping non-updated test')
def test_get_list_members(self):
"""Test listing members of the specified list succeeds"""
self.api.get_list_members(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_get_specific_list(self):
"""Test getting specific list succeeds"""
self.api.get_specific_list(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_get_list_subscriptions(self):
"""Test collection of the lists the specified user is
subscribed to succeeds"""
self.api.get_list_subscriptions(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_show_owned_lists(self):
"""Test collection of lists the specified user owns succeeds"""
self.api.show_owned_lists(screen_name='twitter')
# Saved Searches
@unittest.skip('skipping non-updated test')
def test_get_saved_searches(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
self.api.get_saved_searches()
@unittest.skip('skipping non-updated test')
def test_create_get_destroy_saved_search(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
saved_search = self.api.create_saved_search(query='#Twitter')
saved_search_id = saved_search['id_str']
self.api.show_saved_search(id=saved_search_id)
self.api.destroy_saved_search(id=saved_search_id)
# Places & Geo
@unittest.skip('skipping non-updated test')
def test_get_geo_info(self):
"""Test getting info about a geo location succeeds"""
self.api.get_geo_info(place_id='df51dec6f4ee2b2c')
@unittest.skip('skipping non-updated test')
def test_reverse_geo_code(self):
"""Test reversing geocode succeeds"""
self.api.reverse_geocode(lat='37.76893497', long='-122.42284884')
@unittest.skip('skipping non-updated test')
def test_search_geo(self):
"""Test search for places that can be attached
to a statuses/update succeeds"""
self.api.search_geo(query='Toronto')
@unittest.skip('skipping non-updated test')
def test_get_similar_places(self):
"""Test locates places near the given coordinates which
are similar in name succeeds"""
self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ')
# Trends
@unittest.skip('skipping non-updated test')
def test_get_place_trends(self):
"""Test getting the top 10 trending topics for a specific
WOEID succeeds"""
self.api.get_place_trends(id=1)
@unittest.skip('skipping non-updated test')
def test_get_available_trends(self):
"""Test returning locations that Twitter has trending
topic information for succeeds"""
self.api.get_available_trends()
@unittest.skip('skipping non-updated test')
def test_get_closest_trends(self):
"""Test getting the locations that Twitter has trending topic
information for, closest to a specified location succeeds"""
self.api.get_closest_trends(lat='37', long='-122')
# Help
@unittest.skip('skipping non-updated test')
def test_get_twitter_configuration(self):
"""Test getting Twitter's configuration succeeds"""
self.api.get_twitter_configuration()
@unittest.skip('skipping non-updated test')
def test_get_supported_languages(self):
"""Test getting languages supported by Twitter succeeds"""
self.api.get_supported_languages()
@unittest.skip('skipping non-updated test')
def test_privacy_policy(self):
"""Test getting Twitter's Privacy Policy succeeds"""
self.api.get_privacy_policy()
@unittest.skip('skipping non-updated test')
def test_get_tos(self):
"""Test getting the Twitter Terms of Service succeeds"""
self.api.get_tos()
@unittest.skip('skipping non-updated test')
def test_get_application_rate_limit_status(self):
"""Test getting application rate limit status succeeds"""
self.oauth2_api.get_application_rate_limit_status()
#this part sucks, getting all your secret keys and such from Twitter API: https://dev.twitter.com/docs/auth/tokens-devtwittercom
APP_KEY = "app key here, a bunch of numbers"
APP_SECRET = "app secret here, a bunch more numbers"
OAUTH_TOKEN = "more numbers for your oauth token"
OAUTH_TOKEN_SECRET = "last secret token"
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
#make a list of friends
allfriendsIDs = []
sn = "your_twitter_username"
#get your friends_ids first
response = twitter.get_friends_ids(screen_name=sn)
friends = response['ids']
allfriendsIDs += friends
#now get their information, in batches of 100 because someone told me Twitter doesn't like you to get lots of people at once (what do i know though)
def getUserDetails(ids):
userDetails = []
if len(ids) > 100:
response = twitter.lookup_user(user_id=','.join(
str(a) for a in ids[0:99]))
userDetails += response
return getUserDetails(ids[100:]) + userDetails
else:
response = twitter.lookup_user(user_id=','.join(str(a) for a in ids))
userDetails += response
friends_query_size = 5000
res = db.friends.find_one({"screen_name": screen_name})
if res is None:
db.friends.insert_one({"screen_name": screen_name, "ids": []})
next_cursor = -1
friends_ids = list()
ids_count = 0
n_friends = 600
## Tras iniciar, se hace la llamada a la API con get_friends_ids().
while (next_cursor != 0) and (ids_count < n_friends):
try:
result = twitter.get_friends_ids(screen_name=screen_name,
count=friends_query_size,
cursor=next_cursor)
friends_ids = friends_ids + result['ids']
next_cursor = result['next_cursor']
ids_count += len(result['ids'])
friends_ids = list(set(friends_ids))
friends_ids.sort()
## Se inserta la lista de id en la BB. DD.
res = db.friends.update_one({"screen_name": screen_name},
{'$set': {
"ids": friends_ids
}})
except TwythonRateLimitError as e:
API_SECRET = 'INSERT HERE'
ACCESS_TOKEN = 'INSERT HERE'
ACCESS_TOKEN_SECRET = 'INSERT HERE'
twitter = Twython(API_KEY, API_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
return (twitter)
'''
Determining your Twitter followers and friends
'''
twitter = twitter_oauth_login()
friends_ids = twitter.get_friends_ids(count=5000)
friends_ids = friends_ids['ids']
followers_ids = twitter.get_followers_ids(count=5000)
followers_ids = followers_ids['ids']
print len(friends_ids), len(followers_ids)
friends_set = set(friends_ids)
followers_set = set(followers_ids)
print(
'Number of Twitter users who either are our friend or follow you (union):')
print(len(friends_set.union(followers_set)))
len(friends_set | followers_set)
"""
#full_list = ["mbfhunzaker"] #single user for testing
scraped_users = []
bad_ids = []
for leader in full_list:
scraped_users.append(
leader
) #keeps list of who's already been scraped incase you have to stop/start the code
#^^ is output as the file that will be imported as the "already_done_list" above
cur = -1 # used to function argument to 1st page of followers--idk why -1 if the first page...
try:
while cur != 0: # until you run out of pages of followers...
followers = t.get_friends_ids(
screen_name=leader, cursor=cur
) #this line is the API call that looks up the people that user follows in chunks of 5000
follower_ids = followers[
"ids"] #extracts list of follower ids from API call data
for user_id in follower_ids: #iterates over list of ids
if user_id not in scrapedInfo: #if we dont already have the user in our dictionary of users
print(user_id)
#scrapedInfo[user_id]=leader #method to keep track of who follows
scrapedInfo[
user_id] = 1 #method to keep track of how many follow; sets to 1 if not in our set yet
elif user_id in scrapedInfo: #if we DO have the user in our dictionary of users (e.g. if they're followed by someone we already scraped)
#scrapedInfo[user_id] = scrapedInfo[user_id] + ", " + leader #method to keep track of *who* follows
scrapedInfo[user_id] = scrapedInfo[
user_id] + 1 #method to keep track of how many follows them; increments count of elites who follow them by 1
else:
print(
for line in fp1:
user = int(line.rstrip())
userIDs.append(user)
apiKey = 'Tl4x7q0fMxTTnwU4gacWOaTBW' #str(argList[1])
accessToken = 'AAAAAAAAAAAAAAAAAAAAAA2BewAAAAAABUUGEYgpIQ1z056Xaib2hyrc6mI%3DTjG1tqzGOv7PfiTJAjPtkA2AmVkw4gqmPduB4UnIteNK2xEpAw' #str(argList[2])
twitter = Twython(apiKey,access_token = accessToken,client_args = client_args)
for user in userIDs:
cur = -1
friendList = []
while cur != 0:
try:
response = twitter.get_friends_ids(user_id=user,count=5000,cursor = cur)
print response
except Exception as e:
print e
continue
cur = response['next_cursor']
friendList = friendList + response['ids']
if cur != 0:
continue
dictionary = {}
dictionary[str(user)] = friendList
fp2.write("{0}\n".format(dictionary))
fp1.close()
fp2.close()
def on_demand(items):
""" Use generate() to provide a new tweet for a
user when they tweet 'hit me' """
for tweet in reversed(items):
text = tweet['text'][12:]
if ((text[:6].strip().lower() == "hit me")
and (int(tweet['id']) > lastTweet)):
newJargon = generate()
newTweet = "@" + tweet['user']['screen_name'] + " " + newJargon
newTweet = newTweet[:139]
twitter.update_status(status=newTweet,
in_reply_to_status_id=int(tweet['id']))
def periodic():
""" Periodically tweet out a jargon phrase using generate() """
tweetCheck = randint(0,35)
if (tweetCheck == 5):
newTweet = generate()
twitter.update_status(status=newTweet)
tweets = twitter.get_mentions_timeline()
get_last()
intake(simplify(tweets, twitter.get_friends_ids()['ids']))
on_demand(tweets)
periodic()
t = Twython(app_key=TWITTER_API_KEY,
app_secret=TWITTER_API_KEY_SECRET,
oauth_token=TWITTER_ACCESS_TOKEN,
oauth_token_secret=TWITTER_ACCESS_TOKEN_SECRET)
client2=pymongo.MongoClient(mdc.u2)
i=0
for nn in N:
print i; i+=1
teste=client2.sna.amizades.find({str(IDS[snames.index(nn)]):{"$exists":True}})
teste=[testee for testee in teste]
if not teste:
print nn, "not teste"
tids=t.get_friends_ids(screen_name=nn)["ids"]
id_orig=W[snames.index(nn)]["user"]["id"]
client2.sna.amizades.insert({str(id_orig):tids})
else:
print nn, "teste"
T.sleep(90)
#
#while 1:
# search = t.search(q='#arenaNETmundial',count=150,max_id=tweets[-1]['id']-1)
# i=0
# while len(search['statuses'])>0:
# tweets +=search['statuses']
# #search = t.search(q='#arenaNETmundial', count=150, max_id=tweets[-1]['id']-1)
# print "older", i, len(tweets),search['statuses']; i+=1
#
class Twitter(object):
""" Class for twitter use """
query_string = '#севастополь -украина OR северная OR катер OR SevastopolBoats OR sevboats :) since:%s lang:ru'
weather_string = u'#погода в Севастополе на %s.'
timelimit = 90
max_follow = 100
def __init__(self, auth=TWITTER_OAUTH_INFO):
self.twitter = Twython(**auth)
def post_tweet(self, message):
""" Writes message into twitter stream """
return self.twitter.update_status(status=message, trim_user=True)
def delete_tweet(self, tweet_id):
""" Remove tweet from timeline """
return self.twitter.destroy_status(id=tweet_id, trim_user=True)
def post_debug(self, message):
""" Writes message into console to test """
print message
return dict(id=0, id_str='0', text=message)
def search(self, query=None, count=10):
""" Search tweet with sevboats ref """
if query is None:
since = (datetime.datetime.now() - datetime.timedelta(days=7)).date().strftime('%Y-%m-%d')
query = self.query_string % since if 'since:%s' in self.query_string else self.query_string
return self.twitter.search(q=query, count=count)
def follow(self, user_id):
""" Follow user by id """
try:
return self.twitter.create_friendship(user_id=user_id, follow=True)
except TwythonError:
return dict(status='error', id_str=None)
def unfollow(self, user_id):
""" Un Follow user by id """
try:
return self.twitter.destroy_friendship(user_id=user_id)
except TwythonError:
return dict(status='error', id_str=None)
def follow_list(self, friends_ids):
""" Follow user in search result """
friends = []
for user_id in friends_ids:
friends.append(self.follow(user_id)['id_str'])
return friends
def unfollow_list(self, friends_ids):
""" Un Follow user in ids list """
friends = []
for user_id in friends_ids:
friends.append(self.unfollow(user_id)['id_str'])
return friends
def search_to_list(self, search_list=None):
""" Follow user in search result """
if search_list is None:
search_list = self.search()
users_ids = []
for tweet in search_list['statuses']:
users_ids.append(tweet['user']['id_str'])
return users_ids
def my_followers(self):
""" Get list of my followers """
followers_ids = []
next_cursor = -1
while next_cursor != '0':
cursor = next_cursor
fids = self.twitter.get_followers_ids(cursor=cursor, stringify_ids=True, count=1000)
followers_ids += fids['ids']
next_cursor = fids['next_cursor_str']
time.sleep(self.timelimit)
return followers_ids
def follow_search(self):
""" Follow for all user in search results """
users_ids = self.search_to_list()
self.follow_list(users_ids)
return users_ids
def follow_followers(self):
""" Follow for all followers """
followers_ids = self.my_followers()
friends_ids = self.i_follow()
users_ids = []
for follower in followers_ids:
if follower not in friends_ids:
users_ids.append(follower)
self.follow_list(users_ids[:self.max_follow])
return users_ids
def i_follow(self):
""" Get list of user i follow what """
friends_ids = []
next_cursor = -1
while next_cursor != '0':
cursor = next_cursor
fids = self.twitter.get_friends_ids(cursor=cursor, stringify_ids=True, count=1000)
friends_ids += fids['ids']
next_cursor = fids['next_cursor_str']
time.sleep(self.timelimit)
return friends_ids
def post_image_weather(self):
""" Create weather image and post to twitter """
# get weather image
print 'start'
weth = Weather()
weather_img, date_title = weth.draw_img()
# create stream
print 'streem'
image_io = StringIO()
# weather_img.save(image_io, format='JPEG', quality=100, subsampling=0, optimize=True, progressive=True)
weather_img.save(image_io, format='PNG')
image_io.seek(0)
# post tweet
print 'post'
return self.twitter.update_status_with_media(media=image_io, status=self.weather_string % date_title)
def addUserToDB(sn):
APP_KEY = "UHz2Yr3qedZeNZxblgzqVbER6"
APP_SECRET = "KavH3dnHPOPybPWBWXAWD7jAqfJRxDLqEK24N1m64O2fax0vM0"
twitter = Twython(APP_KEY, APP_SECRET, oauth_version=2)
ACCESS_TOKEN = twitter.obtain_access_token()
twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN)
analyzer = SentimentIntensityAnalyzer()
totalSentimentScore = 0
tweetCount = 0
compoundText = ""
tweets = []
try:
user_timeline = twitter.get_user_timeline(screen_name=sn, count=30)
except TwythonError as e:
print(e)
return False
for tweet in user_timeline:
tweetCount += 1
sentiment_dict = analyzer.polarity_scores(tweet['text'])
totalSentimentScore += sentiment_dict['compound']
compoundText = compoundText + " " + tweet['text']
tweets.append(tweet)
syntax_dict = count_by_word(compoundText)
averageSentiment = 0
if tweetCount != 0:
averageSentiment = totalSentimentScore / tweetCount
## FOLLOWERS ##
followers = []
try:
user_followers = twitter.get_followers_ids(screen_name=sn)
followers = user_followers
except TwythonError as e:
print(e)
return False
following = []
try:
user_friends = twitter.get_friends_ids(screen_name=sn)
following = user_friends
except TwythonError as e:
print(e)
return False
lastUpdated = datetime.datetime.utcnow().isoformat()
syntax_dict = count_by_word(compoundText)
syntax_dict = list(filter(lambda x: x[1] != 1, syntax_dict))
userStore = {
"_id": sn,
"last_updated": lastUpdated,
"avg_sentiment": averageSentiment,
"syntax_dict": syntax_dict[:30],
"following": following['ids'],
"followers": followers['ids'],
"tweets": tweets
}
client = pymongo.MongoClient(
"mongodb+srv://tgabrie2:[email protected]/cs432?retryWrites=true&w=majority",
ssl_cert_reqs=ssl.CERT_NONE)
db = client.cs432
collection = db['test']
collection.insert_one(userStore)
return True
twitter = Twython(APP_KEY,APP_SECRET, access_token=ACCESS_TOKEN)
#creates the dictionary the results will be stored in
stored=defaultdict(list)
#Reads in the csv with the Twitter handles
ifile = open('inpath.csv', "rb")
for name in ifile:
next_cursor=-1 #initiates the cursor
try:
while(next_cursor != 0):#keeps looping through as long as there are pages there
print(name)
time.sleep(60) # sets a time delay of 60 seconds to keep within the rate limits
#gets the people followed by the screen name
following=twitter.get_friends_ids(screen_name=str(name), cursor=next_cursor)
for x in following[u'ids']: #loops through the ids returned by the Twitter api
name=name.encode('utf-8')
stored[str(name)].append(x) # and appends them to the dictionary where the key is the associated persons user id
next_cursor=following['next_cursor'] #gets the next cursor value
except: # if the call fails which may be because there are no more pages
print('unable to access the people followed by '+str(name)+','+str(next_cursor)+'\n')
#writes the dictionary as a json file to the computer
with open('outpath.json', 'w') as f:
json.dump(stored, f)
# from __future__ import print_function
from twython import Twython, TwythonError
twitter = Twython('PVRb7LmKFwBctrLKvULQTnSqi', access_token='AAAAAAAAAAAAAAAAAAAAAPaPWwAAAAAAkiazdF9EzC%2FtcilOFW0S7kYxh94%3DtI84ctlH9V3fZ4Cia3BX0jUmUWkQp2ERtTFfajoFherjwMJJrl')
for line in open("217189228_follows"):
listid = line.split(", ")
# print listid
num_attempt=0
for a in listid:
try:
num_attempt+=1;
if num_attempt>=14:
print "sleep"
time.sleep(15*60)
num_attempt=0
ids1 = twitter.get_friends_ids(id = a)
target = open(a+"_follows",'w')
for id in ids1['ids']:
target.write(str(id)+', ')
# target.write('\n')
except TwythonError as e:
print e
# print(ids1['ids'],file = target)
# print ids1['ids']
# for line in open("followee_217189228"):
# listid = line.split(", ")
# for a in listid:
# user_timeline=twitter.get_user_timeline(user_id=a, count=200)
# target = open(a,'a')
# for tweet in user_timeline:
# target.write(tweet['text'].encode('utf-8')+'\n')
def retrieve_friends_ids(users, file_name):
# CONSUMER_KEY1 = my_creds['CONSUMER_KEY']
# CONSUMER_SECRET1 = my_creds['CONSUMER_SECRET']
# OAUTH_TOKEN1 = my_creds['TOKEN_KEY']
# OAUTH_TOKEN_SECRET1 = my_creds['TOKEN_SECRET']
CONSUMER_KEY2 = my_credsa['CONSUMER_KEY']
CONSUMER_SECRET2 = my_credsa['CONSUMER_SECRET']
OAUTH_TOKEN2 = my_credsa['TOKEN_KEY']
OAUTH_TOKEN_SECRET2 = my_credsa['TOKEN_SECRET']
CONSUMER_KEY3 = my_credsb['CONSUMER_KEY']
CONSUMER_SECRET3 = my_credsb['CONSUMER_SECRET']
OAUTH_TOKEN3 = my_credsb['TOKEN_KEY']
OAUTH_TOKEN_SECRET3 = my_credsb['TOKEN_SECRET']
CONSUMER_KEY4 = my_credsc['CONSUMER_KEY']
CONSUMER_SECRET4 = my_credsc['CONSUMER_SECRET']
OAUTH_TOKEN4 = my_credsc['TOKEN_KEY']
OAUTH_TOKEN_SECRET4 = my_credsc['TOKEN_SECRET']
CONSUMER_KEY5 = my_credsd['CONSUMER_KEY']
CONSUMER_SECRET5 = my_credsd['CONSUMER_SECRET']
OAUTH_TOKEN5 = my_credsd['TOKEN_KEY']
OAUTH_TOKEN_SECRET5 = my_credsd['TOKEN_SECRET']
CONSUMER_KEY6 = my_creds['CONSUMER_KEY']
CONSUMER_SECRET6 = my_creds['CONSUMER_SECRET']
OAUTH_TOKEN6 = my_creds['TOKEN_KEY']
OAUTH_TOKEN_SECRET6 = my_creds['TOKEN_SECRET']
creds = defaultdict(str)
# creds['creds1'] = {'CONSUMER_KEY': CONSUMER_KEY1, 'CONSUMER_SECRET': CONSUMER_SECRET1, 'TOKEN_KEY': OAUTH_TOKEN1, 'TOKEN_SECRET': OAUTH_TOKEN_SECRET1}
creds['creds1'] = {
'CONSUMER_KEY': CONSUMER_KEY2,
'CONSUMER_SECRET': CONSUMER_SECRET2,
'TOKEN_KEY': OAUTH_TOKEN2,
'TOKEN_SECRET': OAUTH_TOKEN_SECRET2
}
creds['creds2'] = {
'CONSUMER_KEY': CONSUMER_KEY3,
'CONSUMER_SECRET': CONSUMER_SECRET3,
'TOKEN_KEY': OAUTH_TOKEN3,
'TOKEN_SECRET': OAUTH_TOKEN_SECRET3
}
creds['creds3'] = {
'CONSUMER_KEY': CONSUMER_KEY4,
'CONSUMER_SECRET': CONSUMER_SECRET4,
'TOKEN_KEY': OAUTH_TOKEN4,
'TOKEN_SECRET': OAUTH_TOKEN_SECRET4
}
creds['creds4'] = {
'CONSUMER_KEY': CONSUMER_KEY5,
'CONSUMER_SECRET': CONSUMER_SECRET5,
'TOKEN_KEY': OAUTH_TOKEN5,
'TOKEN_SECRET': OAUTH_TOKEN_SECRET5
}
creds['creds5'] = {
'CONSUMER_KEY': CONSUMER_KEY6,
'CONSUMER_SECRET': CONSUMER_SECRET6,
'TOKEN_KEY': OAUTH_TOKEN6,
'TOKEN_SECRET': OAUTH_TOKEN_SECRET6
}
cred_inuse = 'creds1'
last_use_time = [{
'name': 'creds1',
'time': 0
}, {
'name': 'creds2',
'time': 0
}, {
'name': 'creds3',
'time': 0
}, {
'name': 'creds4',
'time': 0
}, {
'name': 'creds5',
'time': 0
}]
print(creds[cred_inuse])
twitter_twy = Twython(creds[cred_inuse]['CONSUMER_KEY'],
creds[cred_inuse]['CONSUMER_SECRET'],
creds[cred_inuse]['TOKEN_KEY'],
creds[cred_inuse]['TOKEN_SECRET'])
# twitter_twy = Twython(CONSUMER_KEY2, CONSUMER_SECRET2, OAUTH_TOKEN2, OAUTH_TOKEN_SECRET2)
count_rate_exceed = 0
with codecs.open(file_name, 'w') as out_file:
for u in users:
ids = []
next_cursor = -1
# print("Retrieving friends of {}".format(u))
out_file.write(' '.join(['-user', u, '\n']))
while (next_cursor):
try:
returned = twitter_twy.get_friends_ids(user_id=u,
count=5000,
cursor=next_cursor,
stringify_ids=True)
print('.', end='')
for result in returned['ids']:
ids.append(result)
next_cursor = returned["next_cursor"]
count_rate_exceed = 0
except TwythonRateLimitError:
count_rate_exceed += 1
print('Too many requests')
# if count_rate_exceed > 3:
# time.sleep(200)
# else:
last_use_time = sorted(last_use_time,
key=lambda x: x['time'])
time_diff = time.time() - last_use_time[0]['time']
if time_diff < 900 and time_diff > 0:
print('sleeping for {} seconds'.format(900 -
time_diff))
time.sleep(900 - time_diff)
last_use_time[0]['time'] = time.time()
cred_inuse = last_use_time[0]['name']
print('New credential: {}'.format(cred_inuse))
twitter_twy = Twython(creds[cred_inuse]['CONSUMER_KEY'],
creds[cred_inuse]['CONSUMER_SECRET'],
creds[cred_inuse]['TOKEN_KEY'],
creds[cred_inuse]['TOKEN_SECRET'])
except TwythonAuthError:
print("Couldn't authenticate {}".format(cred_inuse))
cred_inuse = cred_inuse[0:-1] + str(
int(cred_inuse[-1]) % 5 + 1)
print('trying {}'.format(cred_inuse))
twitter_twy = Twython(creds[cred_inuse]['CONSUMER_KEY'],
creds[cred_inuse]['CONSUMER_SECRET'],
creds[cred_inuse]['TOKEN_KEY'],
creds[cred_inuse]['TOKEN_SECRET'])
print('\n')
print("friends of {}.***count:{}".format(u, len(ids)))
out_file.write(','.join(ids))
out_file.write('\n')
out_file.write('-------------------------\n')
print("Process finished.")
newTweet = "@" + tweet['user']['screen_name'] + " " + newJargon
newTweet = newTweet[:139]
twitter.update_status(status=newTweet, in_reply_to_status_id=int(tweet['id']))
def periodic():
""" Periodically tweet out a jargon phrase using generate() """
tweetCheck = random.randint(0,15)
if (tweetCheck == 5):
newTweet = generate()
twitter.update_status(status=newTweet)
def administration(items):
""" follow or reject new users who put in commands """
for tweet in reversed(items):
if (tweet['user']['id'] == 22884755):
text = tweet['text'][10:].split(" ")
if (text[1] == "approve"):
twitter.create_friendship(screen_name=text[2])
twitter.update_status(status="@"+ text[2] + " Good news, you've been approved! Please retry any additions prior to this message again.")
elif (text[1] == "reject"):
twitter.update_status(status="@"+ text[2] + " Sorry, I'm not going to add you right now.")
tweets = twitter.get_mentions_timeline()
getLast()
intake(simplify(twitter.get_mentions_timeline(), twitter.get_friends_ids()['ids']))
onDemand(tweets)
administration(tweets)
periodic()
class TwitterBot():
'''
Crawls twitter and updates social network and tagged resources to repository
'''
def __init__(self):
self.service = TWITTER
APP_KEY = "w7zjbW1qHtzX2GxbrHbeNgyPr"
APP_SECRET = "qJFawkFxwYZX6U3DYNvJI3czWEjix2owU9wTXZ9BorMkhFU4Yb"
self.twapi = Twython(APP_KEY, APP_SECRET)
self.userBaseUri = 'http://twitter.com/%s' # % user screen_name
self.tagBaseUri = 'http://twitter.com/#search?q=#%s' # % trend key word
self.tweetBaseUri = 'http://twitter.com/%s/status/%s' # % (user screen_name, status id)
@staticmethod
def getname():
return self.service
@staticmethod
def getDefaultAvatar():
return defAvatar
def setStartUserId(self, start_user_id):
self.start_user_id = start_user_id
def setMaxLevel(self, max_level):
self.max_level = max_level
def setLastAdded(self, last_added):
self.last_added = last_added
def updateDatabase(self, user_metadata, user_ntw, udate=None):
raise NotImplementedError()
def fetchUser(self, username):
return (None, {'knows':[], 'fans':[]})
def getUserUri(self, username):
return self.userBaseUri % username
@staticmethod
def factory(user, params=None, depth=2, last_added=None, verbose=False):
# Twitter crawler
twcrawl = TwitterBot()
twcrawl.setStartUserId(user)
twcrawl.setMaxLevel(depth)
twcrawl.setLastAdded(last_added)
return twcrawl
def updateDatabase(self, user_metadata, user_ntw, udate=None):
username = user_metadata['username']
userUri = self.userBaseUri % username
resourceUri = None
added = 0
# add user to repository
dbaccess.User.addToDB(userUri, self.service, username,
TwitterBot.getDefaultAvatar(), udate=udate)
# add resources, tags and relations between them to repository
for resource in user_metadata['data']:
# add tweet resource to repository
resourceId = resource['id']
resourceTitle = resource['text']
resourceUri = self.tweetBaseUri % (username, resourceId)
dbaccess.Tweet.addToDB(resourceUri, resourceTitle, udate=udate)
added += 1
# add tag(trend) relations to repository
for tag in self.parseTags(resourceTitle):
tagUri = self.tagBaseUri % urllib.quote(tag.encode('utf-8'))
# add tag to repository
dbaccess.Tag.addToDB(tagUri, tag, udate=udate)
# add user, resource, tag relation to repository
dbaccess.UserResourceTag.addToDB(userUri, resourceUri, tagUri, udate=udate)
# add url relations between the tweet and urls contained
urls = map(self.getRealUrl, self.parseUrls(resourceTitle))
for url in urls:
dbaccess.Bookmark.addToDB(url, None)
dbaccess.Tweet.addReferenceToDB(resourceUri, url)
# add user social network relations to repository
# add friends
for u in user_ntw['knows']:
knownUserUri = self.userBaseUri % u
dbaccess.User.addToDB(knownUserUri, self.service, u,
TwitterBot.getDefaultAvatar(), udate=udate)
dbaccess.User.addToDBUserKnowsUser(userUri, knownUserUri, udate=udate)
# add followers
for u in user_ntw['fans']:
otherUserUri = self.userBaseUri % u
dbaccess.User.addToDB(otherUserUri, self.service, u,
TwitterBot.getDefaultAvatar(), udate=udate)
dbaccess.User.addToDBUserKnowsUser(otherUserUri, userUri, udate=udate)
# return number of added resources and last added resource uri
return added
def fetchUserMetadata(self, username):
# get user tweets (only last 20)
data = self.twapi.get_user_timeline(screen_name=username)
return { 'username': username, 'data': data}
def fetchUserNetwork(self, username):
chunk_size = 100 # max returned by bulkUserLookup
# Get Friends
fids, cursor = [], -1
fchunks = [fids[i:i + chunk_size] for i in range(0, len(fids), chunk_size)]
while True:
f = self.twapi.get_friends_ids(screen_name=username, cursor=cursor)
fids.extend(f['ids'])
if f['next_cursor'] == cursor or len(f['ids']) == 0:
break
cursor = f['next_cursor']
# get screen names from ids in chunks of 100
screen_names = []
for chunk in fchunks:
screen_names = []
screen_names.extend([userdata['screen_name']
for userdata in self.twapi.lookup_user(user_id=chunk)])
ntw_knows = screen_names
print ntw_knows
# Get Followers
fids, cursor = [], -1
while True:
f = self.twapi.get_followers_ids(screen_name=username, cursor=cursor)
fids.extend(f['ids'])
if f['next_cursor'] == cursor or len(f['ids']) == 0:
break
cursor = f['next_cursor']
fchunks = [fids[i:i + chunk_size] for i in range(0, len(fids), chunk_size)]
fan_screen_names = []
for chunk in fchunks:
# get all data for all user ids
fan_screen_names.extend([userdata['screen_name']
for userdata in self.twapi.lookup_user(user_id=chunk)])
ntw_fans = fan_screen_names
return {'knows': ntw_knows, 'fans': ntw_fans}
def fetchUser(self, username):
return (self.fetchUserMetadata(username), self.fetchUserNetwork(username))
def parseTags(self, text):
'''
Parses trends from tweet: '#trend #tre#nd ## #trend,two #tren%^& smth,#da #1dc #100 #1dc$ #100^',
finds trends: trend, tre, trend, tren, da, 1dc, 100, 1dc, 100.
Text parameter MUST be UNICODE encoded: u'some text'
'''
return re.findall('(?<=[\W^#]#)\w+', ' ' + text, re.UNICODE)
def parseUrls(self, text):
'''Returns a list of urls from the text.
'''
return re.findall('http://\S+', text)
def getRealUrl(self, url):
'''Returns the redirected url if *url* parameter is a shortened url (a redirect was produced),
or itself if not.
'''
try:
u = urllib2.urlopen(url)
realurl = u.geturl()
if realurl == None:
raise urllib2.HTTPError('Could not fetch url')
except:
realurl = url
return realurl
def crawlUserNetwork(self, user_id=None, max_level=0, start_time=None):
'''
user_id - an id to uniquely identify a user (can be a username or userid)
'''
user_id = user_id or self.start_user_id
max_level = max_level or self.max_level
queue = [user_id] # init queue
visited = {user_id: 0} # init visited nodes
added = 0 # set number of added resources
while queue:
v = queue.pop(0) # pop next user node
level = visited[v] # get their level
# get user metadata and social network
user_metadata, user_ntw = self.fetchUser(v)
# update database with user data, tagged resources and social network relations
a = self.updateDatabase(user_metadata, user_ntw, udate=start_time)
added += a
# explore following nodes
for w in user_ntw['knows']:
if w not in visited and level < max_level:
queue.append(w)
visited[w] = level + 1
return added
def get_followers_following(usr_id):
index = 0
print "GETTING FOLLOWERS AND FOLLOWING: " + str(usr_id)
friend_cursor = -1
follower_cursor = -1
APP_KEY = APP_KEYS[index]
APP_SECRET = APP_SECRETS[index]
OAUTH_TOKEN = OAUTH_TOKENS[index]
OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index]
while (1 == 1):
try:
twitter = Twython(APP_KEY, APP_SECRET)
auth = twitter.get_authentication_tokens()
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN,
OAUTH_TOKEN_SECRET)
friends = ""
while (friend_cursor != 0):
following = twitter.get_friends_ids(id=usr_id,
cursor=friend_cursor)
for ID in following['ids']:
friends += str(ID) + " "
friend_cursor = following["next_cursor"]
friends = friends[:-1]
if len(friends) == 0:
friends = "null"
follow = ""
while (follower_cursor != 0):
followers = twitter.get_followers_ids(id=usr_id,
cursor=follower_cursor)
for ID in followers['ids']:
follow += str(ID) + " "
follower_cursor = followers["next_cursor"]
follow = follow[:-1]
if len(follow) == 0:
follow = "null"
return (friends, follow)
except Exception as e:
print e
if "429 (Too Many Requests)" in str(e):
#global index
index += 1
if index == len(APP_KEYS):
index = 0
print "sleepy time - 15 minutes"
print datetime.datetime.now()
time.sleep(910)
return get_followers_following(usr_id)
elif "401 (Unauthorized)" in str(e):
print "401 error"
return ("null", "null")
elif "404 (Not Found)" in str(e):
print "404 error"
return ("null", "null")
else:
print e
return ("null", "null")
class TwythonAPITestCase(unittest.TestCase):
def setUp(self):
self.api = Twython(app_key, app_secret,
oauth_token, oauth_token_secret)
# Timelines
def test_get_mentions_timeline(self):
'''Test returning mentions timeline for authenticated user succeeds'''
self.api.get_mentions_timeline()
def test_get_user_timeline(self):
'''Test returning timeline for authenticated user and random user
succeeds'''
self.api.get_user_timeline() # Authenticated User Timeline
self.api.get_user_timeline(screen_name='twitter') # Random User Timeline
def test_get_protected_user_timeline_following(self):
'''Test returning a protected user timeline who you are following
succeeds'''
self.api.get_user_timeline(screen_name=protected_twitter_1)
def test_get_protected_user_timeline_not_following(self):
'''Test returning a protected user timeline who you are not following
fails and raise a TwythonAuthError'''
self.assertRaises(TwythonAuthError, self.api.get_user_timeline,
screen_name=protected_twitter_2)
def test_get_home_timeline(self):
'''Test returning home timeline for authenticated user succeeds'''
self.api.get_home_timeline()
# Tweets
def test_get_retweets(self):
'''Test getting retweets of a specific tweet succeeds'''
self.api.get_retweets(id=test_tweet_id)
def test_show_status(self):
'''Test returning a single status details succeeds'''
self.api.show_status(id=test_tweet_id)
def test_update_and_destroy_status(self):
'''Test updating and deleting a status succeeds'''
status = self.api.update_status(status='Test post just to get deleted :(')
self.api.destroy_status(id=status['id_str'])
def test_retweet(self):
'''Test retweeting a status succeeds'''
retweet = self.api.retweet(id='99530515043983360')
self.api.destroy_status(id=retweet['id_str'])
def test_retweet_twice(self):
'''Test that trying to retweet a tweet twice raises a TwythonError'''
retweet = self.api.retweet(id='99530515043983360')
self.assertRaises(TwythonError, self.api.retweet,
id='99530515043983360')
# Then clean up
self.api.destroy_status(id=retweet['id_str'])
def test_get_oembed_tweet(self):
'''Test getting info to embed tweet on Third Party site succeeds'''
self.api.get_oembed_tweet(id='99530515043983360')
def test_get_retweeters_ids(self):
'''Test getting ids for people who retweeted a tweet succeeds'''
self.api.get_retweeters_ids(id='99530515043983360')
# Search
def test_search(self):
'''Test searching tweets succeeds'''
self.api.search(q='twitter')
# Direct Messages
def test_get_direct_messages(self):
'''Test getting the authenticated users direct messages succeeds'''
self.api.get_direct_messages()
def test_get_sent_messages(self):
'''Test getting the authenticated users direct messages they've
sent succeeds'''
self.api.get_sent_messages()
def test_send_get_and_destroy_direct_message(self):
'''Test sending, getting, then destory a direct message succeeds'''
message = self.api.send_direct_message(screen_name=protected_twitter_1,
text='Hey d00d!')
self.api.get_direct_message(id=message['id_str'])
self.api.destroy_direct_message(id=message['id_str'])
def test_send_direct_message_to_non_follower(self):
'''Test sending a direct message to someone who doesn't follow you
fails'''
self.assertRaises(TwythonError, self.api.send_direct_message,
screen_name=protected_twitter_2, text='Yo, man!')
# Friends & Followers
def test_get_user_ids_of_blocked_retweets(self):
'''Test that collection of user_ids that the authenticated user does
not want to receive retweets from succeeds'''
self.api.get_user_ids_of_blocked_retweets(stringify_ids='true')
def test_get_friends_ids(self):
'''Test returning ids of users the authenticated user and then a random
user is following succeeds'''
self.api.get_friends_ids()
self.api.get_friends_ids(screen_name='twitter')
def test_get_followers_ids(self):
'''Test returning ids of users the authenticated user and then a random
user are followed by succeeds'''
self.api.get_followers_ids()
self.api.get_followers_ids(screen_name='twitter')
def test_lookup_friendships(self):
'''Test returning relationships of the authenticating user to the
comma-separated list of up to 100 screen_names or user_ids provided
succeeds'''
self.api.lookup_friendships(screen_name='twitter,ryanmcgrath')
def test_get_incoming_friendship_ids(self):
'''Test returning incoming friendship ids succeeds'''
self.api.get_incoming_friendship_ids()
def test_get_outgoing_friendship_ids(self):
'''Test returning outgoing friendship ids succeeds'''
self.api.get_outgoing_friendship_ids()
def test_create_friendship(self):
'''Test creating a friendship succeeds'''
self.api.create_friendship(screen_name='justinbieber')
def test_destroy_friendship(self):
'''Test destroying a friendship succeeds'''
self.api.destroy_friendship(screen_name='justinbieber')
def test_update_friendship(self):
'''Test updating friendships succeeds'''
self.api.update_friendship(screen_name=protected_twitter_1,
retweets='true')
self.api.update_friendship(screen_name=protected_twitter_1,
retweets='false')
def test_show_friendships(self):
'''Test showing specific friendship succeeds'''
self.api.show_friendship(target_screen_name=protected_twitter_1)
def test_get_friends_list(self):
'''Test getting list of users authenticated user then random user is
following succeeds'''
self.api.get_friends_list()
self.api.get_friends_list(screen_name='twitter')
def test_get_followers_list(self):
'''Test getting list of users authenticated user then random user are
followed by succeeds'''
self.api.get_followers_list()
self.api.get_followers_list(screen_name='twitter')
# Users
def test_get_account_settings(self):
'''Test getting the authenticated user account settings succeeds'''
self.api.get_account_settings()
def test_verify_credentials(self):
'''Test representation of the authenticated user call succeeds'''
self.api.verify_credentials()
def test_update_account_settings(self):
'''Test updating a user account settings succeeds'''
self.api.update_account_settings(lang='en')
def test_update_delivery_service(self):
'''Test updating delivery settings fails because we don't have
a mobile number on the account'''
self.assertRaises(TwythonError, self.api.update_delivery_service,
device='none')
def test_update_profile(self):
'''Test updating profile succeeds'''
self.api.update_profile(include_entities='true')
def test_update_profile_colors(self):
'''Test updating profile colors succeeds'''
self.api.update_profile_colors(profile_background_color='3D3D3D')
def test_list_blocks(self):
'''Test listing users who are blocked by the authenticated user
succeeds'''
self.api.list_blocks()
def test_list_block_ids(self):
'''Test listing user ids who are blocked by the authenticated user
succeeds'''
self.api.list_block_ids()
def test_create_block(self):
'''Test blocking a user succeeds'''
self.api.create_block(screen_name='justinbieber')
def test_destroy_block(self):
'''Test unblocking a user succeeds'''
self.api.destroy_block(screen_name='justinbieber')
def test_lookup_user(self):
'''Test listing a number of user objects succeeds'''
self.api.lookup_user(screen_name='twitter,justinbieber')
def test_show_user(self):
'''Test showing one user works'''
self.api.show_user(screen_name='twitter')
def test_search_users(self):
'''Test that searching for users succeeds'''
self.api.search_users(q='Twitter API')
def test_get_contributees(self):
'''Test returning list of accounts the specified user can
contribute to succeeds'''
self.api.get_contributees(screen_name='TechCrunch')
def test_get_contributors(self):
'''Test returning list of accounts that contribute to the
authenticated user fails because we are not a Contributor account'''
self.assertRaises(TwythonError, self.api.get_contributors,
screen_name=screen_name)
def test_remove_profile_banner(self):
'''Test removing profile banner succeeds'''
self.api.remove_profile_banner()
def test_get_profile_banner_sizes(self):
'''Test getting list of profile banner sizes fails because
we have not uploaded a profile banner'''
self.assertRaises(TwythonError, self.api.get_profile_banner_sizes)
# Suggested Users
def test_get_user_suggestions_by_slug(self):
'''Test getting user suggestions by slug succeeds'''
self.api.get_user_suggestions_by_slug(slug='twitter')
def test_get_user_suggestions(self):
'''Test getting user suggestions succeeds'''
self.api.get_user_suggestions()
def test_get_user_suggestions_statuses_by_slug(self):
'''Test getting status of suggested users succeeds'''
self.api.get_user_suggestions_statuses_by_slug(slug='funny')
# Favorites
def test_get_favorites(self):
'''Test getting list of favorites for the authenticated
user succeeds'''
self.api.get_favorites()
def test_create_and_destroy_favorite(self):
'''Test creating and destroying a favorite on a tweet succeeds'''
self.api.create_favorite(id=test_tweet_id)
self.api.destroy_favorite(id=test_tweet_id)
# Lists
def test_show_lists(self):
'''Test show lists for specified user'''
self.api.show_lists(screen_name='twitter')
def test_get_list_statuses(self):
'''Test timeline of tweets authored by members of the
specified list succeeds'''
self.api.get_list_statuses(list_id=test_list_id)
def test_create_update_destroy_list_add_remove_list_members(self):
'''Test create a list, adding and removing members then
deleting the list succeeds'''
the_list = self.api.create_list(name='Stuff')
list_id = the_list['id_str']
self.api.update_list(list_id=list_id, name='Stuff Renamed')
# Multi add/delete members
self.api.create_list_members(list_id=list_id,
screen_name='johncena,xbox')
self.api.delete_list_members(list_id=list_id,
screen_name='johncena,xbox')
# Single add/delete member
self.api.add_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list(list_id=list_id)
def test_get_list_memberships(self):
'''Test list of lists the authenticated user is a member of succeeds'''
self.api.get_list_memberships()
def test_get_list_subscribers(self):
'''Test list of subscribers of a specific list succeeds'''
self.api.get_list_subscribers(list_id=test_list_id)
def test_subscribe_is_subbed_and_unsubscribe_to_list(self):
'''Test subscribing, is a list sub and unsubbing to list succeeds'''
self.api.subscribe_to_list(list_id=test_list_id)
# Returns 404 if user is not a subscriber
self.api.is_list_subscriber(list_id=test_list_id,
screen_name=screen_name)
self.api.unsubscribe_from_list(list_id=test_list_id)
def test_is_list_member(self):
'''Test returning if specified user is member of a list succeeds'''
# Returns 404 if not list member
self.api.is_list_member(list_id=test_list_id, screen_name='jack')
def test_get_list_members(self):
'''Test listing members of the specified list succeeds'''
self.api.get_list_members(list_id=test_list_id)
def test_get_specific_list(self):
'''Test getting specific list succeeds'''
self.api.get_specific_list(list_id=test_list_id)
def test_get_list_subscriptions(self):
'''Test collection of the lists the specified user is
subscribed to succeeds'''
self.api.get_list_subscriptions(screen_name='twitter')
def test_show_owned_lists(self):
'''Test collection of lists the specified user owns succeeds'''
self.api.show_owned_lists(screen_name='twitter')
# Saved Searches
def test_get_saved_searches(self):
'''Test getting list of saved searches for authenticated
user succeeds'''
self.api.get_saved_searches()
def test_create_get_destroy_saved_search(self):
'''Test getting list of saved searches for authenticated
user succeeds'''
saved_search = self.api.create_saved_search(query='#Twitter')
saved_search_id = saved_search['id_str']
self.api.show_saved_search(id=saved_search_id)
self.api.destroy_saved_search(id=saved_search_id)
# Places & Geo
def test_get_geo_info(self):
'''Test getting info about a geo location succeeds'''
self.api.get_geo_info(place_id='df51dec6f4ee2b2c')
def test_reverse_geo_code(self):
'''Test reversing geocode succeeds'''
self.api.reverse_geocode(lat='37.76893497', long='-122.42284884')
def test_search_geo(self):
'''Test search for places that can be attached
to a statuses/update succeeds'''
self.api.search_geo(query='Toronto')
def test_get_similar_places(self):
'''Test locates places near the given coordinates which
are similar in name succeeds'''
self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ')
# Trends
def test_get_place_trends(self):
'''Test getting the top 10 trending topics for a specific
WOEID succeeds'''
self.api.get_place_trends(id=1)
def test_get_available_trends(self):
'''Test returning locations that Twitter has trending
topic information for succeeds'''
self.api.get_available_trends()
def test_get_closest_trends(self):
'''Test getting the locations that Twitter has trending topic
information for, closest to a specified location succeeds'''
self.api.get_closest_trends(lat='37', long='-122')
class TwitterBot():
'''
Crawls twitter and updates social network and tagged resources to repository
'''
def __init__(self):
self.service = TWITTER
APP_KEY = "w7zjbW1qHtzX2GxbrHbeNgyPr"
APP_SECRET = "qJFawkFxwYZX6U3DYNvJI3czWEjix2owU9wTXZ9BorMkhFU4Yb"
self.twapi = Twython(APP_KEY, APP_SECRET)
self.userBaseUri = 'http://twitter.com/%s' # % user screen_name
self.tagBaseUri = 'http://twitter.com/#search?q=#%s' # % trend key word
self.tweetBaseUri = 'http://twitter.com/%s/status/%s' # % (user screen_name, status id)
@staticmethod
def getname():
return self.service
@staticmethod
def getDefaultAvatar():
return defAvatar
def setStartUserId(self, start_user_id):
self.start_user_id = start_user_id
def setMaxLevel(self, max_level):
self.max_level = max_level
def setLastAdded(self, last_added):
self.last_added = last_added
def updateDatabase(self, user_metadata, user_ntw, udate=None):
raise NotImplementedError()
def fetchUser(self, username):
return (None, {'knows': [], 'fans': []})
def getUserUri(self, username):
return self.userBaseUri % username
@staticmethod
def factory(user, params=None, depth=2, last_added=None, verbose=False):
# Twitter crawler
twcrawl = TwitterBot()
twcrawl.setStartUserId(user)
twcrawl.setMaxLevel(depth)
twcrawl.setLastAdded(last_added)
return twcrawl
def updateDatabase(self, user_metadata, user_ntw, udate=None):
username = user_metadata['username']
userUri = self.userBaseUri % username
resourceUri = None
added = 0
# add user to repository
dbaccess.User.addToDB(userUri,
self.service,
username,
TwitterBot.getDefaultAvatar(),
udate=udate)
# add resources, tags and relations between them to repository
for resource in user_metadata['data']:
# add tweet resource to repository
resourceId = resource['id']
resourceTitle = resource['text']
resourceUri = self.tweetBaseUri % (username, resourceId)
dbaccess.Tweet.addToDB(resourceUri, resourceTitle, udate=udate)
added += 1
# add tag(trend) relations to repository
for tag in self.parseTags(resourceTitle):
tagUri = self.tagBaseUri % urllib.quote(tag.encode('utf-8'))
# add tag to repository
dbaccess.Tag.addToDB(tagUri, tag, udate=udate)
# add user, resource, tag relation to repository
dbaccess.UserResourceTag.addToDB(userUri,
resourceUri,
tagUri,
udate=udate)
# add url relations between the tweet and urls contained
urls = map(self.getRealUrl, self.parseUrls(resourceTitle))
for url in urls:
dbaccess.Bookmark.addToDB(url, None)
dbaccess.Tweet.addReferenceToDB(resourceUri, url)
# add user social network relations to repository
# add friends
for u in user_ntw['knows']:
knownUserUri = self.userBaseUri % u
dbaccess.User.addToDB(knownUserUri,
self.service,
u,
TwitterBot.getDefaultAvatar(),
udate=udate)
dbaccess.User.addToDBUserKnowsUser(userUri,
knownUserUri,
udate=udate)
# add followers
for u in user_ntw['fans']:
otherUserUri = self.userBaseUri % u
dbaccess.User.addToDB(otherUserUri,
self.service,
u,
TwitterBot.getDefaultAvatar(),
udate=udate)
dbaccess.User.addToDBUserKnowsUser(otherUserUri,
userUri,
udate=udate)
# return number of added resources and last added resource uri
return added
def fetchUserMetadata(self, username):
# get user tweets (only last 20)
data = self.twapi.get_user_timeline(screen_name=username)
return {'username': username, 'data': data}
def fetchUserNetwork(self, username):
chunk_size = 100 # max returned by bulkUserLookup
# Get Friends
fids, cursor = [], -1
fchunks = [
fids[i:i + chunk_size] for i in range(0, len(fids), chunk_size)
]
while True:
f = self.twapi.get_friends_ids(screen_name=username, cursor=cursor)
fids.extend(f['ids'])
if f['next_cursor'] == cursor or len(f['ids']) == 0:
break
cursor = f['next_cursor']
# get screen names from ids in chunks of 100
screen_names = []
for chunk in fchunks:
screen_names = []
screen_names.extend([
userdata['screen_name']
for userdata in self.twapi.lookup_user(user_id=chunk)
])
ntw_knows = screen_names
print ntw_knows
# Get Followers
fids, cursor = [], -1
while True:
f = self.twapi.get_followers_ids(screen_name=username,
cursor=cursor)
fids.extend(f['ids'])
if f['next_cursor'] == cursor or len(f['ids']) == 0:
break
cursor = f['next_cursor']
fchunks = [
fids[i:i + chunk_size] for i in range(0, len(fids), chunk_size)
]
fan_screen_names = []
for chunk in fchunks:
# get all data for all user ids
fan_screen_names.extend([
userdata['screen_name']
for userdata in self.twapi.lookup_user(user_id=chunk)
])
ntw_fans = fan_screen_names
return {'knows': ntw_knows, 'fans': ntw_fans}
def fetchUser(self, username):
return (self.fetchUserMetadata(username),
self.fetchUserNetwork(username))
def parseTags(self, text):
'''
Parses trends from tweet: '#trend #tre#nd ## #trend,two #tren%^& smth,#da #1dc #100 #1dc$ #100^',
finds trends: trend, tre, trend, tren, da, 1dc, 100, 1dc, 100.
Text parameter MUST be UNICODE encoded: u'some text'
'''
return re.findall('(?<=[\W^#]#)\w+', ' ' + text, re.UNICODE)
def parseUrls(self, text):
'''Returns a list of urls from the text.
'''
return re.findall('http://\S+', text)
def getRealUrl(self, url):
'''Returns the redirected url if *url* parameter is a shortened url (a redirect was produced),
or itself if not.
'''
try:
u = urllib2.urlopen(url)
realurl = u.geturl()
if realurl == None:
raise urllib2.HTTPError('Could not fetch url')
except:
realurl = url
return realurl
def crawlUserNetwork(self, user_id=None, max_level=0, start_time=None):
'''
user_id - an id to uniquely identify a user (can be a username or userid)
'''
user_id = user_id or self.start_user_id
max_level = max_level or self.max_level
queue = [user_id] # init queue
visited = {user_id: 0} # init visited nodes
added = 0 # set number of added resources
while queue:
v = queue.pop(0) # pop next user node
level = visited[v] # get their level
# get user metadata and social network
user_metadata, user_ntw = self.fetchUser(v)
# update database with user data, tagged resources and social network relations
a = self.updateDatabase(user_metadata, user_ntw, udate=start_time)
added += a
# explore following nodes
for w in user_ntw['knows']:
if w not in visited and level < max_level:
queue.append(w)
visited[w] = level + 1
return added
stored = defaultdict(list)
#Reads in the csv with the Twitter handles
ifile = open('inpath.csv', "rb")
for name in ifile:
next_cursor = -1 #initiates the cursor
try:
while (next_cursor !=
0): #keeps looping through as long as there are pages there
print(name)
time.sleep(
60
) # sets a time delay of 60 seconds to keep within the rate limits
#gets the people followed by the screen name
following = twitter.get_friends_ids(screen_name=str(name),
cursor=next_cursor)
for x in following[
u'ids']: #loops through the ids returned by the Twitter api
name = name.encode('utf-8')
stored[str(name)].append(
x
) # and appends them to the dictionary where the key is the associated persons user id
next_cursor = following['next_cursor'] #gets the next cursor value
except: # if the call fails which may be because there are no more pages
print('unable to access the people followed by ' + str(name) + ',' +
str(next_cursor) + '\n')
#writes the dictionary as a json file to the computer
with open('outpath.json', 'w') as f:
json.dump(stored, f)
def createDatabase(request):
twitter = Twython(
settings.TWITTER_APP_KEY,
settings.TWITTER_APP_SECRET,
request.session['oauth_token'],
request.session['oauth_token_secret']
)
rels = {}
current_userid = request.session['oauth_token'].split('-')[0];
m = i = 5
mid = m/2
usersToQuery = [current_userid]
users = []
current_userdeets = twitter.verify_credentials()
users.append(current_userdeets)
while i > 0:
userid = usersToQuery[m-i]
print "Index ", str(i)
print "Querying userId ", str(userid)
followers = twitter.get_followers_ids(count=100, user_id=userid)['ids']
for f in followers:
if not f in rels:
rels[f] = [userid]
elif userid not in rels[f]:
rels[f].append(userid)
if len(followers) > 0:
users = users + twitter.lookup_user(user_id=",".join(map(str, followers)))
friends = twitter.get_friends_ids(count=100, user_id=userid)['ids']
for f in friends:
if not userid in rels:
rels[userid] = [f]
elif f not in rels[userid]:
rels[userid].append(f)
if len(friends) > 0:
users = users + twitter.lookup_user(user_id=",".join(map(str, friends)))
if i == m:
usersToQuery = usersToQuery + followers[0:mid] + friends[0:mid]
i = i-1
fields = ['screen_name', 'name', 'location', 'created_at', 'id', 'favourites_count', 'url', 'lang', 'followers_count', 'friends_count', 'statuses_count', 'profile_image_url']
allUsers = []
usersIndex = {}
for u in users:
if u['id'] in usersIndex:
continue
tu = {}
for f in fields:
tu[f] = u[f]
allUsers.append(tu)
usersIndex[u['id']] = 1
# print "writing.."
# save
# with io.open(os.path.dirname(__file__) + '/users.txt', 'w', encoding='utf-8') as f:
# f.write(unicode(json.dumps(allUsers, ensure_ascii=False)))
# with io.open(os.path.dirname(__file__) + '/rels.txt', 'w', encoding='utf-8') as f:
# f.write(unicode(json.dumps(rels, ensure_ascii=False)))
print 'Writing to neo4j..'
graph_db = getPy2neoService()
graph_db.clear()
batch = neo4j.WriteBatch(graph_db)
batches = {}
for u in allUsers:
batches[str(u['id'])] = batch.create(node(**u))
for r in rels:
for p in rels[r]:
if str(r) in batches and str(p) in batches:
batch.create(rel(batches[str(r)], "FOLLOWS", batches[str(p)]))
else:
print "Skipping.."
results = batch.submit()
print 'Writing done..'
return current_userdeets['screen_name']
class Follow_Garbage_Collector():
def __init__(self):
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.INFO)
self.config_file = "/config/Follow_Garbage_Collector.ini"
try:
with open(self.config_file, "r") as f:
f.close()
try:
self.config = configparser.ConfigParser()
self.config.read(self.config_file)
self.twitter_app_key = self.config['TWITTER']['APP_KEY']
self.twitter_app_secret = self.config['TWITTER']['APP_SECRET']
self.twitter_oauth_token = \
self.config['TWITTER']['OAUTH_TOKEN']
self.twitter_oauth_token_secret = \
self.config['TWITTER']['OAUTH_TOKEN_SECRET']
self.twitter_unfollow = True if (
self.config['TWITTER']['UNFOLLOW'] == "ON") else False
self.twitter_years_inactive = \
int(self.config['TWITTER']['YEARS_INACTIVE'])
self.twitter_excluded_tweeps = list(
self.config['TWITTER']['EXCLUDED_TWEEPS'].split(","))
self.pushover_user_key = self.config['PUSHOVER']['USER_KEY']
self.pushover_token_api = self.config['PUSHOVER']['TOKEN_API']
self.pushover_sound = self.config['PUSHOVER']['SOUND']
except KeyError:
logging.error("Can't get keys from INI file. "
"Please check for mistakes.")
sys.exit()
except IOError or FileNotFoundError:
logging.error(f"Can't open file {self.config_file}, "
f"creating example INI file.")
shutil.copyfile('/app/Follow_Garbage_Collector.ini.example',
'/config/Follow_Garbage_Collector.ini.example')
sys.exit()
# Convert UTC times to local times
def datetime_from_utc_to_local(self, utc_datetime):
now_timestamp = time()
offset = datetime.fromtimestamp(
now_timestamp) - datetime.utcfromtimestamp(now_timestamp)
return utc_datetime + offset
# convert a tweetdatetime to datetime_utc
def tweetdatetime_to_datetime_utc(self, tweetDate):
return self.datetime_from_utc_to_local(
datetime.strptime(tweetDate, "%a %b %d %H:%M:%S +0000 %Y"))
def run(self):
# Log a run
logging.info("Executing Follow Garbage Collector.")
# Setting for PushOver
self.appPushover = Application(self.pushover_token_api)
self.userPushover = self.appPushover.get_user(self.pushover_user_key)
try:
# This time we want to set our q to search for our keywords
self.twitter = Twython(
self.twitter_app_key,
self.twitter_app_secret,
self.twitter_oauth_token,
self.twitter_oauth_token_secret,
)
friends = self.twitter.get_friends_ids()
for friend in friends["ids"]:
user_timeline = self.twitter.get_user_timeline(user_id=friend,
count=1)
for tweet in user_timeline:
diffDate = datetime.now() - \
self.datetime_from_utc_to_local(
self.tweetdatetime_to_datetime_utc(
tweet["created_at"]))
if diffDate.days >= 365 * self.twitter_years_inactive:
tweetDate = datetime.strftime(
self.datetime_from_utc_to_local(
self.tweetdatetime_to_datetime_utc(
tweet["created_at"])),
"%Y-%m-%d %H:%M:%S",
)
if (not tweet["user"]["screen_name"]
in self.twitter_excluded_tweeps):
if self.twitter_unfollow:
self.twitter.destroy_friendship(user_id=friend)
self.message = self.userPushover.send_message(
message=f'Flushed @'
f'{tweet["user"]["screen_name"]} '
f'- {tweet["user"]["name"]}\n{tweetDate}\n'
f'{tweet["text"]}',
sound=self.pushover_sound)
# Log a flush
logging.info(
f'Flushed @'
f'{tweet["user"]["screen_name"]} '
f'- {tweet["user"]["name"]} - {tweetDate}')
# trying not to upset the Twitter Gods
sleep(2)
except TwythonError as e:
print(e)
print "Index ", str(i) print "Querying userId ", str(userid) followers = twitter.get_followers_ids(count=100, user_id=userid)['ids'] t = time.time() for f in followers: if not f in rels: rels[f] = [userid] elif userid not in rels[f]: rels[f].append(userid) print time.time() - t users = users + twitter.lookup_user(user_id=",".join(map(str, followers))) friends = twitter.get_friends_ids(count=100, user_id=userid)['ids'] t = time.time() for f in friends: if not userid in rels: rels[userid] = [f] elif f not in rels[userid]: rels[userid].append(f) print time.time() - t users = users + twitter.lookup_user(user_id=",".join(map(str, friends))) if i == m: usersToQuery = usersToQuery + followers[0:mid] + friends[0:mid] i = i-1
class RecolectorSiguiendoShort(Recolector):
def __init__(self, escritores):
super(RecolectorSiguiendoShort, self).__init__(escritores)
self.authorizator = GetAuthorizations(15)
self.twitter = None
self.apoyo = ApoyoTwitter()
self.tipo_id = 5
self.inicializa()
self.cursor = -1
def inicializa(self):
self.authorizator.load_twitter_token(self.tipo_id)
api_key, access_token = self.authorizator.get_twython_token()
self.twitter = Twython(api_key, access_token=access_token)
def recolecta(self, query=None, id_user=-1, complete=False):
self.cursor = -1
arrayUsuarios = []
if query is None and id_user == -1:
raise Exception('Al menos debe haber un parametro usable')
if query is not None:
if query[0] == '@':
query = query[1:]
id_user = self.apoyo.getUserIDByScreenName(query)
if id_user == None:
raise Exception('El usuario debe estar en la base de datos')
retornoFinal = {"ids": []}
while True:
retorno = self.privateRealizaConsultaById(id_user)
if retorno == []:
break
retornoFinal["ids"].extend(retorno["ids"])
if len(retorno["ids"]) < 5000:
break
if complete == False:
break
#se almacenan las relaciones en una tupla (usuario1, usuario2)
#esto quiere decir que el usuario1 sigue al usuario2.
lista_relaciones = []
for identificador in retornoFinal["ids"]:
lista_relaciones.append((id_user, identificador))
self.guarda(lista_relaciones)
def guarda(self, arrayDatos):
for escritor in self.escritores:
escritor.escribe(arrayDatos)
def privateRealizaConsultaById(self, identificador):
if self.authorizator.is_limit_api(self.tipo_id):
raise Exception('LIMITE')
try:
retorno = self.twitter.get_friends_ids(user_id=identificador,
cursor=self.cursor,
count='5000')
self.authorizator.add_query_to_key(self.tipo_id)
if len(retorno["ids"]) == 0:
return []
return retorno
except Exception, e:
self.authorizator.add_query_to_key(self.tipo_id)
print e
if "429" in str(e):
raise Exception('LIMITE')
return []
newTweet = generate()
twitter.update_status(status=newTweet)
def administration(items):
""" follow or reject new users who put in commands """
for tweet in reversed(items):
if (tweet['user']['id'] == 22884755):
text = tweet['text'][10:].split(" ")
if (text[1] == "approve"):
twitter.create_friendship(screen_name=text[2])
twitter.update_status(
status="@" + text[2] +
" Good news, you've been approved! Please retry any additions prior to this message again."
)
elif (text[1] == "reject"):
twitter.update_status(
status="@" + text[2] +
" Sorry, I'm not going to add you right now.")
tweets = twitter.get_mentions_timeline()
getLast()
intake(
simplify(twitter.get_mentions_timeline(),
twitter.get_friends_ids()['ids']))
onDemand(tweets)
administration(tweets)
periodic()
