def main(argv):
# Query terms
print(sys.stderr, 'Filtering the public timeline for locations: melbourne')
port = 5984
hostname = '127.0.0.1'
db = 'tweet'
try:
opts, args = getopt.getopt(argv, "p:h:d:")
except getopt.GetoptError as error:
print(error)
sys.exit(2)
for opt, arg in opts:
if opt == '-p':
port = arg
elif opt in "-h":
hostname = arg
elif opt in "-d":
db = arg
t = oauth_login() # Returns an instance of twitter.Twitter
twitter_stream = twitter.TwitterStream(
auth=t.auth) # Reference the self.auth parameter
melbourne = "144.35,-38.26,145.30,-37.30"
stream = twitter_stream.statuses.filter(
locations=melbourne
) # Use the bounding box of Melbourne as filter parameter.
db = setup_db(hostname=hostname, port=port, db=db)
for tweet in stream:
if "text" in tweet.keys():
if "coordinates" in tweet.keys(
) and tweet['coordinates'] is not None:
if 144.35 < tweet['coordinates']['coordinates'][
0] < 145.30 and -38.26 < tweet['coordinates'][
'coordinates'][1] < -37.30:
# Futher filtering for tweets with coorinates and in Melbourne.
try:
print(tweet['text'])
save_to_db(tweet, db)
except:
pass
try:
print kw
print args
return twitter_api_func(*args,**kw)
except twitter.api.TwitterHTTPError,e:
error_count=0
wait_period=handle_twitter_http_error(e,wait_period)
if wait_period is None:
return
except URLError,e:
error_count+=1
print >> sys.stderr,"URLError encountered. Continuing"
if error_count>max_errors:
print >> sys.stderr,"Too many consecutive errors...bailing out"
raise
except BadStatusLine,e:
error_count+=1
print >> sys.stderr,"BadStatusLine encountered. Continuing"
if error_count>max_errors:
print >> sys.stderr,"Too many consecutive errors... bailing out"
raise
from oauth_login import oauth_login
twitter_api=oauth_login()
# response=twitter_api.users.lookup(screen_name='SocialWebMining')
# print twitter_api.users.lookup(screen_name='SocialWebMining')
response=make_twitter_request(twitter_api.users.lookup,screen_name="SocialWebMining")
print json.dumps(response,indent=1)
def get_redis_id(key_name, screen_name=None, user_id=None):
if screen_name is not None: #ex) 'screen_name$oisoo$friends_ids'
return 'screen_name$' + screen_name + '$' + key_name
elif user_id is not None:
return 'user_id$' + user_id + '$' + key_name
else:
raise Exception("No screen_name or user_id provided to get_redis_id")
if __name__ == '__main__':
SCREEN_NAME = sys.argv[1]
MAX_IDS = int(sys.argv[2])
r = redis.Redis()
t = oauth_login(sys.argv[3])
# Harvest some friend ids
get_friends_ids = functools.partial(make_tweepy_request, t, t.friends_ids)
cursor = -1
ids = []
while cursor != 0:
if(MAX_IDS > 5000):
cnt = 5000
else:
cnt = MAX_IDS
# Use make_twitter_request via the partially bound callable...
response = get_friends_ids(screen_name=SCREEN_NAME, cursor=cursor , count = cnt)
# -*- coding: utf-8 -*-
import sys
import twitter
from recipe__make_tweepy_request import make_tweepy_request
from oauth_login import oauth_login
import functools
SCREEN_NAME = sys.argv[2]
MAX_IDS = int(sys.argv[3])
TYPE = sys.argv[4] # 0 : Friends, 1 : Follows
if __name__ == '__main__':
# Not authenticating lowers your rate limit to 150 requests per hr.
# Authenticate to get 350 requests per hour.
oauthID = sys.argv[1]
t = oauth_login(oauthID)
#twitter.Twitter(domain='api.twitter.com', api_version='1')
# You could call make_twitter_request(t, t.friends.ids, *args, **kw) or
# use functools to "partially bind" a new callable with these parameters
if (TYPE == str(0)): #Friends ids
get_friends_ids = functools.partial(make_tweepy_request, t,
t.friends_ids)
elif (TYPE == str(1)): #Followers ids
get_followers_ids = functools.partial(make_tweepy_request, t,
t.followers_ids)
# Ditto if you want to do the same thing to get followers...
# getFollowerIds = functools.partial(make_twitter_request, t, t.followers.ids)
cursor = -1
ids = []
i = 1
while cursor != 0:
# Use make_twitter_request via the partially bound callable...
# -*- coding: utf-8 -*- import sys import twitter from recipe__make_tweepy_request import make_tweepy_request from oauth_login import oauth_login import functools SCREEN_NAME = sys.argv[2] MAX_IDS = int(sys.argv[3]) TYPE = sys.argv[4] # 0 : Friends, 1 : Follows if __name__ == '__main__': # Not authenticating lowers your rate limit to 150 requests per hr. # Authenticate to get 350 requests per hour. oauthID = sys.argv[1] t = oauth_login(oauthID) #twitter.Twitter(domain='api.twitter.com', api_version='1') # You could call make_twitter_request(t, t.friends.ids, *args, **kw) or # use functools to "partially bind" a new callable with these parameters if(TYPE == str(0)): #Friends ids get_friends_ids = functools.partial(make_tweepy_request, t, t.friends_ids) elif(TYPE == str(1)): #Followers ids get_followers_ids = functools.partial(make_tweepy_request, t, t.followers_ids) # Ditto if you want to do the same thing to get followers... # getFollowerIds = functools.partial(make_twitter_request, t, t.followers.ids) cursor = -1 ids = [] i = 1 while cursor != 0: # Use make_twitter_request via the partially bound callable... if(MAX_IDS > 5000): cnt = 5000
# print search_results
# json.dumps(search_results)
# return
file_name = '../data/twitter/twitter_json_' + str(search_ctr) + '.txt'
print file_name
# with io.open(file_name, 'w', encoding='utf-8') as f:
# f.write(unicode(json.dumps(search_results, ensure_ascii=False)))
file_h = open(file_name,'w')
file_h.write(unicode(json.dumps(search_results, ensure_ascii=False)))
file_h.close()
search_ctr = search_ctr + 1
sleep_time = 40
time.sleep(sleep_time)
error_count = 0
except twitter.api.TwitterHTTPError, e:
error_count = 0
wait_period = handle_http_error(e, t, wait_period)
if wait_period is None:
return
except URLError, e:
error_count += 1
print >> sys.stderr, "URLError encountered. Continuing."
if error_count > max_errors:
print >> sys.stderr, "Too many consecutive errors...bailing out."
raise
if __name__ == '__main__':
t = oauth_login.oauth_login()
make_twitter_request(t, 3)
projection=None,
**mongo_conn_kw):
client = pymongo.MongoClient(**mongo_conn_kw)
db = client[mongo_db]
coll = db[mongo_db_coll]
if criteria is None:
criteria = {}
if projection is None:
cursor = coll.find(criteria)
else:
cursor = coll.find(criteria, projection)
if return_cursor:
return cursor
else:
return [item for item in cursor]
#sample usage
twitter_api = oauth_login.oauth_login()
INDIA_WOE_ID = 23424977
trends = extract.twitter_trends(twitter_api, INDIA_WOE_ID)
print json.dumps(trends, indent=1)
save_to_mongo(trends, 'tweets', 'USA') #saving to mongo db
results = load_from_mongo('tweets', 'USA') # loading from mongo db
