# The following code depends on these modules. import sys, json, pickle from hcde.twitter.Login import Login from hcde.twitter.Search import Search from hcde.twitter.auth_settings import * # # Creates an authorized login object app = "HCDE530Test01" user = "******" # # The next two lines come from auth_settings and help manage keys app_keys = TWITTER_APP_OAUTH_PAIR(app=app) app_token_fname = TWITTER_APP_TOKEN_FNAME(app=app) # # Creates a login object to authenticate twitter requests lg = Login(app_name=app, app_user=user, token_fname=app_token_fname) lg.set_consumer_key(consumer_key=app_keys['consumer_key']) lg.set_consumer_secret(consumer_secret=app_keys['consumer_secret']) lg.login() # # Create a new search object and set some required # object values to enable a search search = Search() search.set_auth_obj(obj=lg) # Add the authentication object search.set_user_agent(agent="ie") search.set_throttling(True) # throttle the requests search.set_query_result_type(rt="recent") search.set_page_size(sz=100) # # This is the part you expected, we set some search terms, just # one term in this specific case
def main(argv):
if len(argv) < 5:
usage(argv)
p = parse_params(argv)
#print p
twit = Search()
twit.set_user_agent(agent="random")
twit.set_throttling(True)
if (p['debug']):
twit.debug_output = True
lg = None
if (not p['auth'] and not p['user']):
print "Must have authenticating User and Application!"
usage(argv)
return
if (p['auth']):
app = p['auth']
app_keys = TWITTER_APP_OAUTH_PAIR(app=p['auth'])
app_token_fname = TWITTER_APP_TOKEN_FNAME(app=p['auth'])
lg = Login(name="SearchLoginObj",
app_name=p['auth'],
app_user=p['user'],
token_fname=app_token_fname)
if (p['debug']):
lg.set_debug(True)
## Key and secret for specified application
lg.set_consumer_key(consumer_key=app_keys['consumer_key'])
lg.set_consumer_secret(consumer_secret=app_keys['consumer_secret'])
lg.login()
twit.set_auth_obj(obj=lg)
if (not p['query']):
print "Must provide some query terms!"
usage(argv)
if (p['use_continuations']):
print "Using Continuations for requests"
twit.set_continuation(True)
if (p['lang']):
twit.set_query_lang(p['lang'])
print "Query Term", p['query']
twit.set_query_terms(p['query'].encode('utf-8'))
twit.set_page_size(sz=p['size'])
twit.set_query_result_type(rt="recent")
#twit.set_continuation(True)
twit.start_thread()
twit.start_request()
# The request is being made by an asynchronous thread, we need
# to wait until that thread is done before we can see the result.
#
# This convenience routine must be called by a different thread.
# In our case here, we're in the "__main__" thread which can make
# this call and safely wait until the twit thread is done.
twit.wait_request()
if (twit.messages() == 0):
print "No results from query."
m = None
tot = 0
tweet_count = 0
mesg_count = 0
while (twit.messages() > 0 or twit.query_in_process()):
m = twit.get_message()
mesg_count += 1
if (m):
tot = tot + len(m)
#print json.dumps(m, sort_keys=True, indent=4)
for rec in m:
tweet_count += 1
if (p['json']):
print "[m:%3d, tweet number: %3d]" % (mesg_count,
tweet_count)
print json.dumps(rec, sort_keys=True, indent=4)
else:
print "[m:%3d,t:%3d]" % (mesg_count, tweet_count),
print rec['user']['screen_name'].encode(
'utf-8'), rec['user']['name'].encode('utf-8'),
print ":", rec['text'].encode('utf-8')
print
while (twit.messages() < 1 and twit.query_in_process()):
time.sleep(2)
if (twit.had_warning()):
print "WARNING:", twit.get_last_warning()
if (twit.had_error()):
print "ERROR:", twit.get_last_error()
print "Continuation URL:", twit.get_continuation()
print "Total tweets: ", tot
if (p['limits']):
print "Limits:", twit.get_rate_limit(), twit._throttling()
twit.terminate_thread()
return
def main(argv):
if len(argv) < 5:
usage(argv)
p = parse_params(argv)
print p
twit = Search()
twit.set_user_agent(agent="random")
twit.set_throttling(True)
lg = None
if( not p['auth'] and not p['user'] ):
print "Must have authenticating User and Application!"
usage(argv)
return
if( p['auth'] ):
app = p['auth']
app_keys = TWITTER_APP_OAUTH_PAIR(app=p['auth'])
app_token_fname = TWITTER_APP_TOKEN_FNAME(app=p['auth'])
lg = Login( name="SampleTweetCollectorLoginObj",
app_name=p['auth'],
app_user=p['user'],
token_fname=app_token_fname)
if( p['debug'] ):
lg.set_debug(True)
## Key and secret for specified application
lg.set_consumer_key(consumer_key=app_keys['consumer_key'])
lg.set_consumer_secret(consumer_secret=app_keys['consumer_secret'])
lg.login()
twit.set_auth_obj(obj=lg)
if( not p['query'] ):
print "Must provide some query terms!"
usage(argv)
if( not p['airline_handle'] ):
print "Must provide airline handle!"
usage(argv)
if( p['use_continuations'] ):
print "Using Continuations for requests"
twit.set_continuation(True)
print "Collecting tweets for term:", p['query'].encode('utf-8')
twit.set_query_terms(p['query'].encode('utf-8'))
twit.set_query_result_type(rt="recent")
twit.set_page_size(sz=p['page_size'])
#twit.set_extended_tweet_mode(True)
#twit.set_page_size(sz=5)
#db_config = DBConfiguration(db_settings=DATABASE_SETTINGS['main_db'])
#db = ExampleTweetsDB(config=db_config)
db = sqlite3.connect('../database/united.db')
twit.start_thread()
cycle_count = 0
while( cycle_count < p['cycles'] ):
cycle_count += 1
twit.start_request()
twit.wait_request()
collection_loop(db=db,twit=twit,term=p['query'].encode('utf-8'), airline_handle=p['airline_handle'])
if( cycle_count < p['cycles'] ):
print ">>>>"
print ">>>> Waiting a few minutes before next query."
print ">>>>"
time.sleep(200.0)
twit.terminate_thread()
db.close()
return
def search_tweet(auth=None,
user=None,
size=100,
since=None,
until=None,
query=None):
twit = Search()
twit.set_user_agent(agent="random")
twit.set_throttling(True)
if (auth):
app_keys = TWITTER_APP_OAUTH_PAIR(app=auth)
app_token_fname = TWITTER_APP_TOKEN_FNAME(app=auth)
lg = Login(name="SearchLoginObj",
app_name=auth,
app_user=user,
token_fname=app_token_fname)
## Key and secret for specified application
lg.set_consumer_key(consumer_key=app_keys['consumer_key'])
lg.set_consumer_secret(consumer_secret=app_keys['consumer_secret'])
lg.login()
twit.set_auth_obj(obj=lg)
twit.set_continuation(True)
twit.set_extended_tweet_mode(True)
if (since and until):
twit.set_query_date_range(since, until)
print "Query Term", query
twit.set_query_terms(query.encode('utf-8'))
twit.set_page_size(sz=size)
twit.set_query_result_type(rt="recent")
twit.start_thread()
twit.start_request()
# The request is being made by an asynchronous thread, we need
# to wait until that thread is done before we can see the result.
#
# This convenience routine must be called by a different thread.
# In our case here, we're in the "__main__" thread which can make
# this call and safely wait until the twit thread is done.
twit.wait_request()
if (twit.messages() == 0):
print "No results from query."
m = None
tot = 0
mesg_count = 0
tweets = []
while (twit.messages() > 0 or twit.query_in_process()):
m = twit.get_message()
mesg_count += 1
if (m):
tot = tot + len(m)
#print json.dumps(m, sort_keys=True, indent=4)
for rec in m:
tweetText = ''
try:
tweetText = rec['text'].encode('utf-8')
except KeyError, ke:
tweetText = rec['full_text'].encode('utf-8')
tweetObj = {
'id_str': rec['id_str'],
'created_at': rec['created_at'],
'tweet': tweetText,
'retweet_count': rec['retweet_count'],
'favorite_count': rec['favorite_count'],
'lang': rec['lang'],
'in_reply_to_status_id_str':
rec['in_reply_to_status_id_str'],
'place_country_code': rec['place']['country_code'],
'place_full_name': rec['place']['full_name'],
'place_type': rec['place']['place_type'],
'user': {
'user_id_str': rec['user']['id_str'],
'name': rec['user']['name'],
'screen_name':
rec['user']['screen_name'].encode('utf-8'),
'location': rec['user']['location'],
'verified': rec['user']['verified'],
'followers_count': rec['user']['followers_count'],
'utc_offset': rec['user']['utc_offset'],
'time_zone': rec['user']['time_zone']
},
'entities': {
'hashtags': rec['entities']['hashtags'],
'user_mentions': rec['entities']['user_mentions'],
'urls': rec['entities']['urls']
}
}
tweets.append(tweetObj)
while (twit.messages() < 1 and twit.query_in_process()):
time.sleep(2)