def user_setup(output_dir, logfile): print 'Getting Configurations' logging.info('Getting Configurations') if output_dir: helpModule.make_outdir(root+output_dir) print 'Retrieving Twitter Profile' logging.info('Retrieving Twitter Profile') global __profile __profile = profileModule.get_profile(ppath, lpath) logging.debug('Using profile '+__profile.upper()) profilepath = ppath+__profile+'.profile' deets = profileModule.get_deets(profilepath) with open(logfile, 'a') as fo: fo.write(__profile+'\n') print 'Authorizing Twitter Profile' logging.info('Authorizing Twitter Profile') auth = tweepy.OAuthHandler(deets["consumer_key"], deets["consumer_secret"]) auth.set_access_token(deets["access_token"], deets["access_token_secret"]) global __twitterapi__ __twitterapi__ = tweepy.API(auth, parser=tweepy.parsers.JSONParser())
def user_setup(output_dir, logfile): print "Getting Configurations" logging.info("Getting Configurations") if output_dir: helpModule.make_outdir(root + output_dir) print "Retrieving Twitter Profile" logging.info("Retrieving Twitter Profile") global __profile __profile = profileModule.get_profile(ppath, lpath) logging.debug("Using profile " + __profile.upper()) profilepath = ppath + __profile + ".profile" deets = profileModule.get_deets(profilepath) with open(logfile, "a") as fo: fo.write(__profile + "\n") print "Authorizing Twitter Profile" logging.info("Authorizing Twitter Profile") auth = tweepy.OAuthHandler(deets["consumer_key"], deets["consumer_secret"]) auth.set_access_token(deets["access_token"], deets["access_token_secret"]) global __twitterapi__ __twitterapi__ = tweepy.API(auth, parser=tweepy.parsers.JSONParser())
configs = read_config() ppath = configs['ppath'] lpath = configs['lpath'] root = configs['dpath'] if root == 'False': root = './' outfile = root + output_dir + '/' + output_file fo = open(outfile, 'w') fo.close() parentid = str(os.getpid()) logfile = lpath+parentid+'.streamlog' helpModule.make_outdir(root+output_dir) profile = profileModule.get_profile(ppath, lpath) profilepath = ppath+profile+'.profile' deets = profileModule.get_deets(profilepath) #------------------------------------------------------------------------------------------------------------- class StdOutListener(StreamListener): def on_data(self, data): if 'in_reply_to_status' in data: with open(outfile,'a') as tf: tf.write(data) return True
configs = read_config() ppath = configs['ppath'] lpath = configs['lpath'] root = configs['dpath'] if root == 'False': root = './' outfile = root + output_dir + '/' + output_file fo = open(outfile, 'w') fo.close() parentid = str(os.getpid()) logfile = lpath + parentid + '.streamlog' helpModule.make_outdir(root + output_dir) profile = profileModule.get_profile(ppath, lpath) profilepath = ppath + profile + '.profile' deets = profileModule.get_deets(profilepath) #------------------------------------------------------------------------------------------------------------- class StdOutListener(StreamListener): def on_data(self, data): if 'in_reply_to_status' in data: with open(outfile, 'a') as tf: tf.write(data) return True
def searching(configs, query, output_file, output_dir, num): print 'Getting Configurations' time.sleep(0.5) ppath = configs['ppath'] lpath = configs['lpath'] root = configs['dpath'] if root == 'False': root = './' helpModule.make_outdir(root+output_dir) print 'Configuring Files' time.sleep(0.5) outfile = root + output_dir + '/' + output_file fo = open(outfile, 'w') fo.close() print 'Encoding Query' time.sleep(0.5) enc_query = quote_plus(query.encode('UTF-8'), safe=':/') pid = str(os.getpid()) logfile = lpath+pid+'.searchlog' print 'Retrieving Twitter Profile' time.sleep(0.5) profile = profileModule.get_profile(ppath, lpath) profilepath = ppath+profile+'.profile' deets = profileModule.get_deets(profilepath) print 'Authorizing Twitter Profile' time.sleep(0.5) auth = twitter.oauth.OAuth(deets["access_token"], deets["access_token_secret"], deets["consumer_key"], deets["consumer_secret"]) twitter_api = twitter.Twitter(auth=auth) count = 100 print 'Starting Search' time.sleep(0.5) search_results = twitter_api.search.tweets(q=enc_query, count=count) with open(logfile, 'a') as fo: fo.write(profile+'\n') fo.write(query+'\n') statuses = search_results['statuses'] counter = 0 tweets = 0 while True: counter+=len(statuses) print "Tweets Collected: ", counter if int(num)!= 0 and counter >= int(num): print 'Deadline Reached \nExiting' break helpModule.write_to_file(statuses, outfile) statuses = [] try: next_results = search_results['search_metadata']['next_results'] except KeyError, e: break kwargs = dict([ kv.split('=') for kv in unquote(next_results[1:]).split("&") ]) search_results = twitter_api.search.tweets(**kwargs) statuses += search_results['statuses'] with open(logfile, 'a') as fo: fo.write(str(tweets)+'\n') time.sleep(6)
def user_history(configs, user, output_dir): print 'Accessing User', user time.sleep(0.5) print 'Getting Configurations' time.sleep(0.5) ppath = configs['ppath'] lpath = configs['lpath'] root = configs['dpath'] if root == 'False': root = './' helpModule.make_outdir(root+output_dir) output_file = user+'.txt' print 'Configuring Files' time.sleep(0.5) outfile = root + output_dir + '/' + output_file fo = open(outfile, 'w') fo.close() pid = str(os.getpid()) logfile = lpath+pid+'.userlog' print 'Retrieving Twitter Profile' time.sleep(0.5) profile = profileModule.get_profile(ppath, lpath) profilepath = ppath+profile+'.profile' deets = profileModule.get_deets(profilepath) print 'Authorizing Twitter Profile' time.sleep(0.5) auth = tweepy.OAuthHandler(deets["consumer_key"], deets["consumer_secret"]) auth.set_access_token(deets["access_token"], deets["access_token_secret"]) twitter_api = tweepy.API(auth, parser=tweepy.parsers.JSONParser()) count = 200 status = [] print 'Extracting User Tweets' time.sleep(0.5) timeline_results = twitter_api.user_timeline(screen_name = user, count=count) with open(logfile, 'a') as fo: fo.write(profile+'\n') fo.write(user+'\n') status.extend(timeline_results) oldest = status[-1]['id'] - 1 while len(timeline_results) > 0: timeline_results = twitter_api.user_timeline(screen_name = user, count=count, max_id=oldest) status.extend(timeline_results) oldest = status[-1]['id'] - 1 print 'Tweets Collected: ', len(status) with open(logfile, 'a') as fo: fo.write(str(len(status))+'\n') time.sleep(6) print 'Writing To File' helpModule.write_to_file(status, outfile) os.remove(logfile) return