示例#1
0
def user_setup(output_dir, logfile):
        print 'Getting Configurations'
        logging.info('Getting Configurations')

        if output_dir: helpModule.make_outdir(root+output_dir)

  	print 'Retrieving Twitter Profile'
        logging.info('Retrieving Twitter Profile')
	global __profile
        __profile = profileModule.get_profile(ppath, lpath)
	logging.debug('Using profile '+__profile.upper())
        profilepath = ppath+__profile+'.profile'
        deets = profileModule.get_deets(profilepath)

	with open(logfile, 'a') as fo:
                fo.write(__profile+'\n')

        print 'Authorizing Twitter Profile'
        logging.info('Authorizing Twitter Profile')
        auth = tweepy.OAuthHandler(deets["consumer_key"], deets["consumer_secret"])
        auth.set_access_token(deets["access_token"], deets["access_token_secret"])
        global __twitterapi__
	__twitterapi__ = tweepy.API(auth, parser=tweepy.parsers.JSONParser())
示例#2
0
def user_setup(output_dir, logfile):
    print "Getting Configurations"
    logging.info("Getting Configurations")

    if output_dir:
        helpModule.make_outdir(root + output_dir)

    print "Retrieving Twitter Profile"
    logging.info("Retrieving Twitter Profile")
    global __profile
    __profile = profileModule.get_profile(ppath, lpath)
    logging.debug("Using profile " + __profile.upper())
    profilepath = ppath + __profile + ".profile"
    deets = profileModule.get_deets(profilepath)

    with open(logfile, "a") as fo:
        fo.write(__profile + "\n")

    print "Authorizing Twitter Profile"
    logging.info("Authorizing Twitter Profile")
    auth = tweepy.OAuthHandler(deets["consumer_key"], deets["consumer_secret"])
    auth.set_access_token(deets["access_token"], deets["access_token_secret"])
    global __twitterapi__
    __twitterapi__ = tweepy.API(auth, parser=tweepy.parsers.JSONParser())
示例#3
0
lpath = configs['lpath']
root = configs['dpath']
if root == 'False':
	root = './'

outfile = root + output_dir + '/' + output_file

fo = open(outfile, 'w')
fo.close()

parentid = str(os.getpid())
logfile = lpath+parentid+'.streamlog'

helpModule.make_outdir(root+output_dir)

profile = profileModule.get_profile(ppath, lpath)
profilepath = ppath+profile+'.profile'
deets = profileModule.get_deets(profilepath)

#-------------------------------------------------------------------------------------------------------------

class StdOutListener(StreamListener):

    def on_data(self, data):
        if  'in_reply_to_status' in data:
		with open(outfile,'a') as tf:
            		tf.write(data)
        return True

    def on_error(self, status):
        pass
示例#4
0
def searching(configs, query, write_to_file, output_file, num):
    print 'Getting Configurations'
    logging.info('Getting configurations')
    ppath = configs['ppath']
    lpath = configs['lpath']
    root = configs['dpath']
    if root == 'False':
        root = './'

    print 'Configuring Files'
    logging.info('Configuring files')
    if write_to_file:
        outfile = root + output_file
        fo = open(outfile, 'w')
        fo.close()
    else:
        outfile = output_file

    print 'Encoding Query'
    logging.info('Encoding Query')
    enc_query = quote_plus(query.encode('UTF-8'), safe=':/')

    pid = str(os.getpid())
    logfile = lpath + pid + '.searchlog'

    print 'Retrieving Twitter Profile'
    logging.info('Retrieving Twitter Profile')
    profile = profileModule.get_profile(ppath, lpath)
    profilepath = ppath + profile + '.profile'
    deets = profileModule.get_deets(profilepath)

    print 'Authorizing Twitter Profile'
    logging.info('Authorizing Twitter profile')
    auth = twitter.oauth.OAuth(deets["access_token"],
                               deets["access_token_secret"],
                               deets["consumer_key"], deets["consumer_secret"])
    twitter_api = twitter.Twitter(auth=auth)

    count = 100

    print 'Starting Search'
    logging.info('Starting search')
    search_results = twitter_api.search.tweets(q=enc_query, count=count)

    with open(logfile, 'a') as fo:
        fo.write(profile + '\n')
        fo.write(query + '\n')

    statuses = search_results['statuses']
    counter = 0
    tweets = 0

    while True:
        counter += len(statuses)
        print "Tweets Collected: ", counter

        if int(num) != 0 and counter >= int(num):
            print 'Deadline Reached \nExiting'
            logging.info('Deadline reached. No more Tweets collected.')
            break

        helpModule.write_to_file(statuses, outfile)
        statuses = []

        try:
            next_results = search_results['search_metadata']['next_results']
        except KeyError, e:
            break

        kwargs = dict(
            [kv.split('=') for kv in unquote(next_results[1:]).split("&")])

        search_results = twitter_api.search.tweets(**kwargs)
        statuses += search_results['statuses']

        with open(logfile, 'a') as fo:
            fo.write(str(tweets) + '\n')
示例#5
0
lpath = configs['lpath']
root = configs['dpath']
if root == 'False':
    root = './'

outfile = root + output_dir + '/' + output_file

fo = open(outfile, 'w')
fo.close()

parentid = str(os.getpid())
logfile = lpath + parentid + '.streamlog'

helpModule.make_outdir(root + output_dir)

profile = profileModule.get_profile(ppath, lpath)
profilepath = ppath + profile + '.profile'
deets = profileModule.get_deets(profilepath)

#-------------------------------------------------------------------------------------------------------------


class StdOutListener(StreamListener):
    def on_data(self, data):
        if 'in_reply_to_status' in data:
            with open(outfile, 'a') as tf:
                tf.write(data)
        return True

    def on_error(self, status):
        pass
示例#6
0
def searching(configs, query, output_file, output_dir, num):
	print 'Getting Configurations' 
	time.sleep(0.5)
	ppath = configs['ppath']
	lpath = configs['lpath']
	root = configs['dpath']
	if root == 'False':
        	root = './'
	
	helpModule.make_outdir(root+output_dir)
	
	print 'Configuring Files'
	time.sleep(0.5)
	outfile = root + output_dir + '/' + output_file

	fo = open(outfile, 'w')
	fo.close()

	print 'Encoding Query'
	time.sleep(0.5)
	enc_query = quote_plus(query.encode('UTF-8'), safe=':/')

        pid = str(os.getpid())
        logfile = lpath+pid+'.searchlog'
	
	print 'Retrieving Twitter Profile' 
        time.sleep(0.5)
	profile = profileModule.get_profile(ppath, lpath)
        profilepath = ppath+profile+'.profile'
        deets = profileModule.get_deets(profilepath)

	print 'Authorizing Twitter Profile'
	time.sleep(0.5)
	auth = twitter.oauth.OAuth(deets["access_token"], deets["access_token_secret"], deets["consumer_key"], deets["consumer_secret"])
	twitter_api = twitter.Twitter(auth=auth)

	count = 100

	print 'Starting Search'
	time.sleep(0.5)
	search_results = twitter_api.search.tweets(q=enc_query, count=count)

	with open(logfile, 'a') as fo:
		fo.write(profile+'\n')
		fo.write(query+'\n')

	statuses = search_results['statuses']
	counter = 0
	tweets = 0

	while True:
		counter+=len(statuses)
		print "Tweets Collected: ", counter
 
		if int(num)!= 0 and counter >= int(num):
			print 'Deadline Reached \nExiting'
			break		

		helpModule.write_to_file(statuses, outfile)
		statuses = []

		try:
			next_results = search_results['search_metadata']['next_results']
		except KeyError, e:
			break

		kwargs = dict([ kv.split('=') for kv in unquote(next_results[1:]).split("&") ])

		search_results = twitter_api.search.tweets(**kwargs)
		statuses += search_results['statuses']

		with open(logfile, 'a') as fo:
			fo.write(str(tweets)+'\n')

		time.sleep(6)
示例#7
0
def user_history(configs, user, output_dir):
	print 'Accessing User', user
	time.sleep(0.5)
	print 'Getting Configurations' 
	time.sleep(0.5)
	ppath = configs['ppath']
	lpath = configs['lpath']
	root = configs['dpath']
	if root == 'False':
        	root = './'

	helpModule.make_outdir(root+output_dir)
	
	output_file = user+'.txt'

	print 'Configuring Files'
	time.sleep(0.5)
	outfile = root + output_dir + '/' + output_file

	fo = open(outfile, 'w')
	fo.close()

        pid = str(os.getpid())
        logfile = lpath+pid+'.userlog'
	
	print 'Retrieving Twitter Profile' 
        time.sleep(0.5)
	profile = profileModule.get_profile(ppath, lpath)
        profilepath = ppath+profile+'.profile'
        deets = profileModule.get_deets(profilepath)

	print 'Authorizing Twitter Profile'
	time.sleep(0.5)
	auth = tweepy.OAuthHandler(deets["consumer_key"], deets["consumer_secret"])
    	auth.set_access_token(deets["access_token"], deets["access_token_secret"])
    	twitter_api = tweepy.API(auth, parser=tweepy.parsers.JSONParser())
	
	count = 200
	status = []

	print 'Extracting User Tweets'
	time.sleep(0.5)
	timeline_results = twitter_api.user_timeline(screen_name = user, count=count)

	with open(logfile, 'a') as fo:
		fo.write(profile+'\n')
		fo.write(user+'\n')

   	status.extend(timeline_results)
	oldest = status[-1]['id'] - 1

	while len(timeline_results) > 0:
        	timeline_results = twitter_api.user_timeline(screen_name = user, count=count, max_id=oldest)
        	status.extend(timeline_results)
        	oldest = status[-1]['id'] - 1

        	print 'Tweets Collected: ', len(status)

		with open(logfile, 'a') as fo:
                        fo.write(str(len(status))+'\n')

		time.sleep(6)

	print 'Writing To File'
	helpModule.write_to_file(status, outfile)

	os.remove(logfile)
	
	return