def main(): # get arguments args = get_args() # read required json files bounding, db_auth, api_auths = read_jsons() # db url url = "http://" + db_auth["user"] + ":" + db_auth["pwd"] \ + "@" + db_auth["ip"] + ":" + db_auth["port"] + "/" # initialise db and twitter api storage = TweetStore(args.db_name, url) apis = [] for api_auth in api_auths: api = TwitterAPI(api_auth["API_KEY"], api_auth["API_SECRET"], api_auth["ACCESS_TOKEN"], api_auth["ACCESS_TOKEN_SECRET"]) apis.append(api) t1 = threading.Thread(target=twitter_streaming, args=(apis[0], storage, bounding, args.region)) t2 = threading.Thread(target=twitter_user_timeline, args=(apis[1:], storage)) # start streaming and getting timelines t1.start() t2.start()
def __init__(self, consumer_key, consumer_secret, data_dir, access_token_key=None, access_token_secret=None): """ Construct a new Miner for retrieving data from Twitter. if access_token_key or access_token_secret are None, use app authentication with twitter's API :param consumer_key: :param consumer_secret: :param data_dir: main directory to store the data :param access_token_key: :param access_token_secret: """ if access_token_key is None or access_token_secret is None: self.api = TwitterAPI(consumer_key, consumer_secret, auth_type='oAuth2') else: self.api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret) self.writer = DW(data_dir) self.logger = logging.getLogger() self.queues = {type: Queue() for type in JOBS_TYPES} # python queues are thread safe and don't require locks for multi-producers/consumers # create thread for each different job type self.threads = [ Thread(target=Miner._run_consumer, args=(self, 'followers_ids', Miner._mine_followers_ids)), Thread(target=Miner._run_consumer, args=(self, 'friends_ids', Miner._mine_friends_ids)), Thread(target=Miner._run_consumer, args=(self, 'tweets', Miner._mine_tweets)), Thread(target=Miner._run_consumer, args=(self, 'likes', Miner._mine_likes)), Thread(target=Miner._run_consumer, args=(self, 'user_details', Miner._mine_user_details)), Thread(target=Miner._run_consumer, args=(self, 'neighbors', Miner._mine_neighbors)), Thread(target=Miner._run_consumer, args=(self, 'listen', Miner._listen)) ]
def main(): # get arguments args = get_args() # read required json files db_auth, api_auth = read_jsons() # db url url = "http://" + db_auth["user"] + ":" + db_auth["pwd"] \ + "@" + db_auth["ip"] + ":" + db_auth["port"] + "/" # initialise db and twitter api storage = TweetStore(args.db_name, url) api = TwitterAPI(api_auth["API_KEY"], api_auth["API_SECRET"], api_auth["ACCESS_TOKEN"], api_auth["ACCESS_TOKEN_SECRET"]) twitter_search(api, storage, args.query)
import random import re from time import sleep from TwitterAPI.TwitterAPI import TwitterAPI from colors import colors_by_name from shm import led __author__ = 'zander' api = TwitterAPI("3izkk25JAjqyetdxz7UzwN9tr", "W01dNPAmv2FIUmvsqPTkSjIg5364dxT4cbvdH8SltNbFUXzifn", "61233167-E5VTAoVXV02RwbeMrpRdlJhZs6jLGS32JYHxOs97t", "cjdI8OvMHVpTXzFAnmEw1TpwC2i1x2nEdZdh5znqfDyc5") r = api.request("user") def set_led_color(red, green, blue): led.port_red.set(red) led.port_green.set(green) led.port_blue.set(blue) led.starboard_red.set(red) led.starboard_green.set(green) led.starboard_blue.set(blue) set_led_color(0, 100, 255)
# =================================================================== from TweetStore import TweetStore from TwitterAPI.TwitterAPI import TwitterAPI city_bound_googleV3 = [-84.5518189, 33.648079, -84.289389, 33.8876179] chunks_bound = [[-84.5518189, 33.648079, -84.42060395, 33.76784845], \ [-84.5518189, 33.76784845, -84.42060395, 33.8876179], \ [-84.42060395, 33.648079, -84.289389, 33.76784845],\ [-84.42060395, 33.76784845, -84.289389, 33.8876179]] COUCH_DATABASE = 'test_db' TWITTER_ENDPOINT = 'statuses/filter' TWITTER_PARAMS = {'locations': chunks_bound[0]} API_KEY = "lSjoTqZ4ofmtCr0uh7aJZRQcp" API_SECRET = "qfkI0RjyOetHNDE6EJhojNlqRf4B7lbZj2rTQBmTZYHT9sRjlc" ACCESS_TOKEN = "3186003008-eAzy3mSzxHRYuWzji65Xi0JrjTFqJTO81MU2cKK" ACCESS_TOKEN_SECRET = "Rvvo931v0hbMyRX8sg8QG51cVlY8LQuij8zXuA9aP1hIh" storage = TweetStore(COUCH_DATABASE) api = TwitterAPI(API_KEY, API_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET) for item in api.request(TWITTER_ENDPOINT, TWITTER_PARAMS): if 'text' in item: print('%s -- %s\n' % (item['user']['screen_name'], item['text'])) storage.save_tweet(item) elif 'message' in item: print('ERROR %s: %s\n' % (item['code'], item['message']))
def __init__(self, database_manager, stop_words, user_name, zone_index): self.dm = database_manager self.filter = RegexTokenizer() | LowercaseFilter() | StopFilter() | StopFilter(stop_words) self.zone_index = zone_index self.api = TwitterAPI(app_auth[user_name].ckey, app_auth[user_name].csec, app_auth[user_name].atoken, app_auth[user_name].asec)
view.sync(self.db) def save_tweet(self,tw): tw['_id']=tw['id_str'] self.db.save(tw) def count_tweets(self): for doc in self.db.view('twitter/count_tweets'): return doc.value def get_tweets(self): return self.db.view('twitter/get_tweets') from TweetStore import TweetStore from TwitterAPI.TwitterAPI import TwitterAPI api_key='' api_secret='' access_token='' access_secret='' storage=TweetStore('test_db') api=TwitterAPI(api_key,api_secret,access_token,access_secret) for item in api.request('search/tweets',{'q':'taarangana'}): if 'text' in item: print('%s -- %s\n' % (item['user']['screen_name'],item['text'])) storage.save_tweet(item) elif 'message' in item: print('ERROR %s: %s\n' % (item['code'],item['message']))
def get_app_api(): config = get_config() return TwitterAPI(config['consumer_key'], config['consumer_secret'], auth_type='oAuth2')
def get_user_api(): config = get_config() return TwitterAPI(config['consumer_key'], config['consumer_secret'], config['access_token_key'], config['access_token_secret'])
from common import * streamDatabase = StreamDatabase('tweets') Current = [] with open("../../Common/current.txt", 'r') as current: Current = current.readlines() APIKey = Current[1].strip() APISecretKey = Current[2].strip() AccessToken = Current[3].strip() AccessTokenSecret = Current[4].strip() api = TwitterAPI(APIKey, APISecretKey, AccessToken, AccessTokenSecret) while True: try: for tweet in api.request('statuses/filter', {'track': '', 'locations': melbourneBoundingBox}): if 'text' in tweet: docid = tweet['id_str'] if docid in streamDatabase.database: with open('stream_log','a') as f: f.write("["+datetime.datetime.now().__str__()+"]"+'\n') f.write('The tweet is already present\n')