def main():
# get arguments
args = get_args()
# read required json files
bounding, db_auth, api_auths = read_jsons()
# db url
url = "http://" + db_auth["user"] + ":" + db_auth["pwd"] \
+ "@" + db_auth["ip"] + ":" + db_auth["port"] + "/"
# initialise db and twitter api
storage = TweetStore(args.db_name, url)
apis = []
for api_auth in api_auths:
api = TwitterAPI(api_auth["API_KEY"], api_auth["API_SECRET"],
api_auth["ACCESS_TOKEN"],
api_auth["ACCESS_TOKEN_SECRET"])
apis.append(api)
t1 = threading.Thread(target=twitter_streaming,
args=(apis[0], storage, bounding, args.region))
t2 = threading.Thread(target=twitter_user_timeline,
args=(apis[1:], storage))
# start streaming and getting timelines
t1.start()
t2.start()
def __init__(self,
consumer_key,
consumer_secret,
data_dir,
access_token_key=None,
access_token_secret=None):
"""
Construct a new Miner for retrieving data from Twitter.
if access_token_key or access_token_secret are None, use app authentication with
twitter's API
:param consumer_key:
:param consumer_secret:
:param data_dir: main directory to store the data
:param access_token_key:
:param access_token_secret:
"""
if access_token_key is None or access_token_secret is None:
self.api = TwitterAPI(consumer_key,
consumer_secret,
auth_type='oAuth2')
else:
self.api = TwitterAPI(consumer_key, consumer_secret,
access_token_key, access_token_secret)
self.writer = DW(data_dir)
self.logger = logging.getLogger()
self.queues = {type: Queue() for type in JOBS_TYPES}
# python queues are thread safe and don't require locks for multi-producers/consumers
# create thread for each different job type
self.threads = [
Thread(target=Miner._run_consumer,
args=(self, 'followers_ids', Miner._mine_followers_ids)),
Thread(target=Miner._run_consumer,
args=(self, 'friends_ids', Miner._mine_friends_ids)),
Thread(target=Miner._run_consumer,
args=(self, 'tweets', Miner._mine_tweets)),
Thread(target=Miner._run_consumer,
args=(self, 'likes', Miner._mine_likes)),
Thread(target=Miner._run_consumer,
args=(self, 'user_details', Miner._mine_user_details)),
Thread(target=Miner._run_consumer,
args=(self, 'neighbors', Miner._mine_neighbors)),
Thread(target=Miner._run_consumer,
args=(self, 'listen', Miner._listen))
]
def main():
# get arguments
args = get_args()
# read required json files
db_auth, api_auth = read_jsons()
# db url
url = "http://" + db_auth["user"] + ":" + db_auth["pwd"] \
+ "@" + db_auth["ip"] + ":" + db_auth["port"] + "/"
# initialise db and twitter api
storage = TweetStore(args.db_name, url)
api = TwitterAPI(api_auth["API_KEY"], api_auth["API_SECRET"],
api_auth["ACCESS_TOKEN"], api_auth["ACCESS_TOKEN_SECRET"])
twitter_search(api, storage, args.query)
import random
import re
from time import sleep
from TwitterAPI.TwitterAPI import TwitterAPI
from colors import colors_by_name
from shm import led
__author__ = 'zander'
api = TwitterAPI("3izkk25JAjqyetdxz7UzwN9tr",
"W01dNPAmv2FIUmvsqPTkSjIg5364dxT4cbvdH8SltNbFUXzifn",
"61233167-E5VTAoVXV02RwbeMrpRdlJhZs6jLGS32JYHxOs97t",
"cjdI8OvMHVpTXzFAnmEw1TpwC2i1x2nEdZdh5znqfDyc5")
r = api.request("user")
def set_led_color(red, green, blue):
led.port_red.set(red)
led.port_green.set(green)
led.port_blue.set(blue)
led.starboard_red.set(red)
led.starboard_green.set(green)
led.starboard_blue.set(blue)
set_led_color(0, 100, 255)
# ===================================================================
from TweetStore import TweetStore
from TwitterAPI.TwitterAPI import TwitterAPI
city_bound_googleV3 = [-84.5518189, 33.648079, -84.289389, 33.8876179]
chunks_bound = [[-84.5518189, 33.648079, -84.42060395, 33.76784845], \
[-84.5518189, 33.76784845, -84.42060395, 33.8876179], \
[-84.42060395, 33.648079, -84.289389, 33.76784845],\
[-84.42060395, 33.76784845, -84.289389, 33.8876179]]
COUCH_DATABASE = 'test_db'
TWITTER_ENDPOINT = 'statuses/filter'
TWITTER_PARAMS = {'locations': chunks_bound[0]}
API_KEY = "lSjoTqZ4ofmtCr0uh7aJZRQcp"
API_SECRET = "qfkI0RjyOetHNDE6EJhojNlqRf4B7lbZj2rTQBmTZYHT9sRjlc"
ACCESS_TOKEN = "3186003008-eAzy3mSzxHRYuWzji65Xi0JrjTFqJTO81MU2cKK"
ACCESS_TOKEN_SECRET = "Rvvo931v0hbMyRX8sg8QG51cVlY8LQuij8zXuA9aP1hIh"
storage = TweetStore(COUCH_DATABASE)
api = TwitterAPI(API_KEY, API_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
for item in api.request(TWITTER_ENDPOINT, TWITTER_PARAMS):
if 'text' in item:
print('%s -- %s\n' % (item['user']['screen_name'], item['text']))
storage.save_tweet(item)
elif 'message' in item:
print('ERROR %s: %s\n' % (item['code'], item['message']))
def __init__(self, database_manager, stop_words, user_name, zone_index):
self.dm = database_manager
self.filter = RegexTokenizer() | LowercaseFilter() | StopFilter() | StopFilter(stop_words)
self.zone_index = zone_index
self.api = TwitterAPI(app_auth[user_name].ckey, app_auth[user_name].csec,
app_auth[user_name].atoken, app_auth[user_name].asec)
view.sync(self.db)
def save_tweet(self,tw):
tw['_id']=tw['id_str']
self.db.save(tw)
def count_tweets(self):
for doc in self.db.view('twitter/count_tweets'):
return doc.value
def get_tweets(self):
return self.db.view('twitter/get_tweets')
from TweetStore import TweetStore
from TwitterAPI.TwitterAPI import TwitterAPI
api_key=''
api_secret=''
access_token=''
access_secret=''
storage=TweetStore('test_db')
api=TwitterAPI(api_key,api_secret,access_token,access_secret)
for item in api.request('search/tweets',{'q':'taarangana'}):
if 'text' in item:
print('%s -- %s\n' % (item['user']['screen_name'],item['text']))
storage.save_tweet(item)
elif 'message' in item:
print('ERROR %s: %s\n' % (item['code'],item['message']))
def get_app_api():
config = get_config()
return TwitterAPI(config['consumer_key'],
config['consumer_secret'],
auth_type='oAuth2')
def get_user_api():
config = get_config()
return TwitterAPI(config['consumer_key'], config['consumer_secret'],
config['access_token_key'],
config['access_token_secret'])
from common import *
streamDatabase = StreamDatabase('tweets')
Current = []
with open("../../Common/current.txt", 'r') as current:
Current = current.readlines()
APIKey = Current[1].strip()
APISecretKey = Current[2].strip()
AccessToken = Current[3].strip()
AccessTokenSecret = Current[4].strip()
api = TwitterAPI(APIKey, APISecretKey, AccessToken, AccessTokenSecret)
while True:
try:
for tweet in api.request('statuses/filter', {'track': '', 'locations': melbourneBoundingBox}):
if 'text' in tweet:
docid = tweet['id_str']
if docid in streamDatabase.database:
with open('stream_log','a') as f:
f.write("["+datetime.datetime.now().__str__()+"]"+'\n')
f.write('The tweet is already present\n')
