def collect_user_stream():
mongo_host = pipeline.config_mongodb()
collection_name, time_limit, user_ids = config_user_stream()
logger = pipeline.setup_logger('user_stream')
auth = pipeline.get_auth()
listener = sample_stream_listener.SampleStreamListener(logger, mongo_host, collection_name, time_limit, auth)
keywords_stream = Stream(auth, listener)
return keywords_stream, user_ids
def collect_streaming_sample():
config = pipeline.load_config()
mongo_host = pipeline.config_mongodb()
collection_name, time_limit = config_sample_stream(config)
logger = pipeline.setup_logger('streaming.log')
auth = pipeline.get_auth()
listener = sample_stream_listener.SampleStreamListener(logger, mongo_host, collection_name, time_limit, auth)
sample_stream = Stream(auth, listener)
return sample_stream
def collect_location_stream():
mongo_host = pipeline.config_mongodb()
collection_name, time_limit, location = config_location_stream()
logger = pipeline.setup_logger('location_stream')
auth = pipeline.get_auth()
listener = sample_stream_listener.SampleStreamListener(
logger, mongo_host, collection_name, time_limit, auth)
location_stream = Stream(auth, listener)
return location_stream, location
def get_api():
auth = pipeline.get_auth()
auth_api = tweepy.API(auth,
wait_on_rate_limit=True,
wait_on_rate_limit_notify=True)
if not auth_api:
print("Problem connecting to API")
sys.exit(-1)
return auth_api