def testCanScrapeAndSave(self):
"""Tests if the Scraper can both scrape and save the results to a MongoDB database"""
# Before starting, if the test database exists, remove it
# TODO: standardize localhost string
test_client = MongoClient()
test_db = "testdb"
test_client.drop_database(test_db)
twauth = TwitterAuthentication.autodetect_twitter_auth()
scraper = Scraper(twitter_authentication=twauth)
results = scraper.scrape_and_save(terms={"fire"},
count=1,
dbname="testdb")
assert ('fire' in results.keys())
assert (len(results.keys()) == 1)
assert (isinstance(results['fire'], list))
assert (test_client[test_db].get_collection("fire").count() == 1)
test_client.drop_database(test_db)
def _disabled_testLotsOfTweets(self):
"""Tests if the Scraper can retrieve 1000 tweets for one term"""
twauth = TwitterAuthentication.autodetect_twitter_auth()
scraper = Scraper(twitter_authentication=twauth)
results = scraper.scrape(terms={"fire"}, count=1000)
assert ('fire' in results.keys())
assert (len(results.keys()) == 1)
assert (isinstance(results['fire'], list))
assert (len(results['fire']) == 1000)
def testCanScrapeAccount(self):
"""Tests that scraper can scrape one account."""
twauth = TwitterAuthentication.autodetect_twitter_auth()
scraper = Scraper(twitter_authentication=twauth)
results = scraper.scrape_accounts({"@RedCross"}, count=1)
assert ('@RedCross' in results)
assert (len(results.keys()) == 1)
assert (isinstance(results['@RedCross'], list))
assert (isinstance(get_status_text(results['@RedCross'][0]), str))
def __init__(self, twitter_authentication=None):
# type: (Scraper, TwitterAuthentication) -> None
# They did not pass in any authentication. Attempt to auto-detect it.
if not twitter_authentication:
self.twitter_authentication = TwitterAuthentication.autodetect_twitter_auth()
else: # They passed us a TwitterAuthentication object
self.twitter_authentication = twitter_authentication
# Tweepy API object. Can make API calls.
self.api = tweepy.API(self.twitter_authentication.oauth_handler,
wait_on_rate_limit=True, wait_on_rate_limit_notify=True)
# Default amount of tweets to retrieve.
self.default_count = 10
def testCanScrapeMethod(self):
"""Tests that the Scraper's `scrape` method works."""
twauth = TwitterAuthentication.autodetect_twitter_auth()
scraper = Scraper(twitter_authentication=twauth)
results = scraper.scrape(terms={"fire"}, accounts={"@RedCross"})
assert ('fire' in results.keys())
assert ('@RedCross' in results.keys())
assert (len(results.keys()) == 2)
assert (isinstance(results['fire'], list))
assert (isinstance(results['@RedCross'], list))
def testCanScrapeTerm(self):
"""Tests that scraper can scrape one term."""
twauth = TwitterAuthentication.autodetect_twitter_auth()
scraper = Scraper(twitter_authentication=twauth)
results = scraper.scrape_terms({"fire"}, count=1)
assert ('fire' in results)
assert (isinstance(results['fire'], list))
assert (len(results.keys()) == 1)
assert (isinstance(get_status_text(results['fire'][0]), str))
"""
Demonstrates the ability to search for fire regardless of location.
"""
# noinspection PyUnresolvedReferences
import __init__
import tweepy
from pprint import pprint
from twitter_fire_scraper.twitter import TwitterAuthentication
from twitter_fire_scraper.util import get_status_text
if __name__ == "__main__":
# Set up twitter auth.
twauth = TwitterAuthentication.autodetect_twitter_auth()
api = tweepy.API(twauth.oauth_handler)
print(
"Just searching for 'fire'... Probably not going to get us Chicago fire incidents."
)
for status in api.search("fire"):
print(get_status_text(status))
from flask import Flask, request, jsonify, abort from flask_pymongo import PyMongo # from flask.ext.pymongo import PyMongo from twitter_fire_scraper.scraper import Scraper from twitter_fire_scraper.twitter import TwitterAuthentication from twitter_fire_scraper.util import jsonify_status_dict, save_single_status_to_mongodb app = Flask(__name__, static_url_path="/static") app.config['MONGO_DBNAME'] = 'twitterfirescraperapi' # name of your cluster app.config['MONGO_URI'] = 'mongodb+srv://trungpham10:[email protected]/test?retryWrites=true' # mongodb username and password here mongo = PyMongo(app) scraper = Scraper(twitter_authentication=TwitterAuthentication.autodetect_twitter_auth()) @app.route('/add_new', methods=['GET']) def add(): user = mongo.db.users user.insert_one({'city': 'New York'}) return 'New York is added!' @app.route('/scrape_terms', methods=['GET']) def scrape_terms(): # user1 = mongo.db.users print('scrape_terms') print(request.args)
