def _disabled_testLotsOfTweets(self):
"""Tests if the Scraper can retrieve 1000 tweets for one term"""
twauth = TwitterAuthentication.autodetect_twitter_auth()
scraper = Scraper(twitter_authentication=twauth)
results = scraper.scrape(terms={"fire"}, count=1000)
self.assertIn('fire', results.keys())
self.assertEqual(len(results.keys()), 1)
self.assertIsInstance(results['fire'], list)
self.assertEqual(len(results['fire']), 1000)
def testCanScrapeAccount(self):
"""Tests that scraper can scrape one account."""
twauth = TwitterAuthentication.autodetect_twitter_auth()
scraper = Scraper(twitter_authentication=twauth)
results = scraper.scrape_accounts({"@RedCross"}, count=1)
self.assertIn('@RedCross', results)
self.assertEqual(len(results.keys()), 1)
self.assertIsInstance(results['@RedCross'], list)
self.assertIsInstance(results['@RedCross'][0].get_text(), str)
def testCanScrapeTerm(self):
"""Tests that scraper can scrape one term."""
twauth = TwitterAuthentication.autodetect_twitter_auth()
scraper = Scraper(twitter_authentication=twauth)
results = scraper.scrape_terms({"fire"}, count=1)
self.assertIn('fire', results)
self.assertIsInstance(results['fire'], list)
self.assertEqual(len(results.keys()), 1)
self.assertIsInstance(results['fire'][0].get_text(), str)
def testCanScrapeMethod(self):
"""Tests that the Scraper's `scrape` method works."""
twauth = TwitterAuthentication.autodetect_twitter_auth()
scraper = Scraper(twitter_authentication=twauth)
results = scraper.scrape(terms={"fire"}, accounts={"@RedCross"})
self.assertIn('fire', results.keys())
self.assertIn('@RedCross', results.keys())
self.assertEqual(len(results.keys()), 2)
self.assertIsInstance(results['fire'], list)
self.assertIsInstance(results['@RedCross'], list)
def __init__(self, twitter_authentication=None):
# type: (Scraper, TwitterAuthentication) -> None
# They did not pass in any authentication. Attempt to auto-detect it.
if not twitter_authentication:
self.twitter_authentication = TwitterAuthentication.autodetect_twitter_auth(
)
else: # They passed us a TwitterAuthentication object
self.twitter_authentication = twitter_authentication
# Tweepy API object. Can make API calls.
self.api = tweepy.API(self.twitter_authentication.oauth_handler,
wait_on_rate_limit=True,
wait_on_rate_limit_notify=True)
# Default amount of tweets to retrieve.
self.default_count = 10
def test_invalid(self):
# This is one way to ideally initialize a Scraper object.
scraper = Scraper(twitter_authentication=TwitterAuthentication(
consumer_key="i4m4c0ns4m3r",
consumer_secret="itsasecret",
access_token="chuckecheesetokens?",
access_token_secret="ticketmuncherdoesn'twork"))
terms = {"#pizza", "pizza", 'cat', '#cat'}
failure = False
try:
results = scraper.scrape_terms(terms=terms, count=3)
except TweepError as e:
print("Failed successfully with bogus credentials.")
failure = True
if not failure:
raise Exception("Credentials should fail to validate.")
from flask import Flask, render_template
from flask_app.forms import ScrapeSingleTermTestForm
from config import FlaskConfig
from scraper import Scraper
from twitter import TwitterAuthentication
current_folder = os.path.abspath(os.path.dirname(__file__))
static_folder = os.path.join(current_folder, 'static')
template_folder = os.path.join(current_folder, 'templates')
app = Flask(__name__, static_url_path='/static', template_folder=template_folder, static_folder=static_folder)
app.debug = FlaskConfig.DEBUG
scraper = Scraper(twitter_authentication=TwitterAuthentication.autodetect_twitter_auth())
@app.route('/')
def index():
return render_template('index.html')
@app.route("/success")
def success():
return "Whatever you just tried worked. Congrats :)"
@app.route('/scrape_term', methods=("GET", "POST"))
def scrape_term():
form = ScrapeSingleTermTestForm()