from flask import jsonify
from scrapemeagain.config import Config
from scrapemeagain.dockerized.utils import app_factory, get_class_from_path
app = app_factory(__name__)
urlbroker_class = get_class_from_path(Config.URLBROKER_CLASS)
URLBROKER = urlbroker_class()
@app.route("/list-urls-range/")
def list_urls_range():
start, end = URLBROKER.get_urls_range()
return jsonify({"start": start, "end": end})
if __name__ == "__main__":
app.run(host="0.0.0.0", port=Config.URLBROKER_PORT)
get_class_from_path,
)
from scrapemeagain.pipeline import DockerizedPipeline
from scrapemeagain.utils.logger import setup_logging
from scrapemeagain.utils.useragents import get_user_agents
from examplescraper.databaser import DockerizedDatabaser
from examplescraper.scraper import DockerizedExampleScraper
# Update config, setup logging and useragents.
apply_scraper_config()
setup_logging(logger_name="examplescraper")
Config.USER_AGENTS = get_user_agents(__file__)
# Configure DockerizedTorIpChanger.
toripchanger_class = get_class_from_path(Config.TORIPCHANGER_CLASS)
tor_ip_changer = toripchanger_class(
local_http_proxy=Config.LOCAL_HTTP_PROXY,
tor_password=Config.TOR_PASSWORD,
tor_port=Config.TOR_PORT,
new_ip_max_attempts=Config.NEW_IP_MAX_ATTEMPTS,
)
# Prepare the scraping pipeline.
scraper = DockerizedExampleScraper()
databaser = DockerizedDatabaser(scraper.db_file)
pipeline = DockerizedPipeline(scraper, databaser, tor_ip_changer)
pipeline.prepare_pipeline()
if __name__ == "__main__":
# Change IP before starting.
from flask import request
from scrapemeagain.config import Config
from scrapemeagain.dockerized.utils import app_factory, get_class_from_path
app = app_factory(__name__)
datastore_class = get_class_from_path(Config.DATASTORE_DATABASER_CLASS)
DATASTORE = datastore_class()
@app.route("/insert-data/", methods=["POST"])
def insert_data():
DATASTORE.insert(request.json)
return "", 201
@app.route("/commit/")
def commit():
DATASTORE.commit()
return "", 204
if __name__ == "__main__":
app.run(host="0.0.0.0", port=Config.DATASTORE_PORT)
import flask
from toripchanger import TorIpChanger
from scrapemeagain.config import Config
from scrapemeagain.dockerized.utils import (
apply_scraper_config,
get_class_from_path,
)
from scrapemeagain.utils.logger import setup_logging
setup_logging(__name__)
apply_scraper_config()
datastore_class = get_class_from_path(Config.DATASTORE_DATABASER_CLASS)
DATASTORE = datastore_class()
urlbroker_class = get_class_from_path(Config.URLBROKER_CLASS)
URLBROKER = urlbroker_class()
IPSTORE = TorIpChanger(reuse_threshold=Config.IPSTORE_REUSE_THRESHOLD)
app = flask.Flask(__name__)
@app.route("/health/")
def healthcheck():
return ""
@app.route("/ip-is-safe/<ip>/")
def ip_is_safe(ip):
safe = IPSTORE._ip_is_safe(ip)
if safe: