def main(): logging.basicConfig(level=logging.INFO) config = Config() Register(to_bytes('http://%s:%d' % (config.get('bind_address', '127.0.0.1'), config.getint('http_port', 6800))), config.get('register_path', '/scrapyd-cluster/worker'), hosts=config.get('zookeeper_hosts', '127.0.0.1:2181')) argv[1:1] = ['-n', '-y', join(dirname(scrapyd.__file__), 'txapp.py')] run()
#Stage 2 Update (Python 3) from future import standard_library standard_library.install_aliases() from builtins import object import datetime, json import urllib.request, urllib.parse, http.client import os from scrapy.utils.project import get_project_settings settings = get_project_settings() from scrapyd.config import Config scrapyd_config = Config() scrapyd_port = scrapyd_config.getint('http_port', 6800) from dynamic_scraper.models import Scraper class TaskUtils(object): conf = { "MAX_SPIDER_RUNS_PER_TASK": 10, "MAX_CHECKER_RUNS_PER_TASK": 25, } def _run_spider(self, **kwargs): scrapyd_host = os.environ.get('SCRAPYD_HOST', 'localhost') scrapyd_port = os.environ.get('SCRAPYD_PORT', '6800') param_dict = { 'project': 'default', 'spider': kwargs['spider'], 'id': kwargs['id'], 'run_type': kwargs['run_type'], 'do_action': kwargs['do_action']
#Stage 2 Update (Python 3) from future import standard_library standard_library.install_aliases() from builtins import object import datetime, json import urllib.request, urllib.parse, http.client from scrapy.utils.project import get_project_settings settings = get_project_settings() from scrapyd.config import Config scrapyd_config = Config() scrapyd_port = scrapyd_config.getint('http_port', 6800) from dynamic_scraper.models import Scraper class TaskUtils(object): conf = { "MAX_SPIDER_RUNS_PER_TASK": 10, "MAX_CHECKER_RUNS_PER_TASK": 25, } def _run_spider(self, **kwargs): param_dict = { 'project': 'default', 'spider': kwargs['spider'], 'id': kwargs['id'], 'run_type': kwargs['run_type'], 'do_action': kwargs['do_action'] } params = urllib.parse.urlencode(param_dict) headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}