def main():
logging.basicConfig(level=logging.INFO)
config = Config()
Register(to_bytes('http://%s:%d' % (config.get('bind_address', '127.0.0.1'), config.getint('http_port', 6800))),
config.get('register_path', '/scrapyd-cluster/worker'),
hosts=config.get('zookeeper_hosts', '127.0.0.1:2181'))
argv[1:1] = ['-n', '-y', join(dirname(scrapyd.__file__), 'txapp.py')]
run()
#Stage 2 Update (Python 3)
from future import standard_library
standard_library.install_aliases()
from builtins import object
import datetime, json
import urllib.request, urllib.parse, http.client
import os
from scrapy.utils.project import get_project_settings
settings = get_project_settings()
from scrapyd.config import Config
scrapyd_config = Config()
scrapyd_port = scrapyd_config.getint('http_port', 6800)
from dynamic_scraper.models import Scraper
class TaskUtils(object):
conf = {
"MAX_SPIDER_RUNS_PER_TASK": 10,
"MAX_CHECKER_RUNS_PER_TASK": 25,
}
def _run_spider(self, **kwargs):
scrapyd_host = os.environ.get('SCRAPYD_HOST', 'localhost')
scrapyd_port = os.environ.get('SCRAPYD_PORT', '6800')
param_dict = {
'project': 'default',
'spider': kwargs['spider'],
'id': kwargs['id'],
'run_type': kwargs['run_type'],
'do_action': kwargs['do_action']
#Stage 2 Update (Python 3)
from future import standard_library
standard_library.install_aliases()
from builtins import object
import datetime, json
import urllib.request, urllib.parse, http.client
from scrapy.utils.project import get_project_settings
settings = get_project_settings()
from scrapyd.config import Config
scrapyd_config = Config()
scrapyd_port = scrapyd_config.getint('http_port', 6800)
from dynamic_scraper.models import Scraper
class TaskUtils(object):
conf = {
"MAX_SPIDER_RUNS_PER_TASK": 10,
"MAX_CHECKER_RUNS_PER_TASK": 25,
}
def _run_spider(self, **kwargs):
param_dict = {
'project': 'default',
'spider': kwargs['spider'],
'id': kwargs['id'],
'run_type': kwargs['run_type'],
'do_action': kwargs['do_action']
}
params = urllib.parse.urlencode(param_dict)
headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}