def __init__(self, thread_number: int, try_limit: int = 0) -> None:
super().__init__(thread_number=thread_number,
network_try_limit=try_limit,
priority_mode='const')
# Logger
self.log = Log(DSpiderCommon.logger)
self.logger = DSpiderCommon.logger
# Re module init
Ree.init()
# Work data
self.single_task_mode = False
self.tasks_store = {}
self.result = []
self.cookie_jar = {}
# Info
self.info = StatCounter()
self.info.add_task(StatCounter.TASK_FACTORY)
# Common vars
self.domain = UrlGenerator.get_host_from_url(
Config.get_seq('SITE_URL')[0])
self.err_limit = try_limit
# Cache
cache_enabled = Config.get('APP_CACHE_ENABLED', '')
cache_db_host = Config.get('APP_CACHE_DB_HOST', '')
if cache_enabled and cache_db_host:
cache_db_name = Config.get('APP_CACHE_DB_NAME', 'pythonparsers')
cache_db_type = Config.get('APP_CACHE_DB_TYPE', 'mysql')
cache_db_port = int(Config.get('APP_CACHE_DB_PORT', '3306'))
cache_db_user = Config.get('APP_CACHE_DB_USER', 'root')
cache_db_pass = Config.get('APP_CACHE_DB_PASS', '')
if cache_db_user and cache_db_pass:
self.setup_cache(backend=cache_db_type,
database=cache_db_name,
host=cache_db_host,
port=cache_db_port,
user=cache_db_user,
password=cache_db_pass)
else:
self.setup_cache(backend=cache_db_type,
database=cache_db_name,
host=cache_db_host,
port=cache_db_port)
self.logger.info('!!! CACHE MODE ENABLED !!!')
# Debug mode (only 1 iteration of each task)
if Config.get('APP_SINGLE_TASK', ''):
self.logger.info('!!! SINGLE MODE ENABLED !!!')
self.single_task_mode = True
self.logger.info('Init parser ok...')
def __init__(self, thread_number, try_limit=0):
super().__init__(thread_number=thread_number,
network_try_limit=try_limit,
priority_mode='const')
DSpider._check_body_errors = check_body_errors
DSpider._process_error = process_error
DSpider._common_init = common_init
self._common_init(try_limit)
Ree.init()
Ree.is_page_number(Config.get('SITE_PAGE_PARAM'))
def __init__(self, thread_number, try_limit=0):
super().__init__(thread_number=thread_number, network_try_limit=try_limit, priority_mode='const')
DSpider._check_body_errors = check_body_errors
DSpider._process_error = process_error
DSpider._common_init = common_init
self._common_init(try_limit)
Ree.init()
Ree.is_page_number(Config.get('SITE_PAGE_PARAM'))
self.const_zero_stock = Config.get('APP_STOCK_ZERO')
self.const_price_sep = Config.get('APP_PRICE_SEP')
self.const_enc = Config.get('APP_OUTPUT_ENC')
def __init__(self, thread_number, try_limit=0):
super().__init__(thread_number=thread_number,
network_try_limit=try_limit,
priority_mode='const')
DSpider._check_body_errors = check_body_errors
DSpider._process_error = process_error
DSpider._common_init = common_init
self._common_init(try_limit)
Ree.init()
Ree.is_page_number(Config.get('SITE_PAGE_PARAM'))
self.const_price_on_request = Config.get('APP_PRICE_ON_REQUEST')
self.const_stock_zero = Config.get('APP_STOCK_ZERO')
self.const_default_place = 'Полежаевская'