def __init__(self, restart, browser, ext_port, proxy_file, proxy_scheme,
log_q):
self.restart = restart
self.browser_name = browser
self.browser_inst = None
if self.browser_name == 'Firefox':
self.__class__ = FirefoxCrawlerProcess
# May want to just ignore ext_port for other browsers.
self.ext_port = ext_port
self.logger = mplogging.setupSubProcessLogger("Crawler", log_q)
self.proxy = Proxy(proxy_file, proxy_scheme, self.logger)
# To make sure the process doesn't take more than 15 min to complete
# a single visit. Kill the process if it does.
signal.signal(signal.SIGALRM, self.alarm_handler)
# We would like to restart browser every so many visits.
self.num_visits = 0
def __init__(self, restart, browser, ext_port, proxy_file, proxy_scheme,
log_q):
self.restart = restart
self.browser_name = browser
self.browser_inst = None
if self.browser_name == 'Firefox':
self.__class__ = FirefoxCrawlerProcess
# May want to just ignore ext_port for other browsers.
self.ext_port = ext_port
self.logger = mplogging.setupSubProcessLogger("Crawler", log_q)
self.proxy = Proxy(proxy_file, proxy_scheme, self.logger)
# To make sure the process doesn't take more than 15 min to complete
# a single visit. Kill the process if it does.
signal.signal(signal.SIGALRM, self.alarm_handler)
# We would like to restart browser every so many visits.
self.num_visits = 0
def __init__(self, log_q):
"""log_q is the multiprocessing queue that we place records on for
logging. These are processed by LogQueueReader thread in mplogging.
"""
self.cleanup = False
self.logger = mplogging.setupSubProcessLogger("QController", log_q)
