def __init__(self, restart, browser, ext_port, proxy_file, proxy_scheme, log_q): self.restart = restart self.browser_name = browser self.browser_inst = None if self.browser_name == 'Firefox': self.__class__ = FirefoxCrawlerProcess # May want to just ignore ext_port for other browsers. self.ext_port = ext_port self.logger = mplogging.setupSubProcessLogger("Crawler", log_q) self.proxy = Proxy(proxy_file, proxy_scheme, self.logger) # To make sure the process doesn't take more than 15 min to complete # a single visit. Kill the process if it does. signal.signal(signal.SIGALRM, self.alarm_handler) # We would like to restart browser every so many visits. self.num_visits = 0
def __init__(self, log_q): """log_q is the multiprocessing queue that we place records on for logging. These are processed by LogQueueReader thread in mplogging. """ self.cleanup = False self.logger = mplogging.setupSubProcessLogger("QController", log_q)