def main(): proxy = getcfg('http_proxy') if proxy: print 'Using http proxy %s' % (proxy, ) pollers.requestoptions['proxies'] = {'http': proxy, 'https': proxy} try: threading.stack_size(512 * 1024) except BaseException as e: print 'Error changing stack size:', repr(e) # Connect to the database... Database.get() parser = ParserThread() parser.start() for poller in POLLERS.values(): poller.start() # wait for them to finish! for poller in POLLERS.values(): while poller.is_alive(): time.sleep(1) while parser.is_alive(): time.sleep(1) return
def main(): proxy = getcfg("http_proxy") if proxy: print "Using http proxy %s" % (proxy,) pollers.requestoptions["proxies"] = {"http": proxy, "https": proxy} try: threading.stack_size(512 * 1024) except BaseException as e: print "Error changing stack size:", repr(e) # Connect to the database... Database.get() parser = ParserThread() parser.start() for poller in POLLERS.values(): poller.start() # wait for them to finish! for poller in POLLERS.values(): while poller.is_alive(): time.sleep(1) while parser.is_alive(): time.sleep(1) return
def __init__(self, name, interval, poll_url, regexp, method='get'): BasePoller.__init__(self, name, interval) self.poll_url = poll_url self.regexp = regexp self.session = requests.session() self.method = method self.last_seen = getcfg('last_seen_%s' % (self.name, )) self.daemon = True self.downloader = None return