def main():
proxy = getcfg('http_proxy')
if proxy:
print 'Using http proxy %s' % (proxy, )
pollers.requestoptions['proxies'] = {'http': proxy, 'https': proxy}
try:
threading.stack_size(512 * 1024)
except BaseException as e:
print 'Error changing stack size:', repr(e)
# Connect to the database...
Database.get()
parser = ParserThread()
parser.start()
for poller in POLLERS.values():
poller.start()
# wait for them to finish!
for poller in POLLERS.values():
while poller.is_alive():
time.sleep(1)
while parser.is_alive():
time.sleep(1)
return
def main():
proxy = getcfg("http_proxy")
if proxy:
print "Using http proxy %s" % (proxy,)
pollers.requestoptions["proxies"] = {"http": proxy, "https": proxy}
try:
threading.stack_size(512 * 1024)
except BaseException as e:
print "Error changing stack size:", repr(e)
# Connect to the database...
Database.get()
parser = ParserThread()
parser.start()
for poller in POLLERS.values():
poller.start()
# wait for them to finish!
for poller in POLLERS.values():
while poller.is_alive():
time.sleep(1)
while parser.is_alive():
time.sleep(1)
return
def __init__(self, name, interval, poll_url, regexp, method='get'):
BasePoller.__init__(self, name, interval)
self.poll_url = poll_url
self.regexp = regexp
self.session = requests.session()
self.method = method
self.last_seen = getcfg('last_seen_%s' % (self.name, ))
self.daemon = True
self.downloader = None
return
def __init__(self, name, interval, poll_url, regexp, method='get'):
BasePoller.__init__(self, name, interval)
self.poll_url = poll_url
self.regexp = regexp
self.session = requests.session()
self.method = method
self.last_seen = getcfg('last_seen_%s' % (self.name, ))
self.daemon = True
self.downloader = None
return
