def __init__(self, taskdb, projectdb, newtask_queue, status_queue,
out_queue, data_path='./data', resultdb=None):
self.taskdb = taskdb
self.projectdb = projectdb
self.resultdb = resultdb
self.newtask_queue = newtask_queue
self.status_queue = status_queue
self.out_queue = out_queue
self.data_path = data_path
self._send_buffer = deque()
self._quit = False
self._exceptions = 0
self.projects = dict()
self._force_update_project = False
self._last_update_project = 0
self._last_tick = int(time.time())
self._postpone_request = []
self._cnt = {
"5m_time": counter.CounterManager(
lambda: counter.TimebaseAverageEventCounter(30, 10)),
"5m": counter.CounterManager(
lambda: counter.TimebaseAverageWindowCounter(30, 10)),
"1h": counter.CounterManager(
lambda: counter.TimebaseAverageWindowCounter(60, 60)),
"1d": counter.CounterManager(
lambda: counter.TimebaseAverageWindowCounter(10 * 60, 24 * 6)),
"all": counter.CounterManager(
lambda: counter.TotalCounter()),
}
self._cnt['1h'].load(os.path.join(self.data_path, 'scheduler.1h'))
self._cnt['1d'].load(os.path.join(self.data_path, 'scheduler.1d'))
self._cnt['all'].load(os.path.join(self.data_path, 'scheduler.all'))
self._last_dump_cnt = 0
def __init__(self,
inqueue,
outqueue,
poolsize=100,
proxy=None,
async_mode=True):
self.inqueue = inqueue
self.outqueue = outqueue
self.poolsize = poolsize
self._running = False
self._quit = False
self.proxy = proxy
self.async_mode = async_mode
self.ioloop = tornado.ioloop.IOLoop()
self.robots_txt_cache = {}
# binding io_loop to http_client here
if self.async_mode:
self.http_client = MyCurlAsyncHTTPClient(max_clients=self.poolsize,
io_loop=self.ioloop)
else:
self.http_client = tornado.httpclient.HTTPClient(
MyCurlAsyncHTTPClient, max_clients=self.poolsize)
self._cnt = {
'5m':
counter.CounterManager(
lambda: counter.TimebaseAverageWindowCounter(30, 10)),
'1h':
counter.CounterManager(
lambda: counter.TimebaseAverageWindowCounter(60, 60)),
}