if spider_total_number - 1 == spider_serial_number: limit += queue_size % spider_total_number print "skip = ", skip, ", limit = ", limit with gevent.Timeout(None, False): print "This spider is start." orders = queue.find(skip, limit) while orders.count() > 0: for order in orders: thead_pool.spawn(httpclient_pool.request, order) thead_pool.join() print "Start =", httpclient.start,", End =", httpclient.end,", Error =", httpclient.error httpclient.start = 0 httpclient.end = 0 httpclient.error = 0 orders = queue.find(skip, limit) print "This spider is finished." if __name__ == '__main__': queue.init_db() spider()
def test(): queue.init_db() snatch()