if spider_total_number - 1 == spider_serial_number:
limit += queue_size % spider_total_number
print "skip = ", skip, ", limit = ", limit
with gevent.Timeout(None, False):
print "This spider is start."
orders = queue.find(skip, limit)
while orders.count() > 0:
for order in orders:
thead_pool.spawn(httpclient_pool.request, order)
thead_pool.join()
print "Start =", httpclient.start,", End =", httpclient.end,", Error =", httpclient.error
httpclient.start = 0
httpclient.end = 0
httpclient.error = 0
orders = queue.find(skip, limit)
print "This spider is finished."
if __name__ == '__main__':
queue.init_db()
spider()
def test():
queue.init_db()
snatch()
