def crawl(self):
while True:
_data = None
try:
try:
# 取队列消息
_data = self.get_q()
except Empty as e:
# 队列为空，退出
#print '# queue is empty', e
break
_val = _data[1]
item = BagItem(self.home_url, self.brand_type)
item.antPage(_val)
self.push_back(self.items, item.outItem())
sql = item.outTuple()
self.mysqlAccess.insert_item(sql)
# 延时
time.sleep(0.1)
# 通知queue, task结束
self.queue.task_done()
except Exception as e:
print 'Unknown exception crawl item :', e
Common.traceback_log()
self.crawlRetry(_data)
# 通知queue, task结束
self.queue.task_done()
time.sleep(5)
