class XCSpot():
'''A class of XC spots'''
def __init__(self, m_type):
# DB
self.mysqlAccess = MysqlAccess() # mysql access
# channel queue
self.chan_queue = XCQ('channel', 'spot')
self.work = XCWorker()
# 默认类别
self.channel_list = [(
1,
'http://www.ly.com/scenery/scenerysearchlist_22_295__0_0_0_0_0_0_0.html',
1)]
# 页面
self.site_page = None
# 抓取开始时间
self.begin_time = Common.now()
# 分布式主机标志
self.m_type = m_type
def antPage(self):
try:
# 主机器需要配置redis队列
if self.m_type == 'm':
val = ('1', )
channel_list = self.mysqlAccess.selectChannel(val)
if not channel_list:
channel_list = self.channel_list
if channel_list and len(channel_list) > 0:
channel_val_list = []
for c in channel_list:
channel_val_list.append(c + (self.begin_time, ))
# 清空channel redis队列
self.chan_queue.clearQ()
# 保存channel redis队列
self.chan_queue.putlistQ(channel_val_list)
Common.log('# channel queue end')
else:
Common.log('# not find channel...')
# channel
obj = 'channel'
crawl_type = 'spot'
_val = (self.begin_time, )
self.work.process(obj, crawl_type, _val)
except Exception as e:
Common.log('# XCSpot antpage error: %s' % e)
Common.traceback_log()
