def __init__(self):
# DB
self.mysqlAccess = MysqlAccess() # mysql access
# 页面
self.site_page = None
# 抓取开始时间
self.begin_time = Common.now()
def __init__(self):
# tc spot type
self.worker_type = Config.TC_Spot
# DB
self.tc_type = Config.TC_TYPE # queue type
self.mysqlAccess = MysqlAccess() # mysql access
self.redisQueue = RedisQueue() # redis queue
self.mongofsAccess = MongofsAccess() # mongodb fs access
# 抓取设置
self.crawler = TCCrawler()
# message
self.message = Message()
# 抓取时间设定
self.crawling_time = Common.now() # 当前爬取时间
self.begin_time = Common.now()
self.begin_date = Common.today_s()
self.begin_hour = Common.nowhour_s()
def __init__(self, m_type):
# DB
self.mysqlAccess = MysqlAccess() # mysql access
# channel queue
self.chan_queue = XCQ('channel', 'spot')
self.work = XCWorker()
# 默认类别
self.channel_list = [(
1,
'http://www.ly.com/scenery/scenerysearchlist_22_295__0_0_0_0_0_0_0.html',
1)]
# 页面
self.site_page = None
# 抓取开始时间
self.begin_time = Common.now()
# 分布式主机标志
self.m_type = m_type
def __init__(self, _q_type, thread_num=10, a_val=None):
# parent construct
MyThread.__init__(self, thread_num)
# thread lock
self.mutex = threading.Lock()
self.worker_type = Config.XC_Spot
# message
self.message = Message()
# db
self.mysqlAccess = MysqlAccess() # mysql access
self.mongofsAccess = MongofsAccess() # mongodb fs access
# xc queue type
self._q_type = _q_type # new:新增商品
# appendix val
self.a_val = a_val
# activity items
self.items = []
# dial client
self.dial_client = DialClient()
# local ip
self._ip = Common.local_ip()
# router tag
self._tag = 'ikuai'
#self._tag = 'tpent'
# give up item, retry too many times
self.giveup_items = []
def __init__(self, key, q_type, thread_num=10, a_val=None):
# parent construct
MyThread.__init__(self, thread_num)
# thread lock
self.mutex = threading.Lock()
self.xc_type = Config.XC_TYPE # xc type
#self.item_type = q_type # item queue type
# db
self.mysqlAccess = MysqlAccess() # mysql access
self.redisQueue = RedisQueue() # redis queue
self.mongofsAccess = MongofsAccess() # mongodb fs access
# xc queue type
self.xc_queue_type = q_type # new...
self._key = key # redis queue key
# appendix val
self.a_val = a_val
# return items
self.items = []
# dial client
self.dial_client = DialClient()
# local ip
self._ip = Common.local_ip()
# router tag
self._tag = 'ikuai'
#self._tag = 'tpent'
# give up item, retry too many times
self.giveup_items = []
