def heartBeat():
# 心跳
dataMonitor = DataMonitorDao()
results = dataMonitor.getAllHeartBeatTime(cursor_out=None)
print 'check....'
needSendTypes = []
currTimeStr = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
print 'now: ', currTimeStr
for result in results:
(type, update_time) = result
timeArray = time.strptime(str(update_time), "%Y-%m-%d %H:%M:%S")
update_time_long = int(time.mktime(timeArray))
print 'that: ', update_time
if outOfData(update_time_long):
print 'outOfData--------', type, update_time
needSendTypes.append((type, update_time))
if len(needSendTypes) and checkNeedSend():
print '-------need send email---------'
randomStr = str(random.uniform(0, 1))
ret = mail(randomStr + ':' + ','.join(needSendTypes))
for type, update_time in needSendTypes:
print type, update_time
if ret:
print 'send_success'
global lastSendTime
lastSendTime = int(time.time())
def __init__(self, name=None, **kwargs):
super(WXSourceSpider, self).__init__(name=None, **kwargs)
self.count = 0
self.wxSourceDao = WxSourceDao()
self.currIp = ''
self.logDao = LogDao(self.logger, 'weixin_source_catch')
self.dataMonitor = DataMonitorDao()
def __init__(self, name=None, **kwargs):
super(SinaSpider, self).__init__(name=None, **kwargs)
self.count = 0
self.logDao = LogDao(self.logger, 'sina_detail')
self.checkDao = CheckDao()
# 用于缓存css
self.css = {'hash': 'style'}
self.dataMonitor = DataMonitorDao()
def __init__(self, name=None, **kwargs):
super(WXDetailSpider, self).__init__(name=None, **kwargs)
self.count = 0
self.wxSourceDao = WxSourceDao()
self.logDao = LogDao(self.logger, 'weixin_list_detail')
self.checkDao = CheckDao()
self.dataMonitor = DataMonitorDao()
self.wxSources = []
self.brokenAccounts = [] # 当前被禁止了的账号，下次抓取优先抓取
def __init__(self, name=None, **kwargs):
super(WYDetailSpider, self).__init__(name=None, **kwargs)
self.count = 0
self.request_stop = False
self.request_stop_time = 0
self.logDao = LogDao(self.logger, 'wangyi_list_detail')
self.checkDao = CheckDao()
# 用于缓存css
self.css = {'hash': 'style'}
self.dataMonitor = DataMonitorDao()
def heartBeat():
# 心跳
dataMonitor = DataMonitorDao()
dataMonitor.heartBeat('weixin_heartbeat')
def heartBeat():
# 心跳
dataMonitor = DataMonitorDao()
def heartBeat():
# 心跳
dataMonitor = DataMonitorDao()
dataMonitor.heartBeat('wangyi_heartbeat')
def heartBeat():
# 心跳
dataMonitor = DataMonitorDao()
dataMonitor.heartBeat('tengxun_heartbeat')
def heartBeat():
# 心跳
dataMonitor = DataMonitorDao()
dataMonitor.heartBeat('sina_heartbeat')