def run(self):
while True:
num = getCrawlNoRssRequestLength()
logging.info("********need deal request num : %s " % num)
if not num:
if self.runNum >= 1:
logging.info("*****************size:%s********runNum:%s********" % (self.size, self.runNum))
self.runSpider()
break
else:
self.initSpider()
if self.runNum >= self.size:
logging.info("*****************size:%s********runNum:%s********" % (self.size, self.runNum))
self.runSpider()
break
def run(self):
while True:
num = getCrawlNoRssRequestLength()
logging.info("********need deal request num : %s " % num)
if not num:
if self.runNum >= 1:
logging.info(
"*****************size:%s********runNum:%s********" %
(self.size, self.runNum))
self.runSpider()
break
else:
self.initSpider()
if self.runNum >= self.size:
logging.info(
"*****************size:%s********runNum:%s********" %
(self.size, self.runNum))
self.runSpider()
break
def startScript():
times = 0
# beginTime = int(time.time())
while True:
try:
times += 1
num = getCrawlNoRssRequestLength()
logging.info("**********need deal request num :%s************" % num)
if not num:
logging.info("**********sleep:%s************" % MAIN_LOOP_SLEEP_TIME)
time.sleep(MAIN_LOOP_SLEEP_TIME)
else:
os.system('python runSpider.py')
# if times > RUN_SYNC_INTERVAL_TIMES or int(time.time()) - beginTime > RUN_SYNC_INTERVAL_TIME:
# logging.info("**********sync crawl infos ************")
# sync = SyncCrawlInfos()
# sync.index()
# times = 0
# beginTime = int(time.time())
except Exception, e:
logging.info("--------------%s------------" % e)
