def scheduleUrl(self):
"""
run_type为1,脚本需定义run_url方法
"""
DEBUG("scheduleUrl start")
sql = "SELECT `rule_id`,`risk`,`file_name` FROM `%s` WHERE `run_type` = 1 ORDER BY `priority`" % RULE_TABLE
# rules = []
# for rule in db.iter(sql):
# rules.append((str(rule.rule_id), rule.file_name, rule.risk))
rules = [(str(rule.rule_id), rule.file_name, rule.risk) for rule in db.iter(sql) if str(rule.rule_id) not in self.finished_progress]
if not conf.spider_finish: #spider not finished, start crawler
CrawlEngine.start()
sql = "SELECT `url`,`method`,`params`,`referer` FROM %s WHERE `task_id`=%s" % (URL_TABLE, self.task_id)
# reqs = []
# for url in db.iter(sql):
# reqs.append(Url(url.url, url.method, url.params, url.referer))
reqs = [Url(url.url, url.method, url.params, url.referer) for url in db.iter(sql)]
for rule_id, filename, risk in rules:
run_url = attr_from_script(filename, RUN_URL_DEFAULT_FUN)
if run_url:
DEBUG("rule_id:%s filename:%s run_url start" % (rule_id, filename))
for req in reqs:
self.pool.spawn(self.runUrl, rule_id, run_url, req, filename, risk)
gevent.sleep(0)
DEBUG("rule_id:%s filename:%s run_url end" % (rule_id, filename))
DEBUG("scheduleUrl end")
def scheduleUrl(self):
"""
run_type为1,脚本需定义run_url方法
"""
DEBUG("scheduleUrl start")
sql = "SELECT `rule_id`,`risk`,`file_name` FROM `%s` WHERE `run_type` = 1 ORDER BY `priority`" % RULE_TABLE
# rules = []
# for rule in db.iter(sql):
# rules.append((str(rule.rule_id), rule.file_name, rule.risk))
rules = [(str(rule.rule_id), rule.file_name, rule.risk)
for rule in db.iter(sql)
if str(rule.rule_id) not in self.finished_progress]
if not conf.spider_finish: #spider not finished, start crawler
CrawlEngine.start()
sql = "SELECT `url`,`method`,`params`,`referer` FROM %s WHERE `task_id`=%s" % (
URL_TABLE, self.task_id)
# reqs = []
# for url in db.iter(sql):
# reqs.append(Url(url.url, url.method, url.params, url.referer))
reqs = [
Url(url.url, url.method, url.params, url.referer)
for url in db.iter(sql)
]
for rule_id, filename, risk in rules:
run_url = attr_from_script(filename, RUN_URL_DEFAULT_FUN)
if run_url:
DEBUG("rule_id:%s filename:%s run_url start" %
(rule_id, filename))
for req in reqs:
self.pool.spawn(self.runUrl, rule_id, run_url, req,
filename, risk)
gevent.sleep(0)
DEBUG("rule_id:%s filename:%s run_url end" %
(rule_id, filename))
DEBUG("scheduleUrl end")
def run():
CrawlEngine.start()
def run():
CrawlEngine.start()