def scheduler_job():
if request.method == 'POST':
task_id = request.values.get('job_id')
action = request.values.get('operation')
i = timedelta(seconds=10)
run_time = datetime.now() + i
try:
task = db.session.query(Task).filter(Task.id == task_id).first()
task.web_scan_enable = 1
task.state = 2
task.start_time = run_time
db.session.add(task)
db.session.commit()
job = run_engine.apply_async(args=[task_id, action], countdown=10)
job_task_ref = ApJobsTaskRef(job.id, task_id, 'PENDING', run_time)
db.session.add(job_task_ref)
db.session.commit()
except Exception as e:
logger.exception(e)
return jsonify(dict(status=False, desc='重启扫描失败'))
else:
return jsonify(dict(status=True, desc='重启扫描成功'))
def add_task(task_id=None):
name = request.values.get('task_name')
scheme = request.values.get('task_scheme')
domain = request.values.get('task_domain')
source_ip = request.values.get('source_ip')
path = request.values.get('task_path')
cookie = request.values.get('task_cookie')
spider_type = request.values.get('spider_type')
task_policy = request.values.get('task_policy')
urls = request.values.get('urls')
target = request.values.get('target')
multiple_task = True if request.values.get('multiple_task') else False
run_now = True if request.values.get('run_now') else False
run_time = request.values.get('run_time')
rules = request.values.get('rules')
scan_key = request.values.get('scan_key')
try:
# 从接口提交的扫描任务，如果是全面扫描则扫描所有规则
if scan_key:
if not (name and urls and run_time and task_policy):
raise Exception
user_id = verify_scan_key(scan_key).id
if task_policy == '509':
rules = db.session.query(func.group_concat(WebVulFamily.id)).filter(WebVulFamily.parent_id != 0).first()[0]
spider_type = 2
else:
username = current_user.name
user_id = db.session.query(User).filter(User.name == username).first().id
except Exception, e:
logger.exception(e)
return jsonify(dict(status=False, desc='添加更新失败'))
def report_rebuild(job_id=None):
try:
#taskid = request.values.get('taskid')
# job_id = request.values.get('job_id')
job = db.session.query(ApJobsTaskRef).filter(
ApJobsTaskRef.job_id == job_id).first()
total = db.session.query(WebResult).filter(
WebResult.task_id == job.task_id).count()
message = 'web_result total is %s' % total
report = Report()
result = report.storage(job.task_id, job_id)
return jsonify({
'status': True,
'desc': '重新生成报告成功',
'reportid': int(result),
'message': message
})
except Exception, e:
logger.exception(e)
return jsonify({
'status': False,
'desc': '重新生成报告失败',
'reportid': 0,
'mesage': e.message
})
def create_rule_family(rule_family_id=None):
name = request.values.get('name')
describe = request.values.get('describe')
priority = request.values.get('priority')
if 'POST' == request.method:
try:
family = RuleFamily(name, describe, priority)
db.session.add(family)
db.session.commit()
except Exception as e:
logger.exception(e)
return jsonify(dict(status=False, desc='添加失败'))
else:
return jsonify(dict(status=True, desc='添加成功'))
else:
try:
family = db.session.query(RuleFamily).filter(RuleFamily.id == rule_family_id).first()
family.name = name
family.describe = describe
family.priority = priority
db.session.add(family)
db.session.commit()
except Exception as e:
logger.exception(e)
return jsonify(dict(status=False, desc='更新失败'))
else:
return jsonify(dict(status=True, desc='更新成功'))
def del_task_job(task_id):
job_task_ref = db.session.query(ApJobsTaskRef).filter(ApJobsTaskRef.task_id == task_id, ApJobsTaskRef.job_status==1,
ApJobsTaskRef.parent_id == None).first()
try:
if job_task_ref:
if job_task_ref.job_status != 1:
return jsonify(dict(status=False, desc='任务已执行，无法删除'))
res_del_db = None
res_revoke = revoke_job(job_task_ref.job_id)
if res_revoke:
res_del_db = del_job_db(job_task_ref.job_id)
if not res_del_db:
raise Exception
db.session.query(Task).filter(Task.id == task_id).delete()
db.session.query(TaskRuleFamilyRef).filter(TaskRuleFamilyRef.task_id == task_id).delete()
db.session.commit()
# 删除爬虫任务
del_spider_task(task_id)
except Exception as e:
logger.exception(e)
return jsonify(dict(status=False, desc='删除失败'))
else:
return jsonify(dict(status=True, desc='删除成功'))
def rep_vul_audit(id=None):
try:
db.session.query(WebResult).filter(WebResult.id == id).delete()
db.session.commit()
return jsonify(dict(status=True, desc='删除成功'))
except Exception, e:
logger.exception(e)
return jsonify(dict(status=False, desc='删除失败'))
def delete_rule(rule_id):
try:
db.session.query(Rule).filter(Rule.rule_id == rule_id).delete()
db.session.commit()
except Exception as e:
logger.exception(e)
return jsonify(dict(status=False, desc='删除失败'))
else:
return jsonify(dict(status=True, desc='删除成功'))
def del_vul_patch():
try:
res_str = request.values.get('vul_str')
task_id = request.values.get('task_id')
res_list = res_str.rstrip(',').split(',')
for res_id in res_list:
db.session.query(WebResult).filter(WebResult.id == res_id).delete()
db.session.commit()
return redirect('/report/processing/%s' % task_id)
except Exception, e:
logger.exception(e)
abort(404)
def api_job_progress2():
job_ids = request.values.get('job_id')
job_list = json.loads(job_ids).get('jobs')
resp = {}
for job_id in job_list:
task_pro = {}
try:
job = db.session.query(ApJobsTaskRef).filter(ApJobsTaskRef.job_id == job_id).first()
if not job:
task_pro['resp_status'] = False
task_pro['task_info'] = {'errorMsg': '查询失败,job为None'.decode('utf-8')}
else:
task = db.session.query(Task).filter(Task.id == job.task_id).first()
if job.job_status == 1:
response = {
'task_id': task.id,
'state': '未开始'.decode('utf-8'),
'current': 0,
'total': 100,
'status': 0
}
elif job.job_status == 3:
response = {
'task_id': task.id,
'state': '扫描完成'.decode('utf-8'),
'current': 100,
'total': 100,
'status': 3
}
else:
response = task_progress(job.task_id)
if job.run_time:
start_time = datetime.strftime(job.run_time, '%Y-%m-%d %H:%M:%S')
if job.end_time:
end_time = datetime.strftime(job.end_time, '%Y-%m-%d %H:%M:%S')
else:
end_time = '0000-00-00 00:00:00'
task_pro['resp_status'] = True
task_pro['task_info'] = {'task_id': job.task_id, 'task_name': task.name,
'policy': task.web_scan_policy, 'start_time': start_time, 'end_time': end_time,
'state': response.get('state'), 'current': response.get('current'),
'total': response.get('total'), 'status': response.get('status')}
resp[job_id] = task_pro
except Exception as e:
logger.exception(e)
task_pro['resp_status'] = False
task_pro['task_info'] = {'errorMsg': '查询失败'.decode('utf-8')}
resp[job_id] = task_pro
return jsonify(resp)
def storage(self, taskid, jobid=""):
logger.info("report storage %s, %s start" % (taskid, jobid))
try:
reportid = self.saveToDb(taskid, jobid)
# self.savePdf(reportid)
self.savePdf2(jobid)
return reportid
logger.info("report storage %s, %s end" % (taskid, jobid))
except Exception, e:
logger.exception(e)
logger.info("report storage %s, %s exception" % (taskid, jobid))
return 0
def run_report(self, task_id, job_id):
try:
logger.debug("扫描任务task_id:%s,job_id:%s执行报告生成任务 开始" %
(task_id, job_id))
rep = Report()
report_id = rep.storage(task_id, job_id)
if not report_id:
rep.checkPdfExists()
logger.debug("扫描任务task_id:%s,job_id:%s执行报告生成任务 结束" %
(task_id, job_id))
except Exception, e:
logger.exception(e)
logger.debug("扫描任务task_id:%s,job_id:%s执行报告生成任务 异常" %
(task_id, job_id))
def verify_scan_key(key):
try:
s = Serializer(SECRET_KEY, expires_in=SESSION_LIFETIME)
token = db.session.query(TokenMapping).filter(
TokenMapping.uuid == key).first().token
user_json = s.loads(token)
name = user_json['name']
user = db.session.query(User).filter(User.name == name,
User.status == 1).first()
if user:
return user
else:
return None
except Exception, e:
logger.exception(e)
return None
def report_list():
try:
scan_key = request.values.get('scan_key')
search_msg = request.values.get('search_msg', '')
if scan_key:
user_id = verify_scan_key(scan_key).id
else:
user_id = current_user.id
user = db.session.query(User).filter(User.id == user_id).first()
admin_id = db.session.query(Group).filter(Group.name == 'ADMIN').first().id
if str(admin_id) in user.groups.split(','):
query = db.session.query(Task).join(ApJobsTaskRef, ApJobsTaskRef.task_id == Task.id). \
filter(ApJobsTaskRef.job_status == 3, ApJobsTaskRef.parent_id == None).order_by(Task.id.desc())
else:
query = db.session.query(Task).join(ApJobsTaskRef, ApJobsTaskRef.task_id == Task.id). \
filter(ApJobsTaskRef.job_status == 3, ApJobsTaskRef.parent_id == None, Task.user_id == user_id).order_by(Task.id.desc())
if search_msg:
like_msg = '%%%s%%' % search_msg
query = query.filter(or_(Task.id.like(search_msg), Task.name.like(like_msg),
Task.target.like(like_msg)))
page, per_page, offset, search_msg = get_page_items()
tasks = query.limit(per_page).offset(offset).all()
total = query.count()
pagination = get_pagination(page=page,
per_page=per_page,
total=total,
# record_name="server",
format_total=True,
format_number=True
# search=True,
# search_msg=search_msg
)
taskids = [task.id for task in tasks]
reportList = db.session.query(ModelReport.job_id, ModelReport.task_id).filter(ModelReport.task_id.in_(taskids)).order_by(
ModelReport.id.desc()).all()
reports = {}
for report in reportList:
if not reports.has_key(report.task_id):
reports[report.task_id] = report
for task_id in taskids:
if not reports.has_key(task_id):
reports[task_id] = ('', task_id)
except Exception, e:
logger.exception(e)
return render_template('error-not-safe.html')
def add_task(task_id=None):
name = request.values.get('task_name')
name = escape(name.decode('utf-8'))
# scheme = request.values.get('task_scheme')
# domain = request.values.get('task_domain')
source_ip = request.values.get('source_ip')
if source_ip and not re.match('^(\d{1,3}\.){3}\d{1,3}$', source_ip):
return jsonify(dict(status=False, desc='添加失败, 源IP格式错误'))
patch_no = request.values.get('patch_no')
cookie = request.values.get('task_cookie')
spider_enable = request.values.get('spider_enable')
task_policy = request.values.get('task_policy')
rep_model_id = request.values.get('rep_model')
urls = request.values.get('urls')
target = request.values.get('target')
# multiple_task = True if request.values.get('multiple_task') else False
run_now = True if request.values.get('run_now') else False
run_time = request.values.get('run_time')
rules = request.values.get('rules')
scan_key = request.values.get('scan_key')
try:
if not rep_model_id:
rep_model_id = db.session.query(ReportModel).filter(or_(ReportModel.company == '上海云盾信息技术有限公司',
ReportModel.model_name == '盾眼默认模板')).first().model_id
# 从接口提交的扫描任务，如果是全面扫描则扫描所有规则
if scan_key:
if not (name and urls and task_policy):
raise Exception
user_id = verify_scan_key(scan_key).id
# if task_policy == '509':
# rules = db.session.query(func.group_concat(WebVulFamily.id)).filter(WebVulFamily.parent_id != 0).first()[0]
else:
username = current_user.name
user_id = db.session.query(User).filter(User.name == username).first().id
except Exception, e:
logger.exception(e)
return jsonify(dict(status=False, desc='添加更新失败'))
# if run_now:
# # 通过celery任务启动
# # run_time = datetime.now() + i
# job = run_engine.apply_async(args=[task_id, action], countdown=0)
#
# else:
# # 通过celery任务启动
# delay_seconds = (run_time - datetime.now()).seconds
# job = run_engine.apply_async(args=[task_id, action], countdown=delay_seconds)
#
# job_task_ref = ApJobsTaskRef(job.id, task_id, 'PENDING', run_time)
# db.session.add(job_task_ref)
# db.session.commit()
except Exception as e:
logger.exception(e)
return jsonify(dict(status=False, desc='添加失败'))
else:
return jsonify(dict(status=True, desc='添加成功', task_id=task_id))
else:
try:
task = db.session.query(Task).filter(Task.id == task_id).first()
task.name = name
task.target = target
task.web_scan_policy = task_policy
task.spider_type = spider_type
task.web_scan_enable = 1
task.state = 2
task.user_id = user_id
def create_rule(rule_id=None):
rule_name = request.values.get('rule_name')
rule_family = request.values.get('rule_family')
# rule_tag = request.values.get('rule_tag')
rule_tag = '' # tag标签在scan_site.py写入字典scan_cnf里面，此处停用。为不影响其他代码，暂时置空处理。
level = request.values.get('bug_level')
if_head = True if request.values.get('if_head') else False
run_mode = request.values.get('run_mode')
inj_area = request.values.get('inj_area')
inj_way = request.values.get('inj_way')
inj_point = request.values.get('inj_point')
inj_value_str = request.values.get('inj_value')
code_mode = request.values.get('code_mode')
judge_code1 = request.values.get('judge_code1')
judge_code2 = request.values.get('judge_code2')
judge_keyword = request.values.get('judge_keyword')
content_mode = request.values.get('content_mode')
judge_content = request.values.get('judge_content')
similar_mode = request.values.get('similar_mode')
similar = request.values.get('similar')
describe = request.values.get('describe')
solution = request.values.get('solution')
judge_str = request.values.get('judge')
# 规范传入参数，防止XSS
rule_name = escape(rule_name.decode('utf-8'))
describe = escape(describe.decode('utf-8'))
solution = escape(solution.decode('utf-8'))
judge = {}
if code_mode:
code_dict = {'mode': code_mode}
code_value = []
if judge_code1:
code_value.append(judge_code1)
else:
code_value.append('0')
if judge_code2:
code_value.append(judge_code2)
else:
code_value.append('999')
code_dict['value'] = code_value
judge["http_code"] = code_dict
if judge_keyword:
judge["keyword"] = judge_keyword
if content_mode:
content_dict = {'mode': content_mode, 'value': judge_content}
judge["content"] = content_dict
if similar_mode:
similar_dict = {'mode': similar_mode, 'value': float(similar)/100}
judge["similar"] = similar_dict
if 'POST' == request.method:
try:
inj_values = inj_value_str.split('\r\n')
if '' in inj_values:
inj_values.remove('')
# vul_id = rule_name.split('-')[0]
# rule_exists = db.session.query(Rule).filter(Rule.vul_id == vul_id).first()
# if rule_exists:
# return jsonify(dict(status=False, desc='ID为'+vul_id+'的漏洞已经存在'))
family = db.session.query(WebVulFamily).filter(WebVulFamily.desc == rule_family).first()
module = db.session.query(WebVulFamily).filter(WebVulFamily.id == family.parent_id).first()
vul_script = WebVulList(0, rule_name, 1, family.desc, module.desc, 3, None, level,
describe, solution, None, 750, rule_tag, family.id, module.id)
db.session.add(vul_script)
db.session.flush()
vul_id = vul_script.id
vul_script.vul_id = vul_id
db.session.add(vul_script)
db.session.commit()
ref = WebVulFamilyRef(family.parent_id, family.id, vul_id)
db.session.add(ref)
db.session.commit()
# for inj_value in inj_values:
rule_json = {"area": inj_area, "inj_way": inj_way, "inj_point": inj_point, "inj_value": inj_values, "judge": judge}
rule = Rule(family.id, rule_name, json.dumps(rule_json), inj_area, inj_way, inj_point, json.dumps(inj_values),
json.dumps(judge), describe, run_mode, rule_tag, if_head, vul_id)
db.session.add(rule)
db.session.commit()
# 规则从web_vul_list_copy 导入 web_vul_list ，并删除copy中的记录
# web_vul_copy = db.session.query(WebVulListCopy).filter(WebVulListCopy.vul_id == vul_id).first()
# result = add_policy_script(web_vul_copy.vul_name, 3, '', web_vul_copy.level, web_vul_copy.desc,
# web_vul_copy.solu, web_vul_copy.priority, family.id, vul_id=vul_id, tag=rule_tag)
# if not result:
# raise Exception
# db.session.query(WebVulListCopy).filter(WebVulListCopy.vul_id == vul_id).delete()
# db.session.commit()
except Exception as e:
logger.exception(e)
return jsonify(dict(status=False, desc='添加失败'))
else:
return jsonify(dict(status=True, desc='添加成功'))
else:
try:
value_list = json.loads(inj_value_str)
rule_json = {"area": inj_area, "inj_way": inj_way, "inj_point": inj_point, "inj_value": value_list,
"judge": json.loads(judge_str)}
rule = db.session.query(Rule).filter(Rule.rule_id == rule_id).first()
rule.rule_name = rule_name
rule.rule_family = rule_family
rule.rule_json = json.dumps(rule_json)
rule.area = inj_area
rule.inj_way = inj_way
rule.inj_point = inj_point
rule.inj_value = inj_value_str
rule.judge = judge_str
rule.describe = describe
rule.run_mode = run_mode
rule.if_head = if_head
db.session.add(rule)
db.session.commit()
except Exception as e:
logger.exception(e)
return jsonify(dict(status=False, desc='更新失败'))
else:
return jsonify(dict(status=True, desc='更新成功'))