def get_context_data(self, **kwargs):
context = super(TaskListView, self).get_context_data(**kwargs)
task_count = ScanTask.objects.all().count()
if 'p' in self.request.GET:
page = int(self.request.GET['p'])
else:
page = 1
# check page
if page*50 > task_count:
page = 1
rows = ScanTask.objects.all().order_by('-id')[(page-1)*50: page*50]
context['tasks'] = rows
context['page'] = page
max_page = task_count / 50 if task_count % 50 == 0 else (task_count / 50)+1
context['max_page'] = max_page
context['page_range'] = range(int(max_page))[1:]
for task in context['tasks']:
task.is_finished = int(task.is_finished)
task.parameter_config = del_sensitive_for_config(task.parameter_config)
project_id = get_and_check_scantask_project_id(task.id)
project = Project.objects.filter(id=project_id).first()
task.project_name = project.project_name
return context
def get(request, task_id):
task = ScanTask.objects.filter(id=task_id).first()
visit_token = ""
if 'token' in request.GET:
visit_token = request.GET['token']
project_id = get_and_check_scantask_project_id(task.id)
project = Project.objects.filter(id=project_id).first()
taskresults = get_and_check_scanresult(task.id).objects.filter(scan_project_id=project_id, is_active=1).all()
newevilfuncs = get_and_check_evil_func(task.id)
task.is_finished = int(task.is_finished)
task.parameter_config = del_sensitive_for_config(task.parameter_config)
for taskresult in taskresults:
taskresult.is_unconfirm = int(taskresult.is_unconfirm)
if not task:
return HttpResponseNotFound('Task Not Found.')
else:
data = {
'task': task,
'taskresults': taskresults,
'newevilfuncs': newevilfuncs,
'visit_token': visit_token,
'project': project,
}
return render(request, 'dashboard/tasks/task_detail.html', data)
def get(request, task_id):
task = ScanTask.objects.filter(id=task_id).first()
visit_token = ""
if 'token' in request.GET:
visit_token = request.GET['token']
project_id = get_and_check_scantask_project_id(task.id)
project = Project.objects.filter(id=project_id).first()
taskresults = get_and_check_scanresult(task.id).objects.filter(scan_project_id=project_id, is_active=1).all()
newevilfuncs = get_and_check_evil_func(task.id)
task.is_finished = int(task.is_finished)
task.parameter_config = del_sensitive_for_config(task.parameter_config)
for taskresult in taskresults:
taskresult.is_unconfirm = int(taskresult.is_unconfirm)
taskresult.level = 0
if taskresult.cvi_id == '9999':
vender_vul_id = taskresult.vulfile_path.split(":")[-1]
if vender_vul_id:
vv = VendorVulns.objects.filter(id=vender_vul_id).first()
if vv:
taskresult.vulfile_path = "[{}]{}".format(vv.vendor_name, vv.title)
taskresult.level = VENDOR_VUL_LEVEL[vv.severity]
taskresult.vid = vv.id
# 处理多个refer的显示问题
references = []
if re.search(r'"http[^"]+"', taskresult.source_code, re.I):
rs = re.findall(r'"http[^"]+"', taskresult.source_code, re.I)
for r in rs:
references.append(r.strip('"'))
else:
references = [taskresult.source_code.strip('"')]
taskresult.source_code = references
else:
r = Rules.objects.filter(svid=taskresult.cvi_id).first()
taskresult.level = VUL_LEVEL[r.level]
if not task:
return HttpResponseNotFound('Task Not Found.')
else:
data = {
'task': task,
'taskresults': taskresults,
'newevilfuncs': newevilfuncs,
'visit_token': visit_token,
'project': project,
}
return render(request, 'dashboard/tasks/task_detail.html', data)
def tasklog(req, task_id):
task = ScanTask.objects.filter(id=task_id).first()
visit_token = ""
if 'token' in req.GET:
visit_token = req.GET['token']
# check task是否存在
if not task:
return redirect("dashboard:tasks_list")
# check task 的状态，只有完成才能继续
if not task.is_finished:
return HttpResponse(
"Ooooops, Maybe this task still in progress or has error, you can't view the log..."
)
project_id = get_and_check_scantask_project_id(task_id)
srts = get_and_check_scanresult(task_id).objects.filter(
scan_project_id=project_id)
nefs = NewEvilFunc.objects.filter(project_id=project_id)
ResultFlow = get_resultflow_class(task_id)
rfs = ResultFlow.objects.all()
task.parameter_config = " ".join(ast.literal_eval(
task.parameter_config)).replace('\\', '/')
resultflowdict = {}
for rf in rfs:
if rf.vul_id not in resultflowdict:
resultflowdict[rf.vul_id] = {
'id': rf.vul_id,
'flow': [],
}
rfdict = {
'type': rf.node_type,
'content': rf.node_content,
'path': rf.node_path,
'lineno': rf.node_lineno,
'details': rf.node_source
}
resultflowdict[rf.vul_id]['flow'].append(rfdict)
# 扫描结果
data = {
"task": task,
"taskresults": srts,
"newevilfuncs": nefs,
"resultflowdict": resultflowdict,
'visit_token': visit_token
}
return render(req, 'backend/tasklog.html', data)
def index(req):
tasks = ScanTask.objects.all().order_by("-id")[:100]
for task in tasks:
task.is_finished = int(task.is_finished)
task.parameter_config = del_sensitive_for_config(task.parameter_config)
project_id = get_and_check_scantask_project_id(task.id)
project = Project.objects.filter(id=project_id).first()
task.project_name = project.project_name
data = {'tasks': tasks}
return render(req, 'dashboard/index.html', data)
def get(request, task_id):
scantask = ScanTask.objects.filter(id=task_id).first()
if not scantask.is_finished:
return JsonResponse({
"code":
403,
"status":
False,
"message":
"Task {} not finished.".format(task_id)
})
project_id = get_and_check_scantask_project_id(task_id)
nefs = list(NewEvilFunc.objects.filter(project_id=project_id).values())
return JsonResponse({"code": 200, "status": True, "message": nefs})
def get_context_data(self, **kwargs):
context = super(TaskListView, self).get_context_data(**kwargs)
rows = ScanTask.objects.all().order_by('-id')
context['tasks'] = rows
for task in context['tasks']:
task.is_finished = int(task.is_finished)
task.parameter_config = del_sensitive_for_config(task.parameter_config)
project_id = get_and_check_scantask_project_id(task.id)
project = Project.objects.filter(id=project_id).first()
task.project_name = project.project_name
return context
def get(request, task_id):
scantask = ScanTask.objects.filter(id=task_id).first()
if not scantask.is_finished:
return JsonResponse({
"code":
403,
"status":
False,
"message":
"Task {} not finished.".format(task_id)
})
project_id = get_and_check_scantask_project_id(task_id)
scantaskresults = list(
get_and_check_scanresult(task_id).objects.filter(
scan_project_id=project_id, is_active=1).values())
return JsonResponse({
"code": 200,
"status": True,
"message": scantaskresults
})
def check_scantask(task_name, target_path, parameter_config, project_origin, project_des="", auto_yes=False):
s = ScanTask.objects.filter(task_name=task_name, target_path=target_path, parameter_config=parameter_config, is_finished=1).order_by("-id").first()
if s and not auto_yes:
logger.warning("[INIT] ScanTask for {} has been executed.".format(task_name))
logger.warning("[INIT] whether rescan Task {}?(Y/N) (Default N)".format(task_name))
if input().lower() != 'y':
logger.warning("[INIT] whether Show Last Scan Result?(Y/N) (Default Y)")
if input().lower() != 'n':
scan_id = s.id
table = PrettyTable(
['#', 'CVI', 'Rule(ID/Name)', 'Lang/CVE-id', 'Level', 'Target-File:Line-Number',
'Commit(Author)', 'Source Code Content', 'Analysis'])
table.align = 'l'
# check unconfirm
logger.warning("[INIT] whether Show Unconfirm Result?(Y/N) (Default Y)")
project_id = get_and_check_scantask_project_id(scan_id)
logger.info("[INIT] Now Project ID is {}".format(project_id))
if input().lower() != 'n':
srs = get_and_check_scanresult(scan_id).objects.filter(scan_project_id=project_id, is_active=True)
else:
srs = get_and_check_scanresult(scan_id).objects.filter(scan_project_id=project_id, is_active=True, is_unconfirm=False)
if srs:
logger.info("[MainThread] Last Scan id {} Result: ".format(scan_id))
for sr in srs:
rule = Rules.objects.filter(svid=sr.cvi_id).first()
rule_name = rule.rule_name
author = rule.author
level = VUL_LEVEL[rule.level]
row = [sr.result_id, sr.cvi_id, rule_name, sr.language, level, sr.vulfile_path,
author, sr.source_code, sr.result_type]
table.add_row(row)
# show Vuls Chain
ResultFlow = get_resultflow_class(scan_id)
rfs = ResultFlow.objects.filter(vul_id=sr.id)
logger.info("[Chain] Vul {}".format(sr.id))
for rf in rfs:
logger.info("[Chain] {}, {}, {}:{}".format(rf.node_type, rf.node_content, rf.node_path, rf.node_lineno))
show_context(rf.node_path, rf.node_lineno)
logger.info(
"[SCAN] ending\r\n -------------------------------------------------------------------------")
logger.info("[SCAN] Trigger Vulnerabilities ({vn})\r\n{table}".format(vn=len(srs), table=table))
# show New evil Function
nfs = NewEvilFunc.objects.filter(project_id=project_id, is_active=1)
if nfs:
table2 = PrettyTable(
['#', 'NewFunction', 'OriginFunction', 'Related Rules id'])
table2.align = 'l'
idy = 1
for nf in nfs:
row = [idy, nf.func_name, nf.origin_func_name, nf.svid]
table2.add_row(row)
idy += 1
logger.info("[MainThread] New evil Function list by NewCore:\r\n{table}".format(table=table2))
else:
logger.info("[MainThread] Last Scan id {} has no Result.".format(scan_id))
else:
s = ScanTask(task_name=task_name, target_path=target_path, parameter_config=parameter_config)
s.save()
# check and new project
check_and_new_project_id(scantask_id=s.id, task_name=task_name, project_origin=project_origin, project_des=project_des)
else:
s = ScanTask(task_name=task_name, target_path=target_path, parameter_config=parameter_config)
s.save()
# check and new project
check_and_new_project_id(s.id, task_name=task_name, project_origin=project_origin, project_des=project_des)
return s
def start(target, formatter, output, special_rules, a_sid=None, language=None, tamper_name=None, black_path=None, is_unconfirm=False, is_unprecom=False):
"""
Start CLI
:param black_path:
:param tamper_name:
:param language:
:param target: File, FOLDER, GIT
:param formatter:
:param output:
:param special_rules:
:param a_sid: all scan id
:return:
"""
global ast_object
# generate single scan id
s_sid = get_sid(target)
r = Running(a_sid)
data = (s_sid, target)
r.init_list(data=target)
r.list(data)
report = '?sid={a_sid}'.format(a_sid=a_sid)
d = r.status()
d['report'] = report
r.status(d)
task_id = a_sid
# 加载 kunlunmignore
load_kunlunmignore()
# parse target mode and output mode
pa = ParseArgs(target, formatter, output, special_rules, language, black_path, a_sid=None)
target_mode = pa.target_mode
output_mode = pa.output_mode
black_path_list = pa.black_path_list
# target directory
try:
target_directory = pa.target_directory(target_mode)
logger.info('[CLI] Target : {d}'.format(d=target_directory))
# static analyse files info
files, file_count, time_consume = Directory(target_directory, black_path_list).collect_files()
# vendor check
project_id = get_and_check_scantask_project_id(task_id)
Vendors(project_id, target_directory, files)
# detection main language and framework
if not language:
dt = Detection(target_directory, files)
main_language = dt.language
main_framework = dt.framework
else:
main_language = pa.language
main_framework = pa.language
logger.info('[CLI] [STATISTIC] Language: {l} Framework: {f}'.format(l=",".join(main_language), f=main_framework))
logger.info('[CLI] [STATISTIC] Files: {fc}, Extensions:{ec}, Consume: {tc}'.format(fc=file_count,
ec=len(files),
tc=time_consume))
if pa.special_rules is not None:
logger.info('[CLI] [SPECIAL-RULE] only scan used by {r}'.format(r=','.join(pa.special_rules)))
# Pretreatment ast object
ast_object.init_pre(target_directory, files)
ast_object.pre_ast_all(main_language, is_unprecom=is_unprecom)
# scan
scan(target_directory=target_directory, a_sid=a_sid, s_sid=s_sid, special_rules=pa.special_rules,
language=main_language, framework=main_framework, file_count=file_count, extension_count=len(files),
files=files, tamper_name=tamper_name, is_unconfirm=is_unconfirm)
except KeyboardInterrupt as e:
logger.error("[!] KeyboardInterrupt, exit...")
exit()
except Exception:
result = {
'code': 1002,
'msg': 'Exception'
}
Running(s_sid).data(result)
raise
# 输出写入文件
write_to_file(target=target, sid=s_sid, output_format=formatter, filename=output)
def display_result(scan_id, is_ask=False):
table = PrettyTable([
'#', 'CVI', 'Rule(ID/Name)', 'Lang/CVE-id', 'Level',
'Target-File:Line-Number', 'Commit(Author)', 'Source Code Content',
'Analysis'
])
table.align = 'l'
# check unconfirm
if is_ask:
logger.warning(
"[INIT] whether Show Unconfirm Result?(Y/N) (Default Y)")
project_id = get_and_check_scantask_project_id(scan_id)
if is_ask:
if input().lower() != 'n':
srs = get_and_check_scanresult(scan_id).objects.filter(
scan_project_id=project_id, is_active=True)
else:
srs = get_and_check_scanresult(scan_id).objects.filter(
scan_project_id=project_id, is_active=True, is_unconfirm=False)
else:
srs = get_and_check_scanresult(scan_id).objects.filter(
scan_project_id=project_id, is_active=True, is_unconfirm=False)
logger.info("[INIT] Project ID is {}".format(project_id))
if srs:
logger.info("[MainThread] Scan id {} Result: ".format(scan_id))
for sr in srs:
# for vendor scan
if sr.cvi_id == '9999':
vendor_vuls_id = int(sr.vulfile_path.split(':')[-1])
vv = VendorVulns.objects.filter(id=vendor_vuls_id).first()
if vv:
rule_name = vv.title
author = 'SCA'
level = VENDOR_VUL_LEVEL[int(vv.severity)]
# sr.source_code = vv.description
else:
rule_name = 'SCA Scan'
author = 'SCA'
level = VENDOR_VUL_LEVEL[1]
else:
rule = Rules.objects.filter(svid=sr.cvi_id).first()
rule_name = rule.rule_name
author = rule.author
level = VUL_LEVEL[rule.level]
row = [
sr.id, sr.cvi_id, rule_name, sr.language, level,
sr.vulfile_path, author, sr.source_code, sr.result_type
]
table.add_row(row)
# show Vuls Chain
ResultFlow = get_resultflow_class(scan_id)
rfs = ResultFlow.objects.filter(vul_id=sr.id)
logger.info("[Chain] Vul {}".format(sr.id))
for rf in rfs:
logger.info("[Chain] {}, {}, {}:{}".format(
rf.node_type, rf.node_content, rf.node_path,
rf.node_lineno))
try:
if author == 'SCA':
continue
if not show_context(rf.node_path, rf.node_lineno):
logger_console.info(rf.node_source)
except:
logger.error("[SCAN] Error: {}".format(
traceback.print_exc()))
continue
logger.info(
"[SCAN] ending\r\n -------------------------------------------------------------------------"
)
logger.info("[SCAN] Trigger Vulnerabilities ({vn})\r\n{table}".format(
vn=len(srs), table=table))
# show New evil Function
nfs = NewEvilFunc.objects.filter(project_id=project_id, is_active=1)
if nfs:
table2 = PrettyTable(
['#', 'NewFunction', 'OriginFunction', 'Related Rules id'])
table2.align = 'l'
idy = 1
for nf in nfs:
row = [idy, nf.func_name, nf.origin_func_name, nf.svid]
table2.add_row(row)
idy += 1
logger.info(
"[MainThread] New evil Function list by NewCore:\r\n{table}".
format(table=table2))
else:
logger.info("[MainThread] Scan id {} has no Result.".format(scan_id))
