def tasklog(req, task_id):
task = ScanTask.objects.filter(id=task_id).first()
visit_token = ""
if 'token' in req.GET:
visit_token = req.GET['token']
# check task是否存在
if not task:
return redirect("dashboard:tasks_list")
# check task 的状态，只有完成才能继续
if not task.is_finished:
return HttpResponse(
"Ooooops, Maybe this task still in progress or has error, you can't view the log..."
)
project_id = get_and_check_scantask_project_id(task_id)
srts = get_and_check_scanresult(task_id).objects.filter(
scan_project_id=project_id)
nefs = NewEvilFunc.objects.filter(project_id=project_id)
ResultFlow = get_resultflow_class(task_id)
rfs = ResultFlow.objects.all()
task.parameter_config = " ".join(ast.literal_eval(
task.parameter_config)).replace('\\', '/')
resultflowdict = {}
for rf in rfs:
if rf.vul_id not in resultflowdict:
resultflowdict[rf.vul_id] = {
'id': rf.vul_id,
'flow': [],
}
rfdict = {
'type': rf.node_type,
'content': rf.node_content,
'path': rf.node_path,
'lineno': rf.node_lineno,
'details': rf.node_source
}
resultflowdict[rf.vul_id]['flow'].append(rfdict)
# 扫描结果
data = {
"task": task,
"taskresults": srts,
"newevilfuncs": nefs,
"resultflowdict": resultflowdict,
'visit_token': visit_token
}
return render(req, 'backend/tasklog.html', data)
def tasklog(req, task_id):
task = ScanTask.objects.filter(id=task_id).first()
# check task是否存在
if not task:
return redirect("dashboard:tasks_list")
# check task 的状态，只有完成才能继续
if not task.is_finished:
return HttpResponse(
"Ooooops, Maybe this task still in progress or has error, you can't view the log..."
)
srts = ScanResultTask.objects.filter(scan_task_id=task_id)
nefs = NewEvilFunc.objects.filter(scan_task_id=task_id)
ResultFlow = get_resultflow_class(task_id)
rfs = ResultFlow.objects.all()
resultflowdict = {}
for rf in rfs:
if rf.vul_id not in resultflowdict:
resultflowdict[rf.vul_id] = {
'id': rf.vul_id,
'flow': [],
}
rfdict = {
'type': rf.node_type,
'content': rf.node_content,
'path': rf.node_path,
'lineno': rf.node_lineno,
'details': show_context(rf.node_path, rf.node_lineno)
}
resultflowdict[rf.vul_id]['flow'].append(rfdict)
# 扫描结果
data = {
"task": task,
"taskresults": srts,
"newevilfuncs": nefs,
"resultflowdict": resultflowdict,
}
return render(req, 'backend/tasklog.html', data)
def check_and_save_result(task_id, language, vendor_name, vendor_version):
"""
检查并保存结果。
:param vendor_name:
:param vendor_version:
:return:
"""
vvs = get_vendor_vul_by_name(vendor_name.strip())
# vendor_version = abstract_version(vendor_version)
result_list = []
for vv in vvs:
vv_affect_version = vv.affected_versions.split(',')
if not vendor_version or vendor_version in vv_affect_version:
if task_id:
sr = check_update_or_new_scanresult(
scan_task_id=task_id,
cvi_id=VENDOR_CVIID,
language=language,
vulfile_path="VendorVul:{}".format(vv.id),
source_code="{}".format(vv.reference[:180]),
result_type=vendor_source_match,
is_unconfirm=False,
is_active=True
)
# save into get_resultflow_class
ResultFlow = get_resultflow_class(int(task_id))
if sr:
node_source = vv.description
rf = ResultFlow(vul_id=sr.id, node_type='sca_scan',
node_content=vv.title, node_path=vv.reference[:280],
node_source=node_source, node_lineno=0)
rf.save()
else:
result_list.append(vv)
return result_list
def get(request, task_id):
scantask = ScanTask.objects.filter(id=task_id).first()
if not scantask.is_finished:
return JsonResponse({
"code":
403,
"status":
False,
"message":
"Task {} not finished.".format(task_id)
})
ResultFlow = get_resultflow_class(int(task_id))
rfs = ResultFlow.objects.filter().order_by('vul_id')
resultflow_list = list(rfs.values())
return JsonResponse({
"code": 200,
"status": True,
"message": resultflow_list
})
def check_scantask(task_name, target_path, parameter_config, project_origin, project_des="", auto_yes=False):
s = ScanTask.objects.filter(task_name=task_name, target_path=target_path, parameter_config=parameter_config, is_finished=1).order_by("-id").first()
if s and not auto_yes:
logger.warning("[INIT] ScanTask for {} has been executed.".format(task_name))
logger.warning("[INIT] whether rescan Task {}?(Y/N) (Default N)".format(task_name))
if input().lower() != 'y':
logger.warning("[INIT] whether Show Last Scan Result?(Y/N) (Default Y)")
if input().lower() != 'n':
scan_id = s.id
table = PrettyTable(
['#', 'CVI', 'Rule(ID/Name)', 'Lang/CVE-id', 'Level', 'Target-File:Line-Number',
'Commit(Author)', 'Source Code Content', 'Analysis'])
table.align = 'l'
# check unconfirm
logger.warning("[INIT] whether Show Unconfirm Result?(Y/N) (Default Y)")
project_id = get_and_check_scantask_project_id(scan_id)
logger.info("[INIT] Now Project ID is {}".format(project_id))
if input().lower() != 'n':
srs = get_and_check_scanresult(scan_id).objects.filter(scan_project_id=project_id, is_active=True)
else:
srs = get_and_check_scanresult(scan_id).objects.filter(scan_project_id=project_id, is_active=True, is_unconfirm=False)
if srs:
logger.info("[MainThread] Last Scan id {} Result: ".format(scan_id))
for sr in srs:
rule = Rules.objects.filter(svid=sr.cvi_id).first()
rule_name = rule.rule_name
author = rule.author
level = VUL_LEVEL[rule.level]
row = [sr.result_id, sr.cvi_id, rule_name, sr.language, level, sr.vulfile_path,
author, sr.source_code, sr.result_type]
table.add_row(row)
# show Vuls Chain
ResultFlow = get_resultflow_class(scan_id)
rfs = ResultFlow.objects.filter(vul_id=sr.id)
logger.info("[Chain] Vul {}".format(sr.id))
for rf in rfs:
logger.info("[Chain] {}, {}, {}:{}".format(rf.node_type, rf.node_content, rf.node_path, rf.node_lineno))
show_context(rf.node_path, rf.node_lineno)
logger.info(
"[SCAN] ending\r\n -------------------------------------------------------------------------")
logger.info("[SCAN] Trigger Vulnerabilities ({vn})\r\n{table}".format(vn=len(srs), table=table))
# show New evil Function
nfs = NewEvilFunc.objects.filter(project_id=project_id, is_active=1)
if nfs:
table2 = PrettyTable(
['#', 'NewFunction', 'OriginFunction', 'Related Rules id'])
table2.align = 'l'
idy = 1
for nf in nfs:
row = [idy, nf.func_name, nf.origin_func_name, nf.svid]
table2.add_row(row)
idy += 1
logger.info("[MainThread] New evil Function list by NewCore:\r\n{table}".format(table=table2))
else:
logger.info("[MainThread] Last Scan id {} has no Result.".format(scan_id))
else:
s = ScanTask(task_name=task_name, target_path=target_path, parameter_config=parameter_config)
s.save()
# check and new project
check_and_new_project_id(scantask_id=s.id, task_name=task_name, project_origin=project_origin, project_des=project_des)
else:
s = ScanTask(task_name=task_name, target_path=target_path, parameter_config=parameter_config)
s.save()
# check and new project
check_and_new_project_id(s.id, task_name=task_name, project_origin=project_origin, project_des=project_des)
return s
def scan(target_directory, a_sid=None, s_sid=None, special_rules=None, language=None, framework=None, file_count=0,
extension_count=0, files=None, tamper_name=None, is_unconfirm=False):
r = Rule(language)
vulnerabilities = r.vulnerabilities
rules = r.rules(special_rules)
find_vulnerabilities = []
newcore_function_list = {}
def store(result):
if result is not None and isinstance(result, list) is True:
for res in result:
res.file_path = res.file_path
find_vulnerabilities.append(res)
else:
logger.debug('[SCAN] [STORE] Not found vulnerabilities on this rule!')
async def start_scan(target_directory, rule, files, language, tamper_name):
result = scan_single(target_directory, rule, files, language, tamper_name, is_unconfirm, newcore_function_list)
store(result)
if len(rules) == 0:
logger.critical('no rules!')
return False
logger.info('[PUSH] {rc} Rules'.format(rc=len(rules)))
push_rules = []
scan_list = []
for idx, single_rule in enumerate(sorted(rules.keys())):
# init rule class
r = getattr(rules[single_rule], single_rule)
rule = r()
if rule.status is False and len(rules) != 1:
logger.info('[CVI_{cvi}] [STATUS] OFF, CONTINUE...'.format(cvi=rule.svid))
continue
# SR(Single Rule)
logger.debug("""[PUSH] [CVI_{cvi}] {idx}.{vulnerability}({language})""".format(
cvi=rule.svid,
idx=idx,
vulnerability=rule.vulnerability,
language=rule.language
))
# result = scan_single(target_directory, rule, files, language, tamper_name)
scan_list.append(start_scan(target_directory, rule, files, language, tamper_name))
# store(result)
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.gather(*scan_list))
loop.stop()
# print
data = []
data2 = []
table = PrettyTable(
['#', 'CVI', 'Rule(ID/Name)', 'Lang/CVE-id', 'Target-File:Line-Number',
'Commit(Author)', 'Source Code Content', 'Analysis'])
table.align = 'l'
trigger_rules = []
for idx, x in enumerate(find_vulnerabilities):
trigger = '{fp}:{ln}'.format(fp=x.file_path, ln=x.line_number)
commit = u'@{author}'.format(author=x.commit_author)
try:
code_content = x.code_content[:50].strip()
except AttributeError as e:
code_content = x.code_content.decode('utf-8')[:100].strip()
row = [idx + 1, x.id, x.rule_name, x.language, trigger, commit,
code_content.replace('\r\n', ' ').replace('\n', ' '), x.analysis]
row2 = [idx + 1, x.chain]
is_unconfirm_result = False
if "unconfirmed" in x.analysis.lower():
is_unconfirm_result = True
# save to database
sr = ScanResultTask(scan_task_id=a_sid, result_id=idx + 1, cvi_id=x.id, language=x.language,
vulfile_path=trigger, source_code=code_content.replace('\r\n', ' ').replace('\n', ' '),
result_type=x.analysis, is_unconfirm=is_unconfirm_result)
sr.save()
for chain in x.chain:
if type(chain) == tuple:
ResultFlow = get_resultflow_class(int(a_sid))
rf = ResultFlow(vul_id=idx + 1, node_type=chain[0], node_content=chain[1],
node_path=chain[2], node_lineno=chain[3])
rf.save()
data.append(row)
data2.append(row2)
table.add_row(row)
if x.id not in trigger_rules:
logger.debug(' > trigger rule (CVI-{cvi})'.format(cvi=x.id))
trigger_rules.append(x.id)
# clear
x.chain = ""
diff_rules = list(set(push_rules) - set(trigger_rules))
vn = len(find_vulnerabilities)
if vn == 0:
logger.info('[SCAN] Not found vulnerability!')
else:
logger.info("[SCAN] Trigger Rules: {tr} Vulnerabilities ({vn})\r\n{table}".format(tr=len(trigger_rules),
vn=len(find_vulnerabilities),
table=table))
# 输出chain for all
logger.info("[SCAN] Vulnerabilities Chain list: ")
for d in data2:
logger.info("[SCAN] Vul {}".format(d[0]))
for c in d[1]:
logger.info("[Chain] {}".format(c))
if type(c) is not tuple:
continue
show_context(c[2], c[3])
logger.info("[SCAN] ending\r\n" + '-' * (shutil.get_terminal_size().columns - 16))
if len(diff_rules) > 0:
logger.info(
'[SCAN] Not Trigger Rules ({l}): {r}'.format(l=len(diff_rules), r=','.join(diff_rules)))
# show detail about newcore function list
table2 = PrettyTable(
['#', 'NewFunction', 'OriginFunction', 'Related Rules id'])
table2.align = 'l'
idy = 0
for new_function_name in newcore_function_list:
# add new evil func in database
for svid in newcore_function_list[new_function_name]["svid"]:
if new_function_name:
nf = NewEvilFunc(svid=svid, scan_task_id=get_scan_id(), func_name=new_function_name,
origin_func_name=newcore_function_list[new_function_name]["origin_func_name"])
nf.save()
table2.add_row([idy + 1, new_function_name, newcore_function_list[new_function_name]["origin_func_name"], newcore_function_list[new_function_name]["svid"]])
idy += 1
if len(newcore_function_list) > 0:
logger.info("[SCAN] New evil Function list by NewCore:\r\n{}".format(table2))
# completed running data
if s_sid is not None:
Running(s_sid).data({
'code': 1001,
'msg': 'scan finished',
'result': {
'vulnerabilities': [x.__dict__ for x in find_vulnerabilities],
'language': ",".join(language),
'framework': framework,
'extension': extension_count,
'file': file_count,
'push_rules': len(rules),
'trigger_rules': len(trigger_rules),
'target_directory': target_directory
}
})
return True
def display_result(scan_id, is_ask=False):
table = PrettyTable([
'#', 'CVI', 'Rule(ID/Name)', 'Lang/CVE-id', 'Level',
'Target-File:Line-Number', 'Commit(Author)', 'Source Code Content',
'Analysis'
])
table.align = 'l'
# check unconfirm
if is_ask:
logger.warning(
"[INIT] whether Show Unconfirm Result?(Y/N) (Default Y)")
project_id = get_and_check_scantask_project_id(scan_id)
if is_ask:
if input().lower() != 'n':
srs = get_and_check_scanresult(scan_id).objects.filter(
scan_project_id=project_id, is_active=True)
else:
srs = get_and_check_scanresult(scan_id).objects.filter(
scan_project_id=project_id, is_active=True, is_unconfirm=False)
else:
srs = get_and_check_scanresult(scan_id).objects.filter(
scan_project_id=project_id, is_active=True, is_unconfirm=False)
logger.info("[INIT] Project ID is {}".format(project_id))
if srs:
logger.info("[MainThread] Scan id {} Result: ".format(scan_id))
for sr in srs:
# for vendor scan
if sr.cvi_id == '9999':
vendor_vuls_id = int(sr.vulfile_path.split(':')[-1])
vv = VendorVulns.objects.filter(id=vendor_vuls_id).first()
if vv:
rule_name = vv.title
author = 'SCA'
level = VENDOR_VUL_LEVEL[int(vv.severity)]
# sr.source_code = vv.description
else:
rule_name = 'SCA Scan'
author = 'SCA'
level = VENDOR_VUL_LEVEL[1]
else:
rule = Rules.objects.filter(svid=sr.cvi_id).first()
rule_name = rule.rule_name
author = rule.author
level = VUL_LEVEL[rule.level]
row = [
sr.id, sr.cvi_id, rule_name, sr.language, level,
sr.vulfile_path, author, sr.source_code, sr.result_type
]
table.add_row(row)
# show Vuls Chain
ResultFlow = get_resultflow_class(scan_id)
rfs = ResultFlow.objects.filter(vul_id=sr.id)
logger.info("[Chain] Vul {}".format(sr.id))
for rf in rfs:
logger.info("[Chain] {}, {}, {}:{}".format(
rf.node_type, rf.node_content, rf.node_path,
rf.node_lineno))
try:
if author == 'SCA':
continue
if not show_context(rf.node_path, rf.node_lineno):
logger_console.info(rf.node_source)
except:
logger.error("[SCAN] Error: {}".format(
traceback.print_exc()))
continue
logger.info(
"[SCAN] ending\r\n -------------------------------------------------------------------------"
)
logger.info("[SCAN] Trigger Vulnerabilities ({vn})\r\n{table}".format(
vn=len(srs), table=table))
# show New evil Function
nfs = NewEvilFunc.objects.filter(project_id=project_id, is_active=1)
if nfs:
table2 = PrettyTable(
['#', 'NewFunction', 'OriginFunction', 'Related Rules id'])
table2.align = 'l'
idy = 1
for nf in nfs:
row = [idy, nf.func_name, nf.origin_func_name, nf.svid]
table2.add_row(row)
idy += 1
logger.info(
"[MainThread] New evil Function list by NewCore:\r\n{table}".
format(table=table2))
else:
logger.info("[MainThread] Scan id {} has no Result.".format(scan_id))