def get_info(self, **kwargs):
"""
:param kwargs:
:return:
"""
origin_file = strip(kwargs.get('origin_file'))
match_content = strip(kwargs.get('match_content'))
evidence_start_line_offset = parse_int(
kwargs.get('evidence_start_line_offset'), -1)
evidence_count = parse_int(kwargs.get('evidence_count'), 5)
result = {
'start_line': 0,
'end_line': 0,
'code_example': '',
}
if os.path.isfile(origin_file) and match_content:
contents = []
with open(origin_file, 'rb') as fp:
contents = fp.readlines()
result['start_line'] = get_str_line(match_content, contents)
if result['start_line'] > 0:
result['end_line'] = result['start_line'] + 1
result['code_example'] = get_line_content(
file=contents,
start_line=result['start_line'] +
int(evidence_start_line_offset),
count=int(evidence_count))
if result['code_example']:
result['code_example'] = ''.join(result['code_example'])
return result
def verify(self, **kwargs):
"""
:param kwargs:
:return:
"""
reference_value = strip(kwargs.get("reference_value"))
scan_path = strip(kwargs.get("scan_path"))
status, result = False, ''
if os.path.isfile(reference_value):
reference_value = '{0}/'.format(os.path.dirname(reference_value))
if os.path.isdir(reference_value):
if scan_path:
work_dir = reference_value.replace(scan_path, '')
else:
work_dir = reference_value
for r in self._regex:
_result = r.search(work_dir)
if _result:
result = _result.group(0)
status = True
break
return status, result
def verify(self, **kwargs):
"""
:param kwargs:
:return:
"""
component_name = strip(kwargs.get('component_name'))
version = strip(kwargs.get('version'))
group_id = strip(kwargs.get('group_id'), None)
status, result = False, ''
if self._match_type and self._match_type.lower() != 'name':
if self._match_content == group_id:
for r in self._regex:
_result = r.search(version)
if _result:
result = _result.group(0)
status = True
break
else:
if component_name and self._match_content == component_name:
for r in self._regex:
_result = r.search(version)
if _result:
result = _result.group(0)
status = True
break
return status, result
def verify(self, **kwargs):
"""
:param kwargs:
:return:
"""
reference_value = strip(kwargs.get('file'))
status, result = False, ''
if os.path.isfile(reference_value) and isfunction(self._func):
status, result = self._func(file=reference_value)
return status, result
def test_strip(self):
self.assertEqual(strip(b' 1 '), b'1')
self.assertEqual(strip(" "), '')
self.assertEqual(strip(1), 1)
self.assertEqual(strip([]), [])
self.assertEqual(strip(" Admin ", "lower"), "admin")
self.assertEqual(strip(" Admin ", "upper"), "ADMIN")
def verify(self, **kwargs):
"""
:param kwargs:
:return:
"""
reference_value = strip(kwargs.get('reference_value'))
status, result = False, ''
if os.path.isfile(reference_value):
for r in self._regex:
_result = r.search(reference_value)
if _result:
result = _result.group(0)
status = True
break
return status, result
def verify_result(self, **kwargs):
"""
:param kwargs:
:return:
"""
vuln = strip(kwargs.get('vuln'))
status, result = self._func(
code_path=vuln.project_scan_path,
title=vuln.title,
file_name=vuln.file,
start_line=vuln.start_line,
end_line=vuln.end_line,
file_content=vuln.code_example,
)
return status, result
def __init__(self, **kwargs):
self.project_scan_path = kwargs.get('project_scan_path')
self._vuln = {
'rule_key':
strip(kwargs.get("rule_key"), ''),
'risk_id':
strip(kwargs.get("risk_id"), ''),
'category':
parse_category(strip(kwargs.get("category", 'Vulnerability'))),
'title':
strip(kwargs.get("title"), ''),
'file':
strip(kwargs.get("file"), ''),
'author':
strip(kwargs.get("author"), ''),
'author_email':
strip(kwargs.get("author_email"), ''),
'hash':
strip(kwargs.get("hash"), ''), # last commit
'start_line':
parse_int(kwargs.get("start_line"), 0),
'end_line':
parse_int(kwargs.get("end_line"), 0),
'report':
strip(kwargs.get("report"), ''),
'code_example':
kwargs.get("code_example", ''),
'is_false_positive':
parse_bool(kwargs.get("is_false_positive", False)),
'whitelist_rule_id':
kwargs.get("whitelist_rule_id", ''),
'evidence_content':
kwargs.get("evidence_content", ''),
'engine':
kwargs.get("engine", ''),
}
def __kill_sonar_scanner_pid(self, project):
"""
:param project:
:return:
"""
try:
project.logger.info(
'[SonarScanner] Get the pid number of the sonar-scanner ...')
get_sonar_scanner_cmd = 'ps -ef | grep "cd ' + project.scan_path + ' \&\& ' + self._sonar_scanner_path + '"|grep -v grep|awk \'{print($2)}\''
project.logger.debug(
'[SonarScanner] Query pid command: [{0}]'.format(
get_sonar_scanner_cmd))
pid, _ = exec_cmd(get_sonar_scanner_cmd)
pid = parse_int_or_str(pid) or ''
project.logger.debug('[SonarScanner] Get PID: [{0}]'.format(pid))
if pid:
sub_pid_cmd = 'ps -ef | grep "Dproject.home=' + project.scan_path + '"|grep -v grep|awk \'{print($2)}\''
sub_pid, _ = exec_cmd(sub_pid_cmd)
sub_pid = strip(sub_pid)
if '\n' in sub_pid.decode("utf-8"):
sub_pid = [
parse_int_or_str(_p)
for _p in sub_pid.decode("utf-8").split("\n") if _p
]
else:
sub_pid = parse_int_or_str(sub_pid) or ''
kill_sonar_scanner_cmd = 'kill -9 {0} {1}'.format(pid, sub_pid)
status, _ = exec_cmd(kill_sonar_scanner_cmd)
if _:
project.logger.warning(
'[SonarScanner] The termination of the sonar-scanner '
'process failed with the command: [{0}]'.format(
kill_sonar_scanner_cmd))
else:
project.logger.info(
'[SonarScanner] Process [{0},{1}] is terminated.'.
format(pid, sub_pid))
except:
pass
def verify(self, **kwargs):
"""
:param kwargs:
:return:
"""
reference_value = strip(kwargs.get('reference_value'))
status, result = False, ''
if os.path.isfile(reference_value):
file_name, file_ext = os.path.splitext(reference_value)
if self._file_ext and file_ext.lower() not in self._file_ext:
msg = "The file type can only be {0}, the current type: [{1}].".format(
self._file_ext, file_ext)
raise FileLimitSuffixException(msg)
file_size = round(os.path.getsize(reference_value), 2) # Bytes
if self._file_size != 0 and file_size > self._file_size:
msg = "[SKIP] file: {0}, size: {1}KB, limit size: {2}KB".format(
reference_value, file_size, self._file_size)
raise FileLimitSizeException(msg)
with open(reference_value, "rb") as fp:
s = fp.read()
if isinstance(s, bytes):
try:
content = s.decode('utf-8')
except: # There will be problems with the binary
content = str(s)
else:
content = s
for r in self._regex:
_result = r.search(content)
if _result:
result = _result.group(0)
status = True
break
return status, result
def verify_result(self, **kwargs):
"""
:param kwargs:
:return:
"""
content = strip(kwargs.get('content'))
status, result = False, ''
if isinstance(content, bytes):
try:
content = content.decode('utf-8')
except: # There will be problems with the binary
content = str(content)
for r in self._regex:
_result = r.search(content)
if _result:
result = _result.group(0)
status = True
break
return status, result
def __init__(self, **kwargs):
self._id = parse_int(kwargs.get('id'))
self._name = strip(kwargs.get('name'), None)
self._description = strip(kwargs.get('description'), '')
self._key = strip(kwargs.get('key'), 'lower')
self._risk_id = parse_int(kwargs.get('risk_id'))
self._risk_name = strip(kwargs.get('risk_name'))
self._category_id = parse_int(kwargs.get('category_id'))
self._category_name = strip(kwargs.get('category_name'), None)
regex = strip(kwargs.get('regex'), None)
regex_flag = strip(kwargs.get('regex_flag'), None)
self._file_ext = strip(kwargs.get('file_ext'), None) # File extension
self._file_size = parse_int(kwargs.get('size'), 0) # File size
self._match_type = strip(kwargs.get('match_type'),
None) # match type: groupId、name
self._match_ext = strip(kwargs.get('match_ext'),
None) # match type: groupId、name
self._match_content = strip(kwargs.get('match_content'),
None) # match content
self._func = kwargs.get('func')
if self._file_ext:
file_ext = []
self._file_ext = self._file_ext.split(',')
for item in self._file_ext:
ext = item.lower().strip()
file_ext.append(ext[ext.find("."):])
self._file_ext = file_ext
self._flag = None
self._regex = []
if regex_flag and 'I' in regex_flag and 'M' in regex_flag:
self._flag = re.I | re.M
elif regex_flag and 'M' in regex_flag:
self._flag = re.M
elif regex_flag and 'I' in regex_flag:
self._flag = re.I
if regex:
for r in regex.split("\n"):
if " ###" in r:
expr = r[:r.find(" ###")]
expr = expr.strip()
else:
expr = r.strip()
if self._flag:
self._regex.append(re.compile(expr, self._flag))
else:
self._regex.append(re.compile(expr))
def __init__(self, **kwargs):
"""
:param kwargs:
"""
self._logger = None
self._paths_log = ''
self._log_file = ''
self._remote_paths = ''
self._paths_project_origin = None
self._paths_project_scan = None
self._branch_commit = None
self._git = None
self._rotate_handler = None
self._statistics = {
'depends': [],
'file': [],
'total': 0,
'size': 0, # KB
'language': '',
'risk': {},
}
# scan
self._threads = parse_int_or_str(parse_int(kwargs.get('threads'), 20))
self._template = strip(kwargs.get('template'), display_type='lower')
self._template_full_path = strip(kwargs.get('template_full_path'))
self._work_dir = kwargs.get('work_dir') or '/data/seecode'
self._log_level = strip(kwargs.get('log_level')) or 'info'
self._task_id = parse_int(kwargs.get('task_id'))
self._project_name = strip(kwargs.get('project_name'))
self._project_branch = strip(kwargs.get('project_branch'))
self._project_ssh = strip(kwargs.get('project_ssh'))
self._project_web = strip(kwargs.get('project_web'))
self._project_local_path = strip(kwargs.get('project_local_path'))
self._project_type = strip(kwargs.get('project_type') or 'offline',
display_type='lower')
self._project_storage_type = strip(kwargs.get('project_storage_type'),
display_type='lower')
self._project_file_origin_name = strip(
kwargs.get('project_file_origin_name'))
self._project_file_hash = strip(kwargs.get('project_file_hash'))
self._group_name = strip(kwargs.get('group_name'))
self._group_key = strip(kwargs.get('group_key'))
# 强制更新代码，必须为线上项目
self._force_sync_code = parse_bool(kwargs.get('force_sync_code',
False))
# 强制扫描项目
self._force_project_scan = parse_bool(
kwargs.get('force_project_scan', False))
# 同步漏洞到 SeeCode Audit 服务端
self._sync_vuln_to_server = parse_bool(
kwargs.get('sync_vuln_to_server', False))
# 取证偏移
self._evidence_start_line_offset = parse_int(
kwargs.get('evidence_start_line_offset'), -1)
# 合并相同组件漏洞（依赖文件包含）
self._component_same_vuln_merge_enable = parse_bool(
kwargs.get('component_same_vuln_merge_enable', True))
self._evidence_count = parse_int(kwargs.get('evidence_count'), 5)
self._result_file = strip(kwargs.get('result_file'))
self._result_format = strip(kwargs.get('result_format'))
self._is_cli_call = parse_bool(kwargs.get('is_cli_call'))
self._issue_statistic_type = ['vulnerability']
self._send_size = parse_int_or_str(strip(kwargs.get('send_size', 50)))
# distributed
self._distributed = kwargs.get('distributed', conf.distributed)
self.storage = {}
self.__check_scan_parameters()
self.__init_work_dir()
self.__init_logger()
self.cp = CopyCMD()
self.find = FindCMD()
# check scan template
profile = os.path.join(paths.ROOT_PROFILE_PATH,
'{0}.xml'.format(self._template))
stat = os.stat(profile)
if not all((kb, kb.profiles)):
init()
if self._template_full_path and os.path.isfile(
self._template_full_path):
load_scan_template(template_file_path=self._template_full_path)
else:
load_scan_template()
kb.profiles_time[self._template] = stat.st_ctime
if kb.profiles_time[self._template] != stat.st_ctime:
load_scan_template()
kb.profiles_time[self._template] = stat.st_ctime
# result
if self.key not in kb.result:
kb.result[self.key] = {}
if not self._result_file:
self._result_file = '{0}.json'.format(self._task_id
or self._project_name)
if not self._result_file.startswith('/'):
self._result_file = os.path.join(self._paths_log,
self._result_file)
# remote storage
self.__init_distributed()
self._remote_log = os.path.join(self._remote_paths,
os.path.basename(self._log_file))
self._remote_result = os.path.join(self._remote_paths,
os.path.basename(self._result_file))
# scan task
if isinstance(self._task_id, int):
self._task_status = TaskStatus(task_id=self._task_id,
server=conf.server.domain,
server_token=conf.server.token,
logger=self.logger)
if self._sync_vuln_to_server:
self._task_status.update_initialization(
scan_template=self._template,
scan_template_version=kb.profiles[self._template].version,
log_path=self._remote_log,
)
else:
self._task_status = None
if self._template not in kb.profiles:
msg = "[ScanProject] Didn't find '{0}' template.".format(
self._template)
if self._task_status and self._sync_vuln_to_server:
self.update_scan_failed_status(msg)
raise ScanTemplateNotFound(msg)