def __init__(self, **kwargs):
token = kwargs.get('token', None)
self.logger = kwargs.get("logger", _logger)
self.sonar_server = kwargs.get('sonar_server', None)
self.http_timeout_retry = parse_int(kwargs.get('http_timeout_retry'),
3)
self.http_failed_retry = parse_int(kwargs.get('http_failed_retry'), 3)
self.http_timeout = parse_int(kwargs.get('http_timeout'), 10)
self.session = requests.Session()
user_password = '******'.format(token, '')
headers = {
"Authorization":
'Basic {0}'.format(
base64.b64encode(
user_password.encode('utf-8')).decode("utf-8")),
"User-Agent":
'{0}'.format(USER_AGENT),
}
if not self.sonar_server.endswith("/"):
self.sonar_server = '{0}/'.format(self.sonar_server)
self.session.headers.update(headers)
if not self.__validate_authentication():
msg = "The authentication failed. Please check if the token({0}) is correct.".format(
token)
raise SonarQubeAuthenticationFailed(msg)
def get_info(self, **kwargs):
"""
:param kwargs:
:return:
"""
origin_file = strip(kwargs.get('origin_file'))
match_content = strip(kwargs.get('match_content'))
evidence_start_line_offset = parse_int(
kwargs.get('evidence_start_line_offset'), -1)
evidence_count = parse_int(kwargs.get('evidence_count'), 5)
result = {
'start_line': 0,
'end_line': 0,
'code_example': '',
}
if os.path.isfile(origin_file) and match_content:
contents = []
with open(origin_file, 'rb') as fp:
contents = fp.readlines()
result['start_line'] = get_str_line(match_content, contents)
if result['start_line'] > 0:
result['end_line'] = result['start_line'] + 1
result['code_example'] = get_line_content(
file=contents,
start_line=result['start_line'] +
int(evidence_start_line_offset),
count=int(evidence_count))
if result['code_example']:
result['code_example'] = ''.join(result['code_example'])
return result
def test_parse_int(self):
# test 1
list_of_var1 = ['1', 0x1, b'1']
for i in list_of_var1:
self.assertTrue(parse_int(i) == 1)
list_of_var2 = ['abcd', type(int)]
for i in list_of_var2:
self.assertTrue(parse_int(i) == 0)
def __init__(self, **kwargs):
self._id = parse_int(kwargs.get('id'))
self._name = strip(kwargs.get('name'), None)
self._description = strip(kwargs.get('description'), '')
self._key = strip(kwargs.get('key'), 'lower')
self._risk_id = parse_int(kwargs.get('risk_id'))
self._risk_name = strip(kwargs.get('risk_name'))
self._category_id = parse_int(kwargs.get('category_id'))
self._category_name = strip(kwargs.get('category_name'), None)
regex = strip(kwargs.get('regex'), None)
regex_flag = strip(kwargs.get('regex_flag'), None)
self._file_ext = strip(kwargs.get('file_ext'), None) # File extension
self._file_size = parse_int(kwargs.get('size'), 0) # File size
self._match_type = strip(kwargs.get('match_type'),
None) # match type: groupId、name
self._match_ext = strip(kwargs.get('match_ext'),
None) # match type: groupId、name
self._match_content = strip(kwargs.get('match_content'),
None) # match content
self._func = kwargs.get('func')
if self._file_ext:
file_ext = []
self._file_ext = self._file_ext.split(',')
for item in self._file_ext:
ext = item.lower().strip()
file_ext.append(ext[ext.find("."):])
self._file_ext = file_ext
self._flag = None
self._regex = []
if regex_flag and 'I' in regex_flag and 'M' in regex_flag:
self._flag = re.I | re.M
elif regex_flag and 'M' in regex_flag:
self._flag = re.M
elif regex_flag and 'I' in regex_flag:
self._flag = re.I
if regex:
for r in regex.split("\n"):
if " ###" in r:
expr = r[:r.find(" ###")]
expr = expr.strip()
else:
expr = r.strip()
if self._flag:
self._regex.append(re.compile(expr, self._flag))
else:
self._regex.append(re.compile(expr))
def __init__(self, **kwargs):
"""
:param kwargs:
"""
logger = kwargs.get("logger", _logger)
self.transport_encryption = parse_bool(
kwargs.get("transport_encryption",
False)) # conf.server.transport_encryption
self.timeout = parse_int(kwargs.get("timeout"),
10) # conf.http.timeout
timeout_try = parse_int(kwargs.get("timeout_try"),
3) # conf.http.timeout_try)
failed_try = parse_int(kwargs.get("failed_try"),
3) # conf.http.failed_try)
headers = kwargs.get("headers", {}) # conf.http.headers
proxies = kwargs.get("proxies", {}) # conf.http.proxies
server_token = kwargs.get("server_token", None) # conf.server.token
self.not_try_status_code = (401, 404)
self.headers = {
"Authorization": 'Token {0}'.format(server_token),
"Content-Type": 'application/json',
"User-Agent": '{0}'.format(USER_AGENT),
}
self.headers.update(headers)
self.session = requests.Session()
self.session.headers.update(self.headers)
if proxies:
self.session.proxies = proxies # TODO test
self.timeout_try = 1
self.failed_try = 1
if isinstance(timeout_try, int) and timeout_try > self.timeout_try:
self.timeout_try = timeout_try
if isinstance(failed_try, int) and failed_try > self.failed_try:
self.failed_try = failed_try
self._logger = logger
def __init_distributed(self):
"""
:return:
"""
debug = False
if self._distributed:
try:
if self._log_level == "debug":
debug = True
if 'ftp' == self._project_storage_type.lower().strip():
try:
self.storage['ftp'] = FTPWork(
host=self._distributed['ftp']['host'],
port=parse_int(self._distributed['ftp']['port'],
21),
username=self._distributed['ftp']['username'],
password=self._distributed['ftp']['password'],
debug=debug,
)
self._remote_paths = os.path.join(
self._distributed['ftp']['path'], 'tasks',
'{0}'.format(self._task_id or self._project_name))
self.storage[self._project_storage_type].make_dir(
self._remote_paths)
self._logger.info(
'Start creating ftp upload directory: {0}.'.format(
self._remote_paths))
except Exception as ex:
msg = "[ScanProject] Failed to connect to the FTP service. " \
"Please check if the service is available."
raise Exception(msg)
elif 'aws' == self._project_storage_type.lower().strip():
self.storage['aws'] = {}
pass # TODO
elif 'local' == self._project_storage_type.lower().strip():
pass
else:
msg = 'Does not support "{0}" storage mode.'.format(
self._project_storage_type)
self._logger.warning(msg)
except Exception as ex:
import traceback
traceback.print_exc()
msg = "[ScanProject] The distributed parameter configuration may have an error and " \
"cannot connect to the service. {0}".format(str(ex))
raise DistributedConfigurationInvalid(msg)
elif self._project_local_path: # local scan
pass
else:
msg = "[ScanProject] Missing 'distributed' important scan parameters."
raise MissingImportantScanParameters(msg)
def __init__(self, **kwargs):
self.project_scan_path = kwargs.get('project_scan_path')
self._vuln = {
'rule_key':
strip(kwargs.get("rule_key"), ''),
'risk_id':
strip(kwargs.get("risk_id"), ''),
'category':
parse_category(strip(kwargs.get("category", 'Vulnerability'))),
'title':
strip(kwargs.get("title"), ''),
'file':
strip(kwargs.get("file"), ''),
'author':
strip(kwargs.get("author"), ''),
'author_email':
strip(kwargs.get("author_email"), ''),
'hash':
strip(kwargs.get("hash"), ''), # last commit
'start_line':
parse_int(kwargs.get("start_line"), 0),
'end_line':
parse_int(kwargs.get("end_line"), 0),
'report':
strip(kwargs.get("report"), ''),
'code_example':
kwargs.get("code_example", ''),
'is_false_positive':
parse_bool(kwargs.get("is_false_positive", False)),
'whitelist_rule_id':
kwargs.get("whitelist_rule_id", ''),
'evidence_content':
kwargs.get("evidence_content", ''),
'engine':
kwargs.get("engine", ''),
}
def download(self):
"""
:return:
"""
# TODO failed try
logger.info('Start downloading the upgrade package...'.format(
self.local_version, self.current_version))
remote_file = self._upgrade_info['download_path']
if 'ftp' == self._upgrade_info['download_type'].lower().strip():
try:
ftp = FTPWork(host=conf.distributed['ftp']['host'],
port=parse_int(conf.distributed['ftp']['port'],
21),
username=conf.distributed['ftp']['username'],
password=conf.distributed['ftp']['password'],
debug=False)
ftp.download_file(self.local_upgrade_file, remote_file)
return True
except Exception as ex:
logger.exception(ex)
elif 'aws' == self._upgrade_info['download_type'].lower().strip():
pass # TODO
return False
def __init__(self, **kwargs):
self._name = kwargs.get('name', None)
self._key = kwargs.get('key', None)
self._logger = kwargs.get('logger', logger)
# sonar
self._api_domain = kwargs.get('API_DOMAIN', None)
self._api_uri = kwargs.get('API_URI', None)
self._api_token = kwargs.get('API_TOKEN', None)
self._sonar_scanner_path = kwargs.get('SONAR_SCANNER_PATH', None)
self._sonar_project_properties = kwargs.get('SONAR_PROJECT_PROPERTIES',
None)
if self._api_domain and not self._api_domain.endswith('/'):
self._api_domain = '{0}/'.format(self._api_domain)
# general
self._engine_timeout = parse_int(kwargs.get('ENGINE_TIMEOUT'), 1200)
self._http_timeout = parse_int(kwargs.get('HTTP_TIMEOUT'), 10)
self._http_timeout_retry = parse_int(kwargs.get('HTTP_TIMEOUT_RETRY'),
3)
self._http_failed_retry = parse_int(kwargs.get('HTTP_FAILED_RETRY'), 3)
self._file_size_limit = parse_int(kwargs.get('FILE_SIZE_LIMIT'),
1024 * 5) # default 5MB
self._threads = parse_int(kwargs.get('threads'), 20)
def start_monitor(self):
"""
开始发送心跳监控
:return:
"""
if conf.agent.monitor_url:
logger.info(
'Send registration information to the SeeCode server, url: "{0}".'
.format(conf.agent.monitor_url))
# 节点注册
registration_data = {
'hostname': get_hostname(),
'ipv4': '127.0.0.1',
'role': 'client',
'client_version': self.version,
'action': 'node',
}
if conf.server.domain:
domain = urlparse(conf.server.domain).netloc
registration_data['ipv4'] = get_localhost_ip(domain=domain)
else:
registration_data['ipv4'] = get_localhost_ip()
self.request.post_data(conf.agent.monitor_url, registration_data)
# 服务心跳监控
if conf.general.services:
heartbeat_data = {
'hostname': get_hostname(),
'ipv4': registration_data['ipv4'],
'action': 'heartbeat',
'items': []
}
for item in conf.general.services:
try:
if item['keyword'] and item['role'] == 'client':
output, err = exec_cmd(
'ps -ef | grep "{0}" |grep -v grep|wc -l'.
format(item['keyword']))
if err:
logger.error(err)
else:
result = parse_int(output.strip())
if result == 0:
status = 2
else:
status = 1
heartbeat_data['items'].append({
'key':
item['key'],
'status':
status,
})
except Exception as ex:
logger.error(ex)
if heartbeat_data['items']:
services_url = '{0}{1}node/services/'.format(
conf.server.domain, conf.server.api_uri)
self.request.post_data(services_url, heartbeat_data)
else:
logger.critical(
'"monitor_url" parameter is empty, agent monitoring is not possible.'
)
def __init__(self, **kwargs):
"""
:param kwargs:
"""
self._logger = None
self._paths_log = ''
self._log_file = ''
self._remote_paths = ''
self._paths_project_origin = None
self._paths_project_scan = None
self._branch_commit = None
self._git = None
self._rotate_handler = None
self._statistics = {
'depends': [],
'file': [],
'total': 0,
'size': 0, # KB
'language': '',
'risk': {},
}
# scan
self._threads = parse_int_or_str(parse_int(kwargs.get('threads'), 20))
self._template = strip(kwargs.get('template'), display_type='lower')
self._template_full_path = strip(kwargs.get('template_full_path'))
self._work_dir = kwargs.get('work_dir') or '/data/seecode'
self._log_level = strip(kwargs.get('log_level')) or 'info'
self._task_id = parse_int(kwargs.get('task_id'))
self._project_name = strip(kwargs.get('project_name'))
self._project_branch = strip(kwargs.get('project_branch'))
self._project_ssh = strip(kwargs.get('project_ssh'))
self._project_web = strip(kwargs.get('project_web'))
self._project_local_path = strip(kwargs.get('project_local_path'))
self._project_type = strip(kwargs.get('project_type') or 'offline',
display_type='lower')
self._project_storage_type = strip(kwargs.get('project_storage_type'),
display_type='lower')
self._project_file_origin_name = strip(
kwargs.get('project_file_origin_name'))
self._project_file_hash = strip(kwargs.get('project_file_hash'))
self._group_name = strip(kwargs.get('group_name'))
self._group_key = strip(kwargs.get('group_key'))
# 强制更新代码，必须为线上项目
self._force_sync_code = parse_bool(kwargs.get('force_sync_code',
False))
# 强制扫描项目
self._force_project_scan = parse_bool(
kwargs.get('force_project_scan', False))
# 同步漏洞到 SeeCode Audit 服务端
self._sync_vuln_to_server = parse_bool(
kwargs.get('sync_vuln_to_server', False))
# 取证偏移
self._evidence_start_line_offset = parse_int(
kwargs.get('evidence_start_line_offset'), -1)
# 合并相同组件漏洞（依赖文件包含）
self._component_same_vuln_merge_enable = parse_bool(
kwargs.get('component_same_vuln_merge_enable', True))
self._evidence_count = parse_int(kwargs.get('evidence_count'), 5)
self._result_file = strip(kwargs.get('result_file'))
self._result_format = strip(kwargs.get('result_format'))
self._is_cli_call = parse_bool(kwargs.get('is_cli_call'))
self._issue_statistic_type = ['vulnerability']
self._send_size = parse_int_or_str(strip(kwargs.get('send_size', 50)))
# distributed
self._distributed = kwargs.get('distributed', conf.distributed)
self.storage = {}
self.__check_scan_parameters()
self.__init_work_dir()
self.__init_logger()
self.cp = CopyCMD()
self.find = FindCMD()
# check scan template
profile = os.path.join(paths.ROOT_PROFILE_PATH,
'{0}.xml'.format(self._template))
stat = os.stat(profile)
if not all((kb, kb.profiles)):
init()
if self._template_full_path and os.path.isfile(
self._template_full_path):
load_scan_template(template_file_path=self._template_full_path)
else:
load_scan_template()
kb.profiles_time[self._template] = stat.st_ctime
if kb.profiles_time[self._template] != stat.st_ctime:
load_scan_template()
kb.profiles_time[self._template] = stat.st_ctime
# result
if self.key not in kb.result:
kb.result[self.key] = {}
if not self._result_file:
self._result_file = '{0}.json'.format(self._task_id
or self._project_name)
if not self._result_file.startswith('/'):
self._result_file = os.path.join(self._paths_log,
self._result_file)
# remote storage
self.__init_distributed()
self._remote_log = os.path.join(self._remote_paths,
os.path.basename(self._log_file))
self._remote_result = os.path.join(self._remote_paths,
os.path.basename(self._result_file))
# scan task
if isinstance(self._task_id, int):
self._task_status = TaskStatus(task_id=self._task_id,
server=conf.server.domain,
server_token=conf.server.token,
logger=self.logger)
if self._sync_vuln_to_server:
self._task_status.update_initialization(
scan_template=self._template,
scan_template_version=kb.profiles[self._template].version,
log_path=self._remote_log,
)
else:
self._task_status = None
if self._template not in kb.profiles:
msg = "[ScanProject] Didn't find '{0}' template.".format(
self._template)
if self._task_status and self._sync_vuln_to_server:
self.update_scan_failed_status(msg)
raise ScanTemplateNotFound(msg)
