def test_setVerbosity(self):
# test 1
conf.scan.log_level = 'error'
setVerbosity()
logger.error = MagicMock(side_effect=logger.error)
logger.error('test error')
logger.error.assert_called_with('test error')
# test 2
conf.scan.log_level = 'warn'
setVerbosity()
logger.warn = MagicMock(side_effect=logger.warn)
logger.warn('test warn')
logger.warn.assert_called_with('test warn')
# test 3
conf.scan.log_level = 'debug'
setVerbosity()
logger.debug = MagicMock(side_effect=logger.debug)
logger.debug('test debug')
logger.debug.assert_called_with('test debug')
# test 4
conf.scan.log_level = 'info'
setVerbosity()
logger.info = MagicMock(side_effect=logger.info)
logger.info('test info')
logger.info.assert_called_with('test info')
# test 5
setVerbosity()
logger.info = MagicMock(side_effect=logger.info)
logger.info('test default level')
logger.info.assert_called_with('test default level')
def start(**kwargs):
"""
添加扫描任务
:param kwargs: 任务配置
:return:
"""
pro = None
try:
init()
result_file = '{0}.json'.format(kwargs.get("project_name"))
kwargs['result_file'] = result_file
pro = start_scan(**kwargs)
except SoftTimeLimitExceeded as ex:
logger.critical(ex)
if pro:
pro.update_scan_failed_status(ex)
except Exception as ex:
import traceback;traceback.print_exc()
logger.critical(ex)
except KeyboardInterrupt:
logger.error("user aborted")
except SystemExit:
raise
return False
def __test_scan_engine(self):
"""
测试扫描调度引擎是否启动
:return:
"""
logger.info("[ENGINE] Start detecting the scan engine...")
# sonar-scanner
sonar_scanner = 'sonar-scanner'
is_found_sonar_scanner = False
for p in ('/usr/bin/', '/usr/local/bin/'):
s_path = os.path.join(p, sonar_scanner)
if os.path.isfile(s_path):
is_found_sonar_scanner = s_path
self.result['engine'].append('seecode_scanner.lib.engines.sonarscanner ({0})'.format(s_path))
break
if is_found_sonar_scanner:
logger.info('[+] Found the "sonar-scanner" tool with the path "{0}"'.format(is_found_sonar_scanner))
else:
logger.error('[-] The "sonar-scanner" tool was not found, please confirm if it is installed.')
self.result['engine'].append('seecode_scanner.lib.engines.sonarscanner (\033[31mMISSING\033[0m)')
# engine
self.result['engine'].append('seecode_scanner.lib.engines.rulescanner')
self.result['engine'].append('seecode_scanner.lib.engines.pluginscanner')
def current_version(self):
if not self._current_version:
try:
logger.info('Check the latest version...')
url_api = '{0}/api/v2/upgrade/version/'.format(
conf.server.domain)
resp = self.server.get_data(url=url_api)
if resp and resp['code'] == 200:
if 'secret' in resp['data']:
logger.info(
'[+] The server has opened the encrypted communication.'
)
rsa = RSAEncrypt()
self._transport_encryption = True
decrypt_str = rsa.decrypt_str(resp['data']['secret'])
self._upgrade_info = ast.literal_eval(
decrypt_str.decode('utf-8'))
else:
self._upgrade_info = resp['data']
self._current_version = self._upgrade_info['version']
logger.info('The latest version of: [v{0}]'.format(
self._current_version))
except Exception as ex:
import traceback
traceback.print_exc()
logger.error(ex)
return self._current_version
def __test_scan_template(self):
"""
测试扫描模板存在与内容
:return:
"""
logger.info("[SCAN] Start testing whether the scan template of seecode-scanner exists...")
profiles = glob.glob('{0}/*.xml'.format(paths.ROOT_PROFILE_PATH))
for item in profiles:
if not os.path.isfile(item):
logger.error('[-] "{0}" file not found.'.format(item))
else:
logger.info('[+] Found "{0}" file.'.format(item))
self.result['scan'].append(item)
def __test_agent(self):
"""
测试Agent
:return:
"""
logger.info("[SERVER] Start detecting service list files...")
if not os.path.isfile(paths.SERVICES_FILE):
logger.error('[-] "{0}" file not found.'.format(paths.SERVICES_FILE))
else:
logger.info('[+] Found "{0}" file.'.format(paths.SERVICES_FILE))
logger.info("[SERVER] Start detecting core files...")
if not conf.agent.monitor_url:
logger.error('[-] "monitor_url" is not set, the current content is: {0}.'.format(conf.agent.monitor_url))
else:
logger.info('"monitor_url": {0}'.format(conf.agent.monitor_url))
self.result['agent'].append(conf.agent.monitor_url)
if not conf.agent.upgrade_url:
logger.error('[-] "upgrade_url" is not set, the current content is: {0}.'.format(conf.agent.upgrade_url))
else:
logger.info('"upgrade_url": {0}'.format(conf.agent.upgrade_url))
if not conf.agent.task_url:
logger.error('[-] "task_url" is not set, the current content is: {0}.'.format(conf.agent.task_url))
else:
logger.info('"task_url": {0}'.format(conf.agent.task_url))
def __test_config(self):
"""
测试系统配置
:return:
"""
logger.info("[CORE] Start testing whether the core file of seecode-scanner exists...")
if not os.path.isfile(CORE_CONF_FILE):
logger.error('[-] "/etc/seecode_scanner.yml" file not found.')
else:
logger.info('[+] Found "{0}" file.'.format(CORE_CONF_FILE))
logger.info("[CORE] Start checking if the working directory exists...")
if not os.path.isdir(conf.scan.work_dir or paths.ROOT_WORK_PATH):
logger.error('Detected that the "{0}" directory does not exist.'.format(conf.scan.work_dir or paths.ROOT_WORK_PATH))
else:
logger.info('[+] Discover "{0}" directory.'.format(conf.scan.work_dir or paths.ROOT_WORK_PATH))
def generate_pem(self, save_path, nbits=2048):
"""
:param save_path: 保存路径
:param nbits:
:return:
"""
make_dir(save_path)
self.public_key, self.private_key = rsa.newkeys(nbits)
public_pem = os.path.join(save_path, 'public.pem')
private_pem = os.path.join(save_path, 'private.pem')
try:
with open(public_pem, 'w+') as fp:
fp.write(self.public_key.save_pkcs1().decode())
with open(private_pem, 'w+') as fp:
fp.write(self.private_key.save_pkcs1().decode())
except Exception as ex:
logger.error(ex)
return public_pem, private_pem
def start_monitor(self):
"""
开始发送心跳监控
:return:
"""
if conf.agent.monitor_url:
logger.info(
'Send registration information to the SeeCode server, url: "{0}".'
.format(conf.agent.monitor_url))
# 节点注册
registration_data = {
'hostname': get_hostname(),
'ipv4': '127.0.0.1',
'role': 'client',
'client_version': self.version,
'action': 'node',
}
if conf.server.domain:
domain = urlparse(conf.server.domain).netloc
registration_data['ipv4'] = get_localhost_ip(domain=domain)
else:
registration_data['ipv4'] = get_localhost_ip()
self.request.post_data(conf.agent.monitor_url, registration_data)
# 服务心跳监控
if conf.general.services:
heartbeat_data = {
'hostname': get_hostname(),
'ipv4': registration_data['ipv4'],
'action': 'heartbeat',
'items': []
}
for item in conf.general.services:
try:
if item['keyword'] and item['role'] == 'client':
output, err = exec_cmd(
'ps -ef | grep "{0}" |grep -v grep|wc -l'.
format(item['keyword']))
if err:
logger.error(err)
else:
result = parse_int(output.strip())
if result == 0:
status = 2
else:
status = 1
heartbeat_data['items'].append({
'key':
item['key'],
'status':
status,
})
except Exception as ex:
logger.error(ex)
if heartbeat_data['items']:
services_url = '{0}{1}node/services/'.format(
conf.server.domain, conf.server.api_uri)
self.request.post_data(services_url, heartbeat_data)
else:
logger.critical(
'"monitor_url" parameter is empty, agent monitoring is not possible.'
)
def main():
"""
:return:
"""
try:
init()
args = cmd_line()
_mergeOptions(args, conf)
t1 = time.time()
if conf.agent.enable_monitor:
agent = AgentServices()
agent.start_monitor()
exit(0)
elif conf.general.enable_upgrade:
um = UpgradeManage()
um.start_upgrade()
exit(0)
elif conf.celery.enable:
cmd_list = [
'cd', BASEDIR, '&&',
get_python_path(), '$(which celery)', 'worker', '-A',
'seecode_scanner.celeryctl.celery_app', '-c',
str(conf.celery.concurrency), '-Q', 'seecode', '-n',
conf.celery.name
]
logger.info(' '.join(cmd_list))
os.system(' '.join(cmd_list))
exit(0)
elif conf.scan.project_path:
if not all((
args.project_path,
args.project_name,
)):
print("[*] Missing '--scan-path, --name' key parameters.\n")
exit(0)
# scan
start_scan(
task_id=conf.scan.task_id,
template=conf.scan.template,
template_full_path=conf.scan.template_full_path,
threads=conf.scan.threads,
log_level=conf.scan.log_level,
work_dir=conf.scan.work_dir,
project_name=conf.scan.project_name,
project_local_path=conf.scan.project_path,
project_branch=conf.scan.project_branch,
project_ssh=conf.scan.project_ssh,
project_web=conf.scan.project_web,
project_type=conf.scan.project_type,
project_storage_type=conf.scan.project_storage_type,
group_name=conf.scan.group_name,
group_key=conf.scan.group_key,
evidence_start_line_offset=conf.scan.evidence_start_line_offset,
evidence_count=conf.scan.evidence_count,
result_file=conf.scan.result_file,
result_format=conf.scan.result_format,
sync_vuln_to_server=conf.scan.sync_vuln_to_server,
force_sync_code=conf.scan.force_sync_code,
is_cli_call=True,
)
logger.info('Analysis completed, time consuming: {0}s'.format(
round(time.time() - t1, 2)))
except Exception as ex:
import traceback
traceback.print_exc()
if conf.scan.log_level == 'debug':
import traceback
traceback.print_exc()
logger.critical(ex)
exit(-1)
except KeyboardInterrupt:
logger.error("user aborted")
except EOFError:
logger.error("exit")
except SystemExit:
raise
def start_upgrade(self):
"""
:return:
"""
try:
logger.info(
'Start upgrading, check if the local version is consistent with the server version...'
)
if self.is_upgrade:
self.download()
if os.path.isfile(self.local_upgrade_file):
logger.info(
'Start decompressing the encryption upgrade package...'
)
# Decrypted encrypted upgrade package
tar = tarfile.open(self.local_upgrade_file, 'r:gz')
file_names = tar.getnames()
for file_name in file_names:
tar.extract(file_name, self._upgrade_path)
tar.close()
file_list = glob.glob(
os.path.join(self._upgrade_path, '*.bin'))
save_tgz = ''
for l in file_list:
file_name = os.path.splitext(os.path.basename(l))[0]
save_tgz = os.path.join(self._upgrade_path,
'{0}.tgz'.format(file_name))
self.rsa.decrypt_file(l, save_tgz)
break
logger.info(
'Unzip the encryption upgrade package to complete.')
# Decompress and decrypt the compressed package
if os.path.isfile(save_tgz):
logger.info(
'Start decompressing the decryption upgrade package...'
)
decrypt_path = os.path.join(self._upgrade_path,
'decrypt')
tar = tarfile.open(save_tgz, 'r:gz')
file_names = tar.getnames()
for file_name in file_names:
tar.extract(file_name, decrypt_path)
tar.close()
logger.info(
'Decompression and decryption upgrade package completed'
)
logger.info('Start syncing scan templates...')
# profiles
profiles = glob.glob(
os.path.join(decrypt_path, '*.xml'))
for p in profiles:
file_name = os.path.basename(p)
dest = os.path.join(paths.ROOT_PROFILE_PATH,
file_name)
if os.path.isfile(dest):
os.unlink(dest)
shutil.copy(p, dest)
logger.info('Synchronous scan template completion.')
# plugins
blacklist_path = os.path.join(decrypt_path, 'plugins',
'blacklist')
if os.path.isdir(blacklist_path):
logger.info('Start syncing blacklist plugin...')
clean_dir(paths.ROOT_PLUGINS_BLACKLIST_PATH)
shutil.copytree(blacklist_path,
paths.ROOT_PLUGINS_BLACKLIST_PATH)
logger.info(
'Synchronous blacklist plugin completed.')
whitelist_path = os.path.join(decrypt_path, 'plugins',
'whitelist')
if os.path.isdir(whitelist_path):
logger.info('Start syncing whitelist plugin...')
clean_dir(paths.ROOT_PLUGINS_WHITELIST_PATH)
shutil.copytree(whitelist_path,
paths.ROOT_PLUGINS_WHITELIST_PATH)
logger.info(
'Synchronous whitelist plugin completed.')
for d in [
paths.ROOT_PLUGINS_BLACKLIST_PATH,
paths.ROOT_PLUGINS_WHITELIST_PATH
]:
module_file = os.path.join(d, '__init__.py')
if not os.path.isfile(module_file):
with open(module_file, 'wb') as fp:
fp.write(''.encode('utf-8'))
# write version
logger.info(
'Start updating the current version to v{0}.'.format(
self.current_version))
with open(paths.PROFILE_VERSION_PATH, 'wb') as fp:
fp.write(self.current_version.encode("utf-8"))
self.upload_upgrade_status_success()
logger.info(
'Upgrade completed, current version: v{0}'.format(
self.current_version))
else:
self.download()
logger.error(
'Download upgrade package failed, "{0}" not found.'.
format(self._upgrade_info['download_path']))
else:
logger.info(
'[*] No upgrade required, exit the upgrade program.')
except Exception as ex:
import traceback
self.upload_upgrade_status_failure(
reason=str(ex), stack_trace=traceback.format_exc())
logger.critical(ex)