def start(**kwargs):
"""
添加扫描任务
:param kwargs: 任务配置
:return:
"""
pro = None
try:
init()
result_file = '{0}.json'.format(kwargs.get("project_name"))
kwargs['result_file'] = result_file
pro = start_scan(**kwargs)
except SoftTimeLimitExceeded as ex:
logger.critical(ex)
if pro:
pro.update_scan_failed_status(ex)
except Exception as ex:
import traceback;traceback.print_exc()
logger.critical(ex)
except KeyboardInterrupt:
logger.error("user aborted")
except SystemExit:
raise
return False
def start_monitor(self):
"""
开始发送心跳监控
:return:
"""
if conf.agent.monitor_url:
logger.info(
'Send registration information to the SeeCode server, url: "{0}".'
.format(conf.agent.monitor_url))
# 节点注册
registration_data = {
'hostname': get_hostname(),
'ipv4': '127.0.0.1',
'role': 'client',
'client_version': self.version,
'action': 'node',
}
if conf.server.domain:
domain = urlparse(conf.server.domain).netloc
registration_data['ipv4'] = get_localhost_ip(domain=domain)
else:
registration_data['ipv4'] = get_localhost_ip()
self.request.post_data(conf.agent.monitor_url, registration_data)
# 服务心跳监控
if conf.general.services:
heartbeat_data = {
'hostname': get_hostname(),
'ipv4': registration_data['ipv4'],
'action': 'heartbeat',
'items': []
}
for item in conf.general.services:
try:
if item['keyword'] and item['role'] == 'client':
output, err = exec_cmd(
'ps -ef | grep "{0}" |grep -v grep|wc -l'.
format(item['keyword']))
if err:
logger.error(err)
else:
result = parse_int(output.strip())
if result == 0:
status = 2
else:
status = 1
heartbeat_data['items'].append({
'key':
item['key'],
'status':
status,
})
except Exception as ex:
logger.error(ex)
if heartbeat_data['items']:
services_url = '{0}{1}node/services/'.format(
conf.server.domain, conf.server.api_uri)
self.request.post_data(services_url, heartbeat_data)
else:
logger.critical(
'"monitor_url" parameter is empty, agent monitoring is not possible.'
)
def main():
"""
:return:
"""
try:
init()
args = cmd_line()
_mergeOptions(args, conf)
t1 = time.time()
if conf.agent.enable_monitor:
agent = AgentServices()
agent.start_monitor()
exit(0)
elif conf.general.enable_upgrade:
um = UpgradeManage()
um.start_upgrade()
exit(0)
elif conf.celery.enable:
cmd_list = [
'cd', BASEDIR, '&&',
get_python_path(), '$(which celery)', 'worker', '-A',
'seecode_scanner.celeryctl.celery_app', '-c',
str(conf.celery.concurrency), '-Q', 'seecode', '-n',
conf.celery.name
]
logger.info(' '.join(cmd_list))
os.system(' '.join(cmd_list))
exit(0)
elif conf.scan.project_path:
if not all((
args.project_path,
args.project_name,
)):
print("[*] Missing '--scan-path, --name' key parameters.\n")
exit(0)
# scan
start_scan(
task_id=conf.scan.task_id,
template=conf.scan.template,
template_full_path=conf.scan.template_full_path,
threads=conf.scan.threads,
log_level=conf.scan.log_level,
work_dir=conf.scan.work_dir,
project_name=conf.scan.project_name,
project_local_path=conf.scan.project_path,
project_branch=conf.scan.project_branch,
project_ssh=conf.scan.project_ssh,
project_web=conf.scan.project_web,
project_type=conf.scan.project_type,
project_storage_type=conf.scan.project_storage_type,
group_name=conf.scan.group_name,
group_key=conf.scan.group_key,
evidence_start_line_offset=conf.scan.evidence_start_line_offset,
evidence_count=conf.scan.evidence_count,
result_file=conf.scan.result_file,
result_format=conf.scan.result_format,
sync_vuln_to_server=conf.scan.sync_vuln_to_server,
force_sync_code=conf.scan.force_sync_code,
is_cli_call=True,
)
logger.info('Analysis completed, time consuming: {0}s'.format(
round(time.time() - t1, 2)))
except Exception as ex:
import traceback
traceback.print_exc()
if conf.scan.log_level == 'debug':
import traceback
traceback.print_exc()
logger.critical(ex)
exit(-1)
except KeyboardInterrupt:
logger.error("user aborted")
except EOFError:
logger.error("exit")
except SystemExit:
raise
def start_scan(**kwargs):
"""
:param kwargs:
:return:
"""
pro = None
try:
pro = ScanProject(
task_id=kwargs.get("task_id"),
template=kwargs.get("template"),
template_full_path=kwargs.get("template_full_path"),
threads=kwargs.get("threads"),
log_level=kwargs.get("log_level"),
work_dir=kwargs.get("work_dir"),
project_name=kwargs.get("project_name"),
project_local_path=kwargs.get("project_local_path"),
project_branch=kwargs.get("project_branch"),
project_ssh=kwargs.get("project_ssh"),
project_web=kwargs.get("project_web"),
project_type=kwargs.get("project_type"),
project_storage_type=kwargs.get("project_storage_type"),
project_file_origin_name=kwargs.get("project_file_origin_name"),
project_file_hash=kwargs.get("project_file_hash"),
group_name=kwargs.get("group_name"),
group_key=kwargs.get("group_key"),
evidence_start_line_offset=kwargs.get(
"evidence_start_line_offset"),
evidence_count=kwargs.get("evidence_count"),
result_file=kwargs.get("result_file"),
result_format=kwargs.get("result_format"),
sync_vuln_to_server=kwargs.get("sync_vuln_to_server"),
force_sync_code=kwargs.get("force_sync_code"),
is_cli_call=kwargs.get("is_cli_call", False),
)
try:
profile = kb.profiles[pro.template_name]
pro.sync_code()
pro.clear_exclude(exclude_dir='.git', )
pro.analysis_component()
pro.clear_exclude(
exclude_dir=profile.exclude_dir,
exclude_ext=profile.exclude_ext,
exclude_file=profile.exclude_file,
)
for name, engine in profile.engines.items():
engine.start_component(pro)
for name, engine in profile.engines.items():
try:
pro.update_scan_engine_status(engine.key, 'blacklist')
engine.start_blacklist(pro)
except CodeBaseException as ex:
pro.update_scan_failed_status(ex)
for name, engine in profile.engines.items():
pro.update_scan_engine_status(engine.key, 'whitelist')
engine.start_whitelist(pro)
pro.sync_vuln()
except (ScanDirectoryIsEmpty, Exception) as ex:
pro.update_scan_failed_status(ex)
pro.upload_result_log()
except CodeBaseException as ex:
pro.update_scan_failed_status(ex)
return pro
except CodeBaseException as ex:
import traceback
traceback.print_exc()
logger.critical(ex)
return pro
def start_upgrade(self):
"""
:return:
"""
try:
logger.info(
'Start upgrading, check if the local version is consistent with the server version...'
)
if self.is_upgrade:
self.download()
if os.path.isfile(self.local_upgrade_file):
logger.info(
'Start decompressing the encryption upgrade package...'
)
# Decrypted encrypted upgrade package
tar = tarfile.open(self.local_upgrade_file, 'r:gz')
file_names = tar.getnames()
for file_name in file_names:
tar.extract(file_name, self._upgrade_path)
tar.close()
file_list = glob.glob(
os.path.join(self._upgrade_path, '*.bin'))
save_tgz = ''
for l in file_list:
file_name = os.path.splitext(os.path.basename(l))[0]
save_tgz = os.path.join(self._upgrade_path,
'{0}.tgz'.format(file_name))
self.rsa.decrypt_file(l, save_tgz)
break
logger.info(
'Unzip the encryption upgrade package to complete.')
# Decompress and decrypt the compressed package
if os.path.isfile(save_tgz):
logger.info(
'Start decompressing the decryption upgrade package...'
)
decrypt_path = os.path.join(self._upgrade_path,
'decrypt')
tar = tarfile.open(save_tgz, 'r:gz')
file_names = tar.getnames()
for file_name in file_names:
tar.extract(file_name, decrypt_path)
tar.close()
logger.info(
'Decompression and decryption upgrade package completed'
)
logger.info('Start syncing scan templates...')
# profiles
profiles = glob.glob(
os.path.join(decrypt_path, '*.xml'))
for p in profiles:
file_name = os.path.basename(p)
dest = os.path.join(paths.ROOT_PROFILE_PATH,
file_name)
if os.path.isfile(dest):
os.unlink(dest)
shutil.copy(p, dest)
logger.info('Synchronous scan template completion.')
# plugins
blacklist_path = os.path.join(decrypt_path, 'plugins',
'blacklist')
if os.path.isdir(blacklist_path):
logger.info('Start syncing blacklist plugin...')
clean_dir(paths.ROOT_PLUGINS_BLACKLIST_PATH)
shutil.copytree(blacklist_path,
paths.ROOT_PLUGINS_BLACKLIST_PATH)
logger.info(
'Synchronous blacklist plugin completed.')
whitelist_path = os.path.join(decrypt_path, 'plugins',
'whitelist')
if os.path.isdir(whitelist_path):
logger.info('Start syncing whitelist plugin...')
clean_dir(paths.ROOT_PLUGINS_WHITELIST_PATH)
shutil.copytree(whitelist_path,
paths.ROOT_PLUGINS_WHITELIST_PATH)
logger.info(
'Synchronous whitelist plugin completed.')
for d in [
paths.ROOT_PLUGINS_BLACKLIST_PATH,
paths.ROOT_PLUGINS_WHITELIST_PATH
]:
module_file = os.path.join(d, '__init__.py')
if not os.path.isfile(module_file):
with open(module_file, 'wb') as fp:
fp.write(''.encode('utf-8'))
# write version
logger.info(
'Start updating the current version to v{0}.'.format(
self.current_version))
with open(paths.PROFILE_VERSION_PATH, 'wb') as fp:
fp.write(self.current_version.encode("utf-8"))
self.upload_upgrade_status_success()
logger.info(
'Upgrade completed, current version: v{0}'.format(
self.current_version))
else:
self.download()
logger.error(
'Download upgrade package failed, "{0}" not found.'.
format(self._upgrade_info['download_path']))
else:
logger.info(
'[*] No upgrade required, exit the upgrade program.')
except Exception as ex:
import traceback
self.upload_upgrade_status_failure(
reason=str(ex), stack_trace=traceback.format_exc())
logger.critical(ex)