def insert_strategies(self, strategy_id_set):
if not strategy_id_set:
return
logger.info('开始插入策略【%s】', len(strategy_id_set))
strategy_insert_sql = '''INSERT INTO strategy
(strategy_id, strategy_name, author, `desc`, create_time, created_at, updated_at)
VALUES(%s, %s, %s, %s, %s, %s, %s)'''
now = datetime.datetime.now()
count = 0
successful_count = 0
cursor = self.cnx.cursor()
for info in [
x for x in [self.strategy_ind[x] for x in strategy_id_set]
if x is not None
]:
progress(count, len(strategy_id_set),
'插入策略 {}'.format(info.get('strategy_id')))
info['created_at'] = now
info['updated_at'] = now
try:
cursor.execute(strategy_insert_sql, [
info.get(k) for k in [
'strategy_id', 'name', 'author', 'desc', 'create_date',
'created_at', 'updated_at'
]
])
successful_count += 1
except Exception as err:
logger.warning('策略插入失败：%s\n%s', info, err)
self.cnx.commit()
logger.info('成功插入策略【%s个】', successful_count)
def sort_pocs(poc_base_dir, ignore_dirs=['common']):
'''将 POC 放置到其检查的 产品类型/产品名 目录下'''
mod_count = 0
if '.venv' not in ignore_dirs:
ignore_dirs.append('.venv')
for (mod, d, file_name) in iter_modules(poc_base_dir, ignore_dirs):
mod_count += 1
progress(mod_count, mod_count, '处理模块', os.path.join(d, file_name))
(vuln, _) = load_poc_mod(mod)
if vuln is None:
continue
prd = vuln.product
component = Component.get_component(prd)
typ = component.type
should_be_in = os.path.join(poc_base_dir, typ.name,
prd.replace(' ', '_'))
if should_be_in != d:
if not os.path.exists(should_be_in):
os.makedirs(should_be_in)
src_file = os.path.join(d, file_name)
dst_file = os.path.join(should_be_in, file_name)
logger.info('move from {} to {}'.format(src_file, dst_file))
os.rename(src_file, dst_file)
logger.info(
'********* Clean up poc dir: {} ************'.format(poc_base_dir))
_clean_up_poc_dirs(poc_base_dir)
def update_strategies(self, strategy_id_set):
if not strategy_id_set:
return
strategy_update_sql = '''UPDATE strategy
SET strategy_name=%s, author=%s, `desc`=%s, create_time=%s, updated_at=%s
WHERE strategy_id=%s'''
count = 0
successful_count = 0
cursor = self.cnx.cursor()
for info in [
x for x in [self.strategy_ind[x] for x in strategy_id_set]
if x is not None
]:
progress(count, len(strategy_id_set),
'更新策略 {}'.format(info.get('strategy_id')))
info['updated_at'] = datetime.datetime.now()
try:
cursor.execute(strategy_update_sql, [
info.get(k) for k in [
'name', 'author', 'desc', 'create_date', 'updated_at',
'strategy_id'
]
])
successful_count += 1
except Exception as err:
logger.warning('策略更新失败：%s\n%s', info, err)
self.cnx.commit()
logger.info('成功更新策略【%s个】', successful_count)
def indexing_strategies(strategy_dir, index_dir=None):
strategy_ind_file = INDEX_CONFIG.get_strategy_index_file(index_dir)
with open(strategy_ind_file, 'w') as strategy_ind:
mod_count = 0
successful_count = 0
strategy_ids = {}
for (mod, dirpath, filename) in iter_modules(strategy_dir):
pth = os.path.join(dirpath, filename)
mod_count += 1
progress(mod_count, successful_count, '处理模块', pth)
strategy = None
try:
strategy = load_strategy_mod(mod)
except:
logger.exception('模块加载出错: %s', pth)
continue
if strategy.strategy_id in strategy_ids:
logger.warning('相同 id 的策略在 %s 已经出现: %s',
strategy_ids[strategy.strategy_id], pth)
continue
strategy_ids[strategy.strategy_id] = pth
strategy_dict = dump_strategy_to_dict(strategy)
strategy_dict['__file__'] = pth
strategy_dict['__class__'] = strategy.__class__.__name__
_write_obj(strategy_ind, strategy_dict)
successful_count += 1
logger.info('*********** 成功加载 %s 个模块【共计 %s 个】**********', successful_count,
mod_count)
def indexing_pocs(poc_dir, index_dir=None):
(vuln_ind_file,
poc_ind_file) = (INDEX_CONFIG.get_vuln_index_file(index_dir),
INDEX_CONFIG.get_poc_index_file(index_dir))
vuln_ids = set({})
poc_ids = set({})
logger.info('开始查找 %s 下的 POC 信息', poc_dir)
with open(poc_ind_file, 'w') as poc_ind, \
open(vuln_ind_file, 'w') as vuln_ind:
mod_count = 0
successful_count = 0
for (mod, poc_dir, poc_file) in iter_modules(poc_dir):
poc_path = os.path.join(poc_dir, poc_file)
mod_count += 1
progress(mod_count, successful_count, '处理POC模块', poc_path)
(vuln, pocs) = (None, None)
try:
(vuln, pocs) = load_poc_mod(mod)
except:
logger.exception('模块加载出错: %s', poc_path)
if vuln is not None and vuln.vuln_id not in vuln_ids:
vuln_ids.add(vuln.vuln_id)
_write_obj(vuln_ind, dump_vuln_to_dict(vuln))
for poc in pocs:
if poc.poc_id not in poc_ids:
poc_ids.add(poc.poc_id)
poc_dict = dump_poc_to_dict(poc)
poc_dict['__file__'] = os.path.join(poc_dir, poc_file)
poc_dict['__class__'] = poc.__class__.__name__
_write_obj(poc_ind, poc_dict)
successful_count += 1
logger.info('*********** 成功加载 %s 个模块【共计 %s 个】**********', successful_count,
mod_count)
def pocs(self):
component_name = self.get_option('component')
if component_name is None:
return []
logger.info('遍历查找组件 %s 的 POC', component_name)
for poc in iter_pocs_of_component(component_name, self.index_dir):
poc.output.strategy = self
yield poc
def sync_strategy(self):
logger.info('同步策略数据')
existed_strategy_ids = set(self.fetch_strategy_ids())
all_strategy_ids = set(self.strategy_ind.keys())
self.insert_strategies(
all_strategy_ids.difference(existed_strategy_ids))
if self.updating:
self.update_strategies(
all_strategy_ids.intersection(existed_strategy_ids))
def sync_poc(self):
logger.info('同步 POC 数据')
existed_poc_vuln_ind = {}
for item in self.fetch_poc_and_related_vuln_ids():
existed_poc_vuln_ind[item[0]] = item[1]
existed_poc_ids = set(existed_poc_vuln_ind.keys())
all_poc_ids = set(self.poc_vuln_ind.keys())
self.insert_pocs(all_poc_ids.difference(existed_poc_ids))
if self.updating:
self.update_pocs(all_poc_ids.intersection(existed_poc_ids))
logger.info('完成 POC 数据同步')
def pocs(self):
if self.component_name is None:
return []
logger.info('遍历查找组件 %s 的 POC', self.component_name)
all_http_name = ["Apache", "Nginx", "IIS", "uWSGI", "Tomcat", "Node.js"]
if (not self.component_name.upper() == "http".upper()) and self.component_name.upper() in [tmp.upper() for tmp in all_http_name]:
for poc in iter_pocs_of_component(self.component_name, self.index_dir):
poc.output.strategy = self
yield poc
elif self.component_name.upper() == "http".upper():
for component_name in all_http_name:
for poc in iter_pocs_of_component(component_name, self.index_dir):
poc.output.strategy = self
yield poc
def sync_vuln(self, vuln_id_set):
logger.info('同步 Vuln 数据')
all_vuln_ids = set(self.vuln_ind.keys())
existed_vuln_ids = self.fetch_vuln_ids()
self.sync_components()
self.insert_vuln(all_vuln_ids.difference(existed_vuln_ids))
if self.updating:
self.update_vuln(all_vuln_ids.intersection(existed_vuln_ids))
self.vuln_synced = True
self.synced_vuln_ids_in_db = self.fetch_vuln_ids()
logger.info('完成漏洞数据同步')
def main():
args = None
parser = create_cmd_parser()
try:
args = parser.parse_args()
except:
logger.exception('解析错误')
raise
setup_cscan_poc_logger(verbose=args.verbose,
very_verbose=args.very_verbose)
logger.debug('解析组件属性')
components_properties = {}
parse_properties(args, components_properties=components_properties)
logger.info('开始尝试推荐任务')
recommend(components_properties)
def _clean_up_poc_dirs(path):
'''移除所有无用目录（空或者只有 .pyc 的目录）'''
if not os.path.isdir(path):
return
# remove empty subfolders
files = os.listdir(path)
if len(files) != 0:
for f in files:
fullpath = os.path.join(path, f)
if os.path.isdir(fullpath):
_clean_up_poc_dirs(fullpath)
elif fullpath.endswith('.pyc'):
os.remove(fullpath)
# if folder empty, delete it
if len(os.listdir(path)) == 0:
logger.info('Remove empty dir: {}'.format(path))
os.rmdir(path)
def main():
(parser, args) = (None), None
try:
parser = create_parser()
args = parser.parse_args()
except:
return
setup_cscan_poc_logger(verbose=args.verbose,
very_verbose=args.very_verbose)
if args.sort:
if not args.doc_dir:
logger.warning('未指定 --poc-dir')
parser.print_usage()
return
sort_pocs(args.poc_dir)
return
if args.vuln_detail_dir:
args.skip_indexing = True
if not args.skip_indexing:
if not args.skip_indexing_poc and args.poc_dir:
logger.info('开始索引 POC ...')
indexing_pocs(args.poc_dir, args.index_dir)
if not args.skip_indexing_strategy and args.strategy_dir:
logger.info('开始索引策略 ...')
indexing_strategies(args.strategy_dir, args.index_dir)
if args.vuln_detail_dir or not args.skip_syncing:
cnx = mysql.connector.connect(user=args.user,
password=args.passwd,
host=args.host,
database=args.db,
port=args.port,
charset='utf8')
cscan_db = CScanDb(cnx, args.index_dir, args.update)
if args.vuln_detail_dir:
logger.info('同步漏洞详情...')
cscan_db.sync_vuln_detail(args.vuln_detail_dir,
args.vuln_detail_static_dir,
args.vuln_ids)
else:
logger.info('开始同步数据...')
cscan_db.sync_poc()
cscan_db.sync_strategy()
def sync_components(self):
logger.info('同步组件数据')
existed_c_names = set([x[1] for x in self.fetch_component_id_names()])
all_product_names = set(
[x['product'] for x in list(self.vuln_ind.values())])
for common_component in Component.get_common_components():
logger.info('通用组件：%s', common_component)
all_product_names.add(common_component)
self.insert_component(all_product_names.difference(existed_c_names))
if self.updating:
self.update_component(
all_product_names.intersection(existed_c_names))
self.component_synced = True
logger.info('完成组件数据同步')
def insert_pocs(self, poc_id_set):
if not poc_id_set:
return
logger.info('开始插入 POC [count={}]'.format(len(poc_id_set)))
self.sync_vuln(
set([self.poc_vuln_ind[poc_id] for poc_id in poc_id_set]))
poc_insert_sql = '''INSERT INTO poc
(poc_id, poc_name, author, vuln_id, create_time, created_at, updated_at, args)
VALUES(%s, %s, %s, %s, %s, %s, %s, %s)'''
logger.info('准备要插入的 POC 数据')
now = datetime.datetime.now()
poc_infos = [
x for x in [self.poc_ind[x] for x in poc_id_set] if x is not None
]
count = 0
cursor = self.cnx.cursor()
for poc_info in poc_infos:
progress(count, len(poc_infos), '插入 POC')
count += 1
vuln_id = self.poc_vuln_ind[poc_info['poc_id']]
if vuln_id not in self.synced_vuln_ids_in_db:
poc_info['vuln_id'] = None
else:
poc_info['vuln_id'] = vuln_id
poc_info['created_at'] = now
poc_info['updated_at'] = now
poc_info['args'] = poc_info.get('option_schema', None)
poc_info['create_date'] = poc_info.get('create_date', now)
try:
cursor.execute(poc_insert_sql, [
poc_info.get(k) for k in [
'poc_id', 'name', 'author', 'vuln_id', 'create_date',
'created_at', 'updated_at', 'args'
]
])
except Exception as err:
logger.warning('POC 插入失败: {}\n{}'.format(poc_info, err))
self.cnx.commit()
logger.info('成功插入 POC [count={}]'.format(len(poc_id_set)))
def update_pocs(self, poc_id_set):
if (len(poc_id_set) == 0):
return
logger.info('开始更新 POC [count={}]'.format(len(poc_id_set)))
self.sync_vuln(
set([self.poc_vuln_ind[poc_id] for poc_id in poc_id_set]))
poc_update_sql = '''UPDATE poc
SET poc_name=%s, author=%s, vuln_id=%s,
updated_at=%s, args=%s, create_time=%s
WHERE poc_id=%s'''
logger.info('准备要更新的 POC 数据 [count={}]'.format(len(poc_id_set)))
now = datetime.datetime.now()
poc_infos = [
x for x in [self.poc_ind[x] for x in poc_id_set] if x is not None
]
count = 0
cursor = self.cnx.cursor()
for poc_info in poc_infos:
progress(count, len(poc_infos), '更新 POC')
count += 1
vuln_id = self.poc_vuln_ind[poc_info['poc_id']]
if vuln_id not in self.synced_vuln_ids_in_db:
poc_info['vuln_id'] = None
else:
poc_info['vuln_id'] = vuln_id
poc_info['updated_at'] = now
poc_info['args'] = poc_info.get('option_schema', None)
poc_info['create_date'] = poc_info.get('create_date', now)
try:
cursor.execute(poc_update_sql, [
poc_info.get(k) for k in [
'name', 'author', 'vuln_id', 'updated_at', 'args',
'poc_id', 'create_date'
]
])
except Exception as e:
logger.warn('POC 更新失败: {}\n{}\n{}'.format(
poc_info, e, poc_update_sql))
self.cnx.commit()
logger.info('成功更新 POC [count={}]'.format(len(poc_id_set)))
def insert_vuln(self, vuln_id_set):
if (len(vuln_id_set) == 0):
return
logger.info('开始插入 Vuln [count={}]'.format(len(vuln_id_set)))
vuln_insert_sql = '''INSERT INTO vuln
(vuln_id, vuln_name, vuln_type, c_id, c_version, cve_id, disclosure_date,
submit_time, level, source, detail, created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)'''
logger.info('准备要插入的 Vuln 数据')
vuln_infos = [
info
for info in [self.vuln_ind[vuln_id] for vuln_id in vuln_id_set]
if info is not None
]
now = datetime.datetime.now()
cursor = self.cnx.cursor()
count = 0
for vuln_info in vuln_infos:
progress(count, len(vuln_infos), '插入漏洞')
count += 1
c_id = self.get_component_id(vuln_info['product'])
if c_id is None:
continue
vuln_info['c_id'] = c_id
vuln_info['submit_time'] = now
vuln_info['created_at'] = now
vuln_info['updated_at'] = now
try:
cursor.execute(vuln_insert_sql, [
vuln_info.get(k) for k in [
'vuln_id', 'name', 'type', 'c_id', 'product_version',
'cve_id', 'disclosure_date', 'submit_time', 'level',
'ref', 'desc', 'created_at', 'updated_at'
]
])
except Exception as e:
logger.warn('Vuln 插入失败: {}\n{}'.format(vuln_info, e))
self.cnx.commit()
logger.info('成功插入 Vuln [count={}]'.format(len(vuln_id_set)))
def update_component(self, component_name_set):
if (len(component_name_set) == 0):
return
logger.info('更新 Component [count={}]'.format(len(component_name_set)))
component_update_sql = '''UPDATE component
SET c_first=%s, c_type=%s, `desc`=%s, producer=%s, properties=%s, updated_at=%s
WHERE c_name=%s'''
logger.info('准备要更新的 Component 数据')
now = datetime.datetime.now()
name_infos = [
name_info for name_info in [(n, get_product_info(n))
for n in component_name_set]
if name_info[1] is not None
]
cursor = self.cnx.cursor()
count = 0
for (n, info) in name_infos:
progress(count, len(name_infos), '更新组件')
count += 1
info['c_name'] = n
info['updated_at'] = now
try:
cursor.execute(component_update_sql, [
info.get(k) for k in [
'name_pinyin_first', 'type', 'desc', 'producer',
'properties', 'updated_at', 'c_name'
]
])
except Exception as e:
logger.warn('Component 更新失败: {} {}\n{}'.format(n, info, e))
self.cnx.commit()
logger.info('成功更新 Component [count={}]'.format(
len(component_name_set)))
def insert_component(self, component_name_set):
if (len(component_name_set) == 0):
return
logger.info('开始插入 Component [count={}]'.format(
len(component_name_set)))
component_insert_sql = '''INSERT INTO component
(c_id, c_name, c_first, c_type, `desc`, producer,
properties, created_at, updated_at)
VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s)'''
logger.info('准备要插入的 Component 数据')
now = datetime.datetime.now()
name_infos = [
name_info for name_info in [(n, get_product_info(n))
for n in component_name_set]
if name_info[1] is not None
]
cursor = self.cnx.cursor()
i = 0
for (n, info) in name_infos:
progress(i, len(name_infos), '插入组件')
i += 1
info['c_id'] = str(uuid.uuid4())
info['c_name'] = n
info['created_at'] = now
info['updated_at'] = now
try:
cursor.execute(component_insert_sql, [
info.get(k) for k in [
'c_id', 'c_name', 'name_pinyin_first', 'type', 'desc',
'producer', 'properties', 'created_at', 'updated_at'
]
])
except Exception as e:
logger.warn('组件插入失败: {} {}\n{}'.format(n, info, e))
self.cnx.commit()
logger.info('成功插入 Component [count={}]'.format(
len(component_name_set)))
def update_vuln(self, vuln_id_set):
if (len(vuln_id_set) == 0):
return
logger.info('开始更新漏洞 [count={}]'.format(len(vuln_id_set)))
vuln_update_sql = '''UPDATE vuln
SET vuln_name=%s, vuln_type=%s, c_id=%s, c_version=%s, cve_id=%s, disclosure_date=%s, level=%s, source=%s, detail=%s, updated_at=%s
WHERE vuln_id=%s'''
logger.info('准备要更新的 Vuln 数据')
vuln_infos = [
info
for info in [self.vuln_ind[vuln_id] for vuln_id in vuln_id_set]
if info is not None
]
now = datetime.datetime.now()
count = 0
cursor = self.cnx.cursor()
for vuln_info in vuln_infos:
progress(count, len(vuln_infos), '更新漏洞')
count += 1
c_id = self.get_component_id(vuln_info['product'])
vuln_info['c_id'] = c_id
vuln_info['updated_at'] = now
try:
cursor.execute(vuln_update_sql, [
vuln_info.get(k) for k in [
'name', 'type', 'c_id', 'product_version', 'cve_id',
'disclosure_date', 'level', 'ref', 'desc',
'updated_at', 'vuln_id'
]
])
except Exception as e:
logger.warn('Vuln 更新失败: {}\n{}'.format(vuln_info, e))
self.cnx.commit()
logger.info('成功更新漏洞 [count={}]'.format(len(vuln_id_set)))
def sync_vuln_detail(self,
vuln_detail_dir,
vuln_detail_static_dir,
vuln_ids=None):
'''
:param vuln_detail_dir: 漏洞详情存放目录
- htmls/...
- imgs/...
:param vuln_detail_static_dir: Cscan 站点漏洞静态资源目录
:param vuln_ids: 为空/空列表时同步所有，为列表时只同步列表中指定的漏洞 ID
'''
logger.info('同步漏洞详情')
vuln_update_sql = '''UPDATE vuln SET exploit=%s WHERE vuln_id=%s'''
cursor = self.cnx.cursor()
for f in Path(path.join(vuln_detail_dir, 'htmls')).glob('**/*.html'):
vuln_id = f.name.rstrip('.html')
if vuln_ids and vuln_id not in vuln_ids:
continue
logger.info('同步 %s' % f)
try:
cursor.execute(vuln_update_sql, (f.read_text(), vuln_id))
except Exception as e:
logger.error('更新失败 %s' % f, e)
self.cnx.commit()
if vuln_detail_static_dir:
logger.info('同步静态资源')
src_path = path.join(vuln_detail_dir, 'imgs')
img_path = path.join(vuln_detail_static_dir, 'imgs')
if not path.exists(img_path):
os.makedirs(img_path)
for i in os.listdir(src_path):
fp = path.join(src_path, i)
if not path.isfile(fp):
continue
logger.debug('同步 %s' % i)
copyfile(fp, path.join(img_path, i))