def scan(self):
"""
扫描漏洞
:return:
"""
self.method = 0
# 白名单
if self.is_white_list():
logging.info("存在白名单列表中 {0}".format(self.file_path))
return False, 4000
# 特殊文件判断
if self.is_special_file():
logging.info("特殊文件 {0}".format(self.file_path))
return False, 4001
# 注释判断
if self.is_annotation():
logging.info("注释 {0}".format(self.code_content))
return False, 4002
# 仅匹配规则
if self.is_match_only_rule():
logging.info("仅匹配规则 {0}".format(self.rule_location))
found_vul = True
else:
found_vul = False
# 判断参数是否可控
if self.is_can_parse() and self.rule_repair.strip() != '':
try:
parse_instance = parse.Parse(self.rule_location,
self.file_path,
self.line_number,
self.code_content)
if parse_instance.is_controllable_param():
if parse_instance.is_repair(self.rule_repair,
self.block_repair):
logging.info("Static: repaired")
# 标记已修复
self.status = 2
self.process_vulnerabilities()
return True, 1003
else:
found_vul = True
else:
logging.info("参数不可控")
return False, 4004
except:
print(traceback.print_exc())
return False, 4005
if found_vul:
self.code_content = self.code_content.encode('unicode_escape')
if len(self.code_content) > 512:
self.code_content = self.code_content[:500] + '...'
self.process_vulnerabilities()
return True, 1002
else:
logging.critical("Exception core")
return False, 4006
def repair(self):
"""
验证扫描到的漏洞是否修复
:return: (Status, Result)
"""
self.method = 1
# 拼接绝对路径
self.file_path = self.project_directory + self.file_path
# 定位规则为空时或者行号为0,表示此类型语言(该语言所有后缀)文件都算作漏洞
if self.rule_location == '' or self.line_number == 0:
logging.info("Find special files: RID{0}".format(self.rule_id))
# 检查文件是否存在
if os.path.isfile(self.file_path) is False:
# 未找到该文件则更新漏洞状态为已修复
logging.info("已删除文件修复完成 {0}".format(self.file_path))
self.status = self.status_fixed
self.process_vulnerabilities()
return True, 1001
# 文件存在,漏洞还在
return False, 4007
# 取出触发代码(实际文件)
trigger_code = File(self.file_path).lines("{0}p".format(
self.line_number))
if trigger_code is False:
logging.critical("触发代码获取失败 {0}".format(self.code_content))
return False, 4009
self.code_content = trigger_code
# 白名单
if self.is_white_list():
self.status = self.status_fixed
self.process_vulnerabilities()
logging.info("In white list {0}".format(self.file_path))
return False, 4000
# 特殊文件判断
if self.is_special_file():
self.status = self.status_fixed
self.process_vulnerabilities()
logging.info("Special File: {0}".format(self.file_path))
return False, 4001
# 注释判断
if self.is_annotation():
self.status = self.status_fixed
self.process_vulnerabilities()
logging.info("In Annotation {0}".format(self.code_content))
return False, 4002
# 仅匹配规则
if self.is_match_only_rule():
logging.info("Only match {0}".format(self.rule_location))
found_vul = True
else:
found_vul = False
# 判断参数是否可控
if self.is_can_parse() and self.rule_repair.strip() != '':
try:
parse_instance = parse.Parse(self.rule_location,
self.file_path,
self.line_number,
self.code_content)
if parse_instance.is_controllable_param():
if parse_instance.is_repair(self.rule_repair,
self.block_repair):
logging.info("Static: repaired")
# 标记已修复
self.status = self.status_fixed
self.process_vulnerabilities()
return True, 1003
else:
found_vul = True
else:
logging.info("参数不可控")
return False, 4004
except:
print(traceback.print_exc())
return False, 4005
if found_vul:
self.code_content = self.code_content.encode('unicode_escape')
if len(self.code_content) > 512:
self.code_content = self.code_content[:500] + '...'
self.process_vulnerabilities()
return True, 1002
else:
logging.info("Not found")
return False, 4006
def analyse(self):
if self.directory is None:
logging.critical("Please set directory")
sys.exit()
logging.info('Start code static analyse...')
d = directory.Directory(self.directory)
files = d.collect_files(self.task_id)
logging.info('Scan Files: {0}, Total Time: {1}s'.format(
files['file_nums'], files['collect_time']))
ext_language = {
# Image
'.jpg': 'image',
'.png': 'image',
'.bmp': 'image',
'.gif': 'image',
'.ico': 'image',
'.cur': 'image',
# Font
'.eot': 'font',
'.otf': 'font',
'.svg': 'font',
'.ttf': 'font',
'.woff': 'font',
# CSS
'.css': 'css',
'.less': 'css',
'.scss': 'css',
'.styl': 'css',
# Media
'.mp3': 'media',
'.swf': 'media',
# Execute
'.exe': 'execute',
'.sh': 'execute',
'.dll': 'execute',
'.so': 'execute',
'.bat': 'execute',
'.pl': 'execute',
# Edit
'.swp': 'tmp',
# Cert
'.crt': 'cert',
# Text
'.txt': 'text',
'.csv': 'text',
'.md': 'markdown',
# Backup
'.zip': 'backup',
'.bak': 'backup',
'.tar': 'backup',
'.rar': 'backup',
'.tar.gz': 'backup',
'.db': 'backup',
# Config
'.xml': 'config',
'.yml': 'config',
'.spf': 'config',
'.iml': 'config',
'.manifest': 'config',
# Source
'.psd': 'source',
'.as': 'source',
# Log
'.log': 'log',
# Template
'.template': 'template',
'.tpl': 'template',
}
for ext in files:
if ext in ext_language:
logging.info('{0} - {1}'.format(ext, files[ext]))
continue
else:
logging.info(ext)
languages = CobraLanguages.query.all()
rules = CobraRules.query.filter_by(status=1).all()
extensions = None
# `grep` (`ggrep` on Mac)
grep = '/bin/grep'
# `find` (`gfind` on Mac)
find = '/bin/find'
if 'darwin' == sys.platform:
ggrep = ''
gfind = ''
for root, dir_names, file_names in os.walk(
'/usr/local/Cellar/grep'):
for filename in file_names:
if 'ggrep' == filename or 'grep' == filename:
ggrep = os.path.join(root, filename)
for root, dir_names, file_names in os.walk(
'/usr/local/Cellar/findutils'):
for filename in file_names:
if 'gfind' == filename:
gfind = os.path.join(root, filename)
if ggrep == '':
logging.critical("brew install ggrep pleases!")
sys.exit(0)
else:
grep = ggrep
if gfind == '':
logging.critical("brew install findutils pleases!")
sys.exit(0)
else:
find = gfind
"""
all vulnerabilities
vulnerabilities_all[vuln_id] = {'name': 'vuln_name', 'third_v_id': 'third_v_id'}
"""
vulnerabilities_all = {}
vulnerabilities = CobraVuls.query.all()
for v in vulnerabilities:
vulnerabilities_all[v.id] = {
'name': v.name,
'third_v_id': v.third_v_id
}
for rule in rules:
rule.regex_location = rule.regex_location.strip()
rule.regex_repair = rule.regex_repair.strip()
logging.info('Scan rule id: {0} {1} {2}'.format(
self.project_id, rule.id, rule.description))
# Filters
for language in languages:
if language.id == rule.language:
extensions = language.extensions.split('|')
if extensions is None:
logging.critical("Rule Language Error")
sys.exit(0)
# White list
white_list = []
ws = CobraWhiteList.query.filter_by(project_id=self.project_id,
rule_id=rule.id,
status=1).all()
if ws is not None:
for w in ws:
white_list.append(w.path)
try:
if rule.regex_location == "":
filters = []
for index, e in enumerate(extensions):
if index > 1:
filters.append('-o')
filters.append('-name')
filters.append('*' + e)
# Find Special Ext Files
param = [find, self.directory, "-type", "f"] + filters
else:
filters = []
for e in extensions:
filters.append('--include=*' + e)
# explode dirs
explode_dirs = ['.svn', '.cvs', '.hg', '.git', '.bzr']
for explode_dir in explode_dirs:
filters.append('--exclude-dir={0}'.format(explode_dir))
# -n Show Line number / -r Recursive / -P Perl regular expression
param = [grep, "-n", "-r", "-P"] + filters + [
rule.regex_location, self.directory
]
# logging.info(' '.join(param))
p = subprocess.Popen(param, stdout=subprocess.PIPE)
result = p.communicate()
# Exists result
if len(result[0]):
lines = str(result[0]).strip().split("\n")
for line in lines:
line = line.strip()
if line == '':
continue
if rule.regex_location == '':
# Find (special file)
file_path = line.strip().replace(
self.directory, '')
logging.debug('File: {0}'.format(file_path))
exist_result = CobraResults.query.filter_by(
project_id=self.project_id,
rule_id=rule.id,
file=file_path).first()
if exist_result is not None:
# push queue
if exist_result.status == 0:
try:
q = Queue(
self.project_name,
vulnerabilities_all[rule.vul_id]
['name'], vulnerabilities_all[
rule.vul_id]['third_v_id'],
file_path, 0, 0, exist_result.id)
q.push()
except Exception as e:
print(traceback.print_exc())
logging.critical(e.message)
logging.warning("Exists Result")
else:
vul = CobraResults(self.task_id,
self.project_id, rule.id,
file_path, 0, '', 0)
db.session.add(vul)
try:
# push queue
q = Queue(
self.project_name, vulnerabilities_all[
rule.vul_id]['name'],
vulnerabilities_all[
rule.vul_id]['third_v_id'],
file_path, 0, 0, vul.id)
q.push()
except Exception as e:
print(traceback.print_exc())
logging.critical(e.message)
else:
# Grep
line_split = line.split(':', 1)
file_path = line_split[0].strip()
if len(line_split) < 2:
logging.info("Line len < 2 {0}".format(line))
continue
code_content = line_split[1].split(':',
1)[1].strip()
line_number = line_split[1].split(':',
1)[0].strip()
if file_path in white_list or ".min.js" in file_path:
logging.info("In white list or min.js")
else:
only_match = rule.regex_location[:
1] == '(' and rule.regex_location[
-1] == ')'
"""
annotation (注释过滤)
# // /* *
Exclude:
- (rule_location) - 当定位规则左右两边为括号时不过滤注释行,比如硬编码密码
"""
match_result = re.match(
"(#)?(//)?(\*)?(/\*)?", code_content)
if match_result.group(
0) is not None and match_result.group(
0
) is not "" and only_match is not True:
logging.info("In Annotation")
else:
param_value = None
# parse file function structure
if only_match:
found_vul = True
else:
if file_path[
-3:] == 'php' and rule.regex_repair.strip(
) != '':
try:
parse_instance = parse.Parse(
rule.regex_location,
file_path, line_number,
code_content)
if parse_instance.is_controllable_param(
):
if parse_instance.is_repair(
rule.regex_repair,
rule.block_repair):
logging.info(
"Static: repaired")
continue
else:
if parse_instance.param_value is not None:
param_value = parse_instance.param_value
found_vul = True
else:
logging.info(
"Static: uncontrollable param"
)
continue
except:
print(traceback.print_exc())
found_vul = False
else:
found_vul = True
file_path = file_path.replace(
self.directory, '')
if found_vul:
logging.info('In Insert')
exist_result = CobraResults.query.filter_by(
project_id=self.project_id,
rule_id=rule.id,
file=file_path,
line=line_number).first()
if exist_result is not None:
logging.info("Exists Result")
# push queue
if exist_result.status == 0:
try:
q = Queue(
self.project_name,
vulnerabilities_all[
rule.vul_id]
['name'],
vulnerabilities_all[
rule.vul_id]
['third_v_id'],
file_path, line_number,
code_content,
exist_result.id)
q.push()
except Exception as e:
print(
traceback.print_exc())
logging.critical(e.message)
else:
code_content = code_content.encode(
'unicode_escape')
if len(code_content) > 512:
code_content = code_content[:
500] + '...'
code_content = '# Trigger\r' + code_content
if param_value is not None:
code_content = '# Param\r' + param_value + '\r//\r// ------ Continue... ------\r//\r' + code_content
logging.debug(
'File: {0}:{1} {2}'.format(
file_path, line_number,
code_content))
vul = CobraResults(
self.task_id, self.project_id,
rule.id, file_path,
line_number, code_content, 0)
db.session.add(vul)
db.session.commit()
try:
q = Queue(
self.project_name,
vulnerabilities_all[
rule.vul_id]['name'],
vulnerabilities_all[
rule.vul_id]
['third_v_id'], file_path,
line_number, code_content,
vul.id)
q.push()
except Exception as e:
print(traceback.print_exc())
logging.critical(e.message)
logging.info(
'Insert Results Success')
else:
logging.info('Not Found')
except Exception as e:
print(traceback.print_exc())
logging.critical('Error calling grep: ' + str(e))
# Set End Time For Task
t = CobraTaskInfo.query.filter_by(id=self.task_id).first()
t.status = 2
t.file_count = files['file_nums']
t.time_end = int(time.time())
t.time_consume = t.time_end - t.time_start
t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime())
try:
db.session.add(t)
db.session.commit()
except Exception as e:
logging.critical("Set start time failed:" + e.message)
logging.info("Scan Done")
def repair(self):
"""
Scan vulnerabilities is repair
:flow:
- exist file [add]
- test file
- whitelist file
- special file
- annotation
- rule
:return: (Status, Result)
"""
self.method = 1
# Full path
self.file_path = self.project_directory + self.file_path
"""
When the targeting rule is empty or the line number is 0, it means that this type of language (all suffixes in that language) is counted as a vulnerability
Their repair method is only one: delete the file
"""
if self.rule_location == '' or self.line_number == 0:
logging.info("Find special files: RID{0}".format(self.rule_id))
# Check if the file exists
if os.path.isfile(self.file_path) is False:
# If the file is not found, the vulnerability state is fixed
logging.info("Deleted file repair is complete {0}".format(self.file_path))
self.status = self.status_fixed
self.repair_code = self.repair_code_not_exist_file
self.process_vulnerabilities()
return
else:
return
# Not exist file
if os.path.isfile(self.file_path) is False:
self.status = self.status_fixed
self.repair_code = self.repair_code_not_exist_file
self.process_vulnerabilities()
return
# Test file
if self.is_test_file():
self.status = self.status_fixed
self.repair_code = self.repair_code_test_file
self.process_vulnerabilities()
return
"""
Cobra Skip
@cobra const
`@[cC][oO][bB][rR][aA]\s*[cC][oO][nN][sS][tT]`
"""
file_content = File(self.file_path).read_file()
ret_regex_const = re.findall(r'@[cC][oO][bB][rR][aA]\s*[cC][oO][nN][sS][tT]', file_content)
if len(ret_regex_const) > 0:
self.status = self.status_fixed
self.repair_code = self.repair_code_const_file
self.process_vulnerabilities()
return
"""
@cobra third-party
`@[cC][oO][bB][rR][aA]\s*[tT][hH][iI][rR][dD]-[pP][aA][rR][tT][yY]`
"""
ret_regex_third_party = re.findall(r'@[cC][oO][bB][rR][aA]\s*[tT][hH][iI][rR][dD]-[pP][aA][rR][tT][yY]', file_content)
if len(ret_regex_third_party) > 0:
self.status = self.status_fixed
self.repair_code = self.repair_code_third_party
self.process_vulnerabilities()
return
# Remove the trigger code (actual file)
trigger_code = File(self.file_path).lines("{0}p".format(self.line_number))
if trigger_code is False:
logging.critical("Failed to fetch the trigger code {0}".format(self.code_content))
self.status = self.status_fixed
self.repair_code = self.repair_code_empty_code
self.process_vulnerabilities()
return
self.code_content = trigger_code
# Whitelist
if self.is_white_list():
self.status = self.status_fixed
self.repair_code = self.repair_code_whitelist
self.process_vulnerabilities()
logging.info("In white list {0}".format(self.file_path))
return
# Special file
if self.is_special_file():
self.status = self.status_fixed
self.repair_code = self.repair_code_special_file
self.process_vulnerabilities()
logging.info("Special File: {0}".format(self.file_path))
return
# Annotation
if self.is_annotation():
self.status = self.status_fixed
self.repair_code = self.repair_code_annotation
self.process_vulnerabilities()
logging.info("In Annotation {0}".format(self.code_content))
return
# Modify
ret_regex = re.findall(self.rule_location, trigger_code.strip())
if len(ret_regex) == 0:
self.status = self.status_fixed
self.repair_code = self.repair_code_modify
self.process_vulnerabilities()
return
# Fixed
if self.is_can_parse() and self.rule_repair.strip() != '':
try:
parse_instance = parse.Parse(self.rule_location, self.file_path, self.line_number, self.code_content)
if parse_instance.is_repair(self.rule_repair, self.block_repair):
logging.info("Static: repaired")
# Fixed
self.status = self.status_fixed
self.repair_code = self.repair_code_fixed
self.process_vulnerabilities()
return
else:
logging.critical("[repair] not fixed")
return
except:
logging.info(traceback.print_exc())
return
def scan(self):
"""
Scan vulnerabilities
:flow:
- whitelist file
- special file
- test file
- annotation
- rule
:return:
"""
self.method = 0
if self.is_white_list():
logging.info("Whitelist file {0}".format(self.file_path))
return False, 4000
if self.is_special_file():
logging.info("Special file: {0}".format(self.file_path))
return False, 4001
if self.is_test_file():
logging.info("Test file: {0}".format(self.file_path))
return False, 4007
if self.is_annotation():
logging.info("Annotation {0}".format(self.code_content))
return False, 4002
if self.is_match_only_rule():
logging.info("Only Rule: {0}".format(self.rule_location))
found_vul = True
else:
found_vul = False
# parameter is controllable
if self.is_can_parse() and self.rule_repair.strip() != '':
try:
parse_instance = parse.Parse(self.rule_location, self.file_path, self.line_number, self.code_content)
if parse_instance.is_controllable_param():
if parse_instance.is_repair(self.rule_repair, self.block_repair):
# fixed
return True, 1003
else:
found_vul = True
else:
logging.info("parameter is not controllable")
return False, 4004
except:
print(traceback.print_exc())
return False, 4005
if found_vul:
self.code_content = self.code_content.encode('unicode_escape')
if len(self.code_content) > 512:
self.code_content = self.code_content[:500]
self.status = self.status_init
self.repair_code = self.repair_code_init
self.process_vulnerabilities()
return True, 1002
else:
logging.info("Not found vulnerabilities")
return False, 4006
def repair(self):
"""
Scan vulnerabilities is repair
:flow:
- exist file [add]
- test file
- whitelist file
- special file
- annotation
- rule
:return: (Status, Result)
"""
self.method = 1
# Full path
self.file_path = self.project_directory + self.file_path
"""
定位规则为空时或者行号为0,表示此类型语言(该语言所有后缀)文件都算作漏洞
他们的修复方法只有一个:删除文件
"""
if self.rule_location == '' or self.line_number == 0:
logging.info("Find special files: RID{0}".format(self.rule_id))
# 检查文件是否存在
if os.path.isfile(self.file_path) is False:
# 未找到该文件则更新漏洞状态为已修复
logging.info("已删除文件修复完成 {0}".format(self.file_path))
self.status = self.status_fixed
self.repair_code = self.repair_code_not_exist_file
self.process_vulnerabilities()
return
else:
return
# Not exist file
if os.path.isfile(self.file_path) is False:
self.status = self.status_fixed
self.repair_code = self.repair_code_not_exist_file
self.process_vulnerabilities()
return
# Test file
if self.is_test_file():
self.status = self.status_fixed
self.repair_code = self.repair_code_test_file
self.process_vulnerabilities()
return
"""
Cobra Skip
@cobra const
`@[cC][oO][bB][rR][aA]\s*[cC][oO][nN][sS][tT]`
"""
file_content = File(self.file_path).read_file()
ret_regex_const = re.findall(
r'@[cC][oO][bB][rR][aA]\s*[cC][oO][nN][sS][tT]', file_content)
if len(ret_regex_const) > 0:
self.status = self.status_fixed
self.repair_code = self.repair_code_const_file
self.process_vulnerabilities()
return
"""
@cobra third-party
`@[cC][oO][bB][rR][aA]\s*[tT][hH][iI][rR][dD]-[pP][aA][rR][tT][yY]`
"""
ret_regex_third_party = re.findall(
r'@[cC][oO][bB][rR][aA]\s*[tT][hH][iI][rR][dD]-[pP][aA][rR][tT][yY]',
file_content)
if len(ret_regex_third_party) > 0:
self.status = self.status_fixed
self.repair_code = self.repair_code_third_party
self.process_vulnerabilities()
return
# 取出触发代码(实际文件)
trigger_code = File(self.file_path).lines("{0}p".format(
self.line_number))
if trigger_code is False:
logging.critical("触发代码获取失败 {0}".format(self.code_content))
self.status = self.status_fixed
self.repair_code = self.repair_code_empty_code
self.process_vulnerabilities()
return
self.code_content = trigger_code
# Whitelist
if self.is_white_list():
self.status = self.status_fixed
self.repair_code = self.repair_code_whitelist
self.process_vulnerabilities()
logging.info("In white list {0}".format(self.file_path))
return
# Special file
if self.is_special_file():
self.status = self.status_fixed
self.repair_code = self.repair_code_special_file
self.process_vulnerabilities()
logging.info("Special File: {0}".format(self.file_path))
return
# Annotation
if self.is_annotation():
self.status = self.status_fixed
self.repair_code = self.repair_code_annotation
self.process_vulnerabilities()
logging.info("In Annotation {0}".format(self.code_content))
return
# Modify
ret_regex = re.findall(self.rule_location, trigger_code.strip())
if len(ret_regex) == 0:
self.status = self.status_fixed
self.repair_code = self.repair_code_modify
self.process_vulnerabilities()
return
# Fixed
if self.is_can_parse() and self.rule_repair.strip() != '':
try:
parse_instance = parse.Parse(self.rule_location,
self.file_path, self.line_number,
self.code_content)
if parse_instance.is_repair(self.rule_repair,
self.block_repair):
logging.info("Static: repaired")
# Fixed
self.status = self.status_fixed
self.repair_code = self.repair_code_fixed
self.process_vulnerabilities()
return
else:
logging.critical("[repair] not fixed")
return
except:
logging.info(traceback.print_exc())
return
def scan(self):
"""
Scan vulnerabilities
:flow:
- whitelist file
- special file
- test file
- annotation
- rule
:return:
"""
self.method = 0
if self.is_white_list():
self.log('info', "[RET] Whitelist\r\n")
return self.data
if self.is_special_file():
self.log('info', "[RET] Special File\r\n")
return self.data
if self.is_test_file():
self.log('info', "[RET] Test File\r\n")
return self.data
if self.is_annotation():
self.log('info', "[RET] Annotation\r\n")
return self.data
if self.is_match_only_rule():
self.log('info', "Only Rule: {0}".format(self.rule_location))
found_vul = True
else:
found_vul = False
# parameter is controllable
if self.is_can_parse() and self.rule_repair.strip() != '':
try:
parse_instance = parse.Parse(self.rule_location,
self.file_path,
self.line_number,
self.code_content)
param_is_controllable, data = parse_instance.is_controllable_param(
)
if param_is_controllable:
self.data += data
self.log('info', '[RET] Param is controllable\r\n')
is_repair, data = parse_instance.is_repair(
self.rule_repair, self.block_repair)
self.data += data
if is_repair:
# fixed
self.log('info', '[RET] Vulnerability Fixed\r\n')
return self.data
else:
self.log('info', 'Repair: Not fixed')
found_vul = True
else:
self.data += data
self.log('info', '[RET] Param Not Controllable\r\n')
return self.data
except:
traceback.print_exc()
return self.data
if found_vul:
self.code_content = self.code_content.encode('unicode_escape')
if len(self.code_content) > 512:
self.code_content = self.code_content[:500]
self.status = self.status_init
self.repair_code = self.repair_code_init
self.process_vulnerabilities()
return self.data
else:
self.log('info', "[RET] Not found vulnerability\r\n")
return self.data
