def collect_files(self, task_id=None):
t1 = time.clock()
self.files(self.path)
self.result['no_extension'] = {'file_count': 0, 'file_list': []}
for extension, values in self.type_nums.items():
extension = extension.strip()
self.result[extension] = {'file_count': len(values), 'file_list': []}
# .php : 123
logging.debug('{0} : {1}'.format(extension, len(values)))
if task_id is not None:
# Store
ext = CobraExt(task_id, extension, len(values))
db.session.add(ext)
for f in self.file:
es = f.split(os.extsep)
if len(es) >= 2:
# Exists Extension
# os.extsep + es[len(es) - 1]
if f.endswith(extension):
self.result[extension]['file_list'].append(f)
else:
# Didn't have extension
self.result['no_extension']['file_count'] = int(self.result['no_extension']['file_count']) + 1
self.result['no_extension']['file_list'].append(f)
if task_id is not None:
db.session.commit()
t2 = time.clock()
self.result['file_nums'] = self.file_id
self.result['collect_time'] = t2 - t1
return self.result
def framework(self):
"""
Detection framework for project
:param: framework | language
:return: self.rules['name']
"""
for rule in self.rules:
rules_types = ['file', 'directory']
rules_count = len(rule['rules'])
rules_completed = 0
logging.info("------ {0} (C: {1})".format(rule['name'], rules_count))
for rule_type in rules_types:
if rule_type in rule['rules']:
target = os.path.join(self.project_directory, rule['rules'][rule_type])
logging.debug('{0}: {1}'.format(rule_type, target))
if rule_type == 'file':
if os.path.isfile(target):
rules_completed += 1
elif rule_type == 'directory':
if os.path.isdir(target):
rules_completed += 1
if rules_completed == rules_count:
logging.info("Framework: {0}".format(rule['name']))
return rule['name'], rule['language']
return '', ''
def pull(self):
"""Pull a repo from repo_address and repo_directory"""
logging.info('Start Pull Repo...')
if not self.__check_exist():
return False, 'No local repo exist. Please clone first.'
# change work directory to the repo
repo_dir = self.repo_directory
logging.debug('cd directory: {0}'.format(repo_dir))
os.chdir(repo_dir)
cmd = 'git pull origin master'
p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
(pull_out, pull_err) = p.communicate()
logging.info(pull_out)
logging.info(pull_err)
self.parse_err(pull_err)
pull_err = pull_err.replace('{0}:{1}'.format(self.repo_username, self.repo_password), '')
# change work directory back.
os.chdir(repo_dir)
if 'Updating' in pull_out or 'up-to-date' in pull_out:
logging.info('Pull done.')
return True, None
else:
return False, pull_err
def log(self, level, message, test=True):
if test:
self.data.append('[{0}] {1}'.format(level.upper(), message))
if level == 'critical':
logging.critical(message)
elif level == 'warning':
logging.warning(message)
elif level == 'info':
logging.info(message)
elif level == 'debug':
logging.debug(message)
elif level == 'error':
logging.error(message)
def files(self, directory, level=1):
if level == 1:
logging.debug(directory)
for filename in os.listdir(directory):
path = os.path.join(directory, filename)
# Directory Structure
# logging.debug('| ' * (level - 1) + '|--' + filename)
if os.path.isdir(path):
self.files(path, level + 1)
if os.path.isfile(path):
# Statistic File Type Count
file_name, file_extension = os.path.splitext(path)
self.type_nums.setdefault(file_extension.lower(), []).append(filename)
path = path.replace(self.path, '')
self.file.append(path)
self.file_id += 1
logging.debug("{0}, {1}".format(self.file_id, path))
def count_by_time(start, end):
count = db.session.query(
func.count(CobraResults.id).label('count'), CobraResults.status
).filter(
CobraResults.created_at >= '{start} 00:00:00'.format(start=start),
CobraResults.created_at <= '{end} 23:59:59'.format(end=end),
# Active project
CobraProjects.status > 0,
CobraResults.project_id == CobraProjects.id
).group_by(CobraResults.status).all()
logging.debug('VT {start} {end} {count}'.format(start=start, end=end, count=count))
c_dict = {}
for ci in count:
count, status = ci
c_dict[status] = count
if 0 not in c_dict:
c_dict[0] = 0
if 1 not in c_dict:
c_dict[1] = 0
if 2 not in c_dict:
c_dict[2] = 0
return c_dict
def count_by_time(start, end, t='task'):
filter_group = (
CobraTaskInfo.created_at >= '{0} 00:00:00'.format(start),
CobraTaskInfo.created_at <= '{0} 23:59:59'.format(end),
# Active project
CobraProjects.status > 0,
CobraProjects.repository == CobraTaskInfo.target
)
count = 0
if t == 'task':
count = db.session.query(
func.count(CobraTaskInfo.id).label('count')
).filter(
*filter_group
).first()
elif t == 'project':
count = db.session.query(
func.count(func.distinct(CobraTaskInfo.target)).label('count')
).filter(
*filter_group
).first()
elif t == 'line':
count = db.session.query(
func.sum(CobraTaskInfo.code_number).label('count')
).filter(
*filter_group
).first()
elif t == 'file':
count = db.session.query(
func.sum(CobraTaskInfo.file_count).label('count')
).filter(
*filter_group
).first()
if count[0] is None:
return 0
else:
logging.debug('SD {t} {start} {end} {count}'.format(start=start, end=end, t=t, count=int(count[0])))
return int(count[0])
def __init__(self, filename, current_version=None, online_version=None):
self.filename = filename
self.current_version = current_version
self.online_version = online_version
self.username = config.Config('svn', 'username').value
self.password = config.Config('svn', 'password').value
# Test SVN
cmd = self.svn + " info --no-auth-cache --non-interactive --username='******' --password='******' %s" % (
self.username, self.password, self.filename)
p = subprocess.Popen(cmd,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
(diff_out, diff_err) = p.communicate()
if len(diff_err) == 0:
logging.debug("svn diff success")
elif 'authorization failed' in diff_err:
logging.warning("svn diff auth failed")
sys.exit(1)
elif 'Not a valid URL' in diff_err:
logging.warning("svn diff url not a valid")
sys.exit(1)
def __init__(self, filename, current_version=None, online_version=None):
self.filename = filename
self.current_version = current_version
self.online_version = online_version
self.username = config.Config('svn', 'username').value
self.password = config.Config('svn', 'password').value
# Test SVN
cmd = self.svn + " info --no-auth-cache --non-interactive --username='******' --password='******' %s" % (
self.username,
self.password,
self.filename
)
p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
(diff_out, diff_err) = p.communicate()
if len(diff_err) == 0:
logging.debug("svn diff success")
elif 'authorization failed' in diff_err:
logging.warning("svn diff auth failed")
sys.exit(1)
elif 'Not a valid URL' in diff_err:
logging.warning("svn diff url not a valid")
sys.exit(1)
def send_email(report_file=''):
report_file = report_file or get_last_report()
config = Config()
email_config = config.get_email_config()
smtp_server = email_config.get("smtp_server") or 'mail.163.com'
smtp_user = email_config.get('smtp_user') or '*****@*****.**'
smtp_password = email_config.get('smtp_password') or 'hanzhichao123'
subject = email_config.get('subject') or 'Test Report'
body = email_config.get('body') or 'hi, all<br>Test complete, please see the attachments'
receivers = email_config.get('receivers')
msg = MIMEMultipart()
msg.attach(MIMEText(body, 'html', 'utf-8'))
msg['From'] = smtp_user
msg['To'] = ''
msg['Subject'] = Header(subject, 'utf-8')
att1 = MIMEText(open(report_file, 'rb').read(), 'base64', 'utf-8')
att1["Content-Type"] = 'application/octet-stream'
att1["Content-Disposition"] = 'attachment; filename="{}"'.format(os.path.basename(report_file))
msg.attach(att1)
try:
logging.debug("连接SMTP服务器: {}".format(smtp_server))
smtp = smtplib.SMTP(smtp_server) # 从配置文件中读取
logging.debug("登录SMTP服务器 用户名: {} 密码: {}".format(smtp_user, smtp_password))
smtp.login(smtp_user, smtp_password) # 从配置文件中读取
for receiver in receivers:
msg['To'] = receiver
logging.debug("由{}发送邮件给: {}".format(smtp_user, receiver))
smtp.sendmail(smtp_user, receiver, msg.as_string())
logging.info("邮件发送完成！")
except Exception as e:
logging.error(str(e))
finally:
smtp.quit()
def click(self, *args):
"""点击元素，支持id,class name,xpath, text, msg, 点击不到则截图并报错"""
logging.debug("点击: {}".format(args))
self.find(*args).click()
def is_controllable_param(self):
"""
参数是否可控
:return:
"""
param_name = re.findall(self.rule, self.code)
if len(param_name) == 1:
param_name = param_name[0].strip()
self.param_name = param_name
logging.debug('参数: `{0}`'.format(param_name))
# 固定字符串判断
regex_string = self.regex[self.language]['string']
string = re.findall(regex_string, param_name)
if len(string) >= 1 and string[0] != '':
logging.debug("是否字符串: 是")
logging.info("返回: 不可控 (字符串)")
return False
logging.debug("是否字符串: 否")
# 变量判断
if param_name[:1] == '$':
logging.debug("参数是否变量: 是")
# 获取参数赋值代码块
param_block_code = self.block_code(0)
if param_block_code is False:
logging.debug("向上搜索参数区块代码: 未找到")
logging.info("返回: 可控 (代码未找到)")
return True
logging.debug("向上搜索参数区块代码: {0}".format(param_block_code))
# 外部取参赋值
"""
# Need match
$url = $_GET['test'];
$url = $_POST['test'];
$url = $_REQUEST['test'];
$url = $_SERVER['user_agent'];
# Don't match
$url = $_SERVER
$url = $testsdf;
"""
regex_get_param = r'({0}\s*=\s*\$_[GET|POST|REQUEST|SERVER]+(?:\[))'.format(
re.escape(param_name))
regex_get_param_result = re.findall(regex_get_param,
param_block_code)
if len(regex_get_param_result) >= 1:
self.param_value = regex_get_param_result[0]
logging.debug("参数是否直接取自外部: 是")
logging.info("返回: 可控(取外部入参)")
return True
logging.debug("参数是否直接取自外部入参: 否")
# 函数入参
regex_function_param = r'(function\s*\w+\s*\(.*{0})'.format(
re.escape(param_name))
regex_function_param_result = re.findall(
regex_function_param, param_block_code)
if len(regex_function_param_result) >= 1:
self.param_value = regex_function_param_result[0]
logging.debug("参数是否函数入参: 是")
logging.info("返回: 可控(函数入参)")
return True
logging.debug("参数是否直接函数入参: 否")
# 常量赋值
uc_rule = r'{0}\s?=\s?([A-Z_]*)'.format(re.escape(param_name))
uc_rule_result = re.findall(uc_rule, param_block_code)
if len(uc_rule_result) >= 1:
logging.debug("参数变量是否直接赋值常量: 是 `{0} = {1}`".format(
param_name, uc_rule_result[0]))
logging.info("返回: 不可控")
return False
logging.debug("参数变量是否直接赋值常量: 否")
# 固定字符串判断
regex_assign_string = self.regex[
self.language]['assign_string'].format(
re.escape(param_name))
string = re.findall(regex_assign_string, param_block_code)
if len(string) >= 1 and string[0] != '':
logging.debug("是否赋值字符串: 是")
logging.info("返回: 不可控 (字符串)")
return False
logging.debug("是否赋值字符串: 否")
logging.info("返回: 可控(默认情况)")
return True
else:
if self.language == 'java':
# Java 变量就是没有$
param_block_code = self.block_code(0)
if param_block_code is False:
logging.debug("向上搜索参数区块代码: 未找到")
logging.info("返回: 可控 (代码未找到)")
return True
logging.debug("向上搜索参数区块代码: {0}".format(param_block_code))
regex_assign_string = self.regex[
self.language]['assign_string'].format(
re.escape(param_name))
string = re.findall(regex_assign_string, param_block_code)
if len(string) >= 1 and string[0] != '':
logging.debug("是否赋值字符串: 是")
logging.info("返回: 不可控 (字符串)")
return False
logging.debug("是否赋值字符串: 否")
# 是否取外部参数
regex_get_param = r'String\s{0}\s=\s\w+\.getParameter(.*)'.format(
re.escape(param_name))
get_param = re.findall(regex_get_param, param_block_code)
if len(get_param) >= 1 and get_param[0] != '':
logging.debug("是否赋值外部取参: 是")
logging.info("返回: 不可控 (外部取参)")
return False
logging.debug("是否赋值外部取参: 否")
logging.info("返回: 可控 (变量赋值)")
return True
logging.debug("参数是否变量: 否 (没有包含$)")
logging.info("返回: 不可控(参数不为变量)")
return False
else:
logging.warning("未获取到参数名,请检查定位规则")
def analyse(self):
if self.directory is None:
logging.critical("Please set directory")
sys.exit()
logging.info('Start code static analyse...')
d = directory.Directory(self.directory)
files = d.collect_files(self.task_id)
logging.info('Scan Files: {0}, Total Time: {1}s'.format(
files['file_nums'], files['collect_time']))
ext_language = {
# Image
'.jpg': 'image',
'.png': 'image',
'.bmp': 'image',
'.gif': 'image',
'.ico': 'image',
'.cur': 'image',
# Font
'.eot': 'font',
'.otf': 'font',
'.svg': 'font',
'.ttf': 'font',
'.woff': 'font',
# CSS
'.css': 'css',
'.less': 'css',
'.scss': 'css',
'.styl': 'css',
# Media
'.mp3': 'media',
'.swf': 'media',
# Execute
'.exe': 'execute',
'.sh': 'execute',
'.dll': 'execute',
'.so': 'execute',
'.bat': 'execute',
'.pl': 'execute',
# Edit
'.swp': 'tmp',
# Cert
'.crt': 'cert',
# Text
'.txt': 'text',
'.csv': 'text',
'.md': 'markdown',
# Backup
'.zip': 'backup',
'.bak': 'backup',
'.tar': 'backup',
'.rar': 'backup',
'.tar.gz': 'backup',
'.db': 'backup',
# Config
'.xml': 'config',
'.yml': 'config',
'.spf': 'config',
'.iml': 'config',
'.manifest': 'config',
# Source
'.psd': 'source',
'.as': 'source',
# Log
'.log': 'log',
# Template
'.template': 'template',
'.tpl': 'template',
}
for ext in files:
if ext in ext_language:
logging.info('{0} - {1}'.format(ext, files[ext]))
continue
else:
logging.info(ext)
languages = CobraLanguages.query.all()
rules = CobraRules.query.filter_by(status=1).all()
extensions = None
# `grep` (`ggrep` on Mac)
grep = '/bin/grep'
# `find` (`gfind` on Mac)
find = '/bin/find'
if 'darwin' == sys.platform:
ggrep = ''
gfind = ''
for root, dir_names, file_names in os.walk(
'/usr/local/Cellar/grep'):
for filename in file_names:
if 'ggrep' == filename or 'grep' == filename:
ggrep = os.path.join(root, filename)
for root, dir_names, file_names in os.walk(
'/usr/local/Cellar/findutils'):
for filename in file_names:
if 'gfind' == filename:
gfind = os.path.join(root, filename)
if ggrep == '':
logging.critical("brew install ggrep pleases!")
sys.exit(0)
else:
grep = ggrep
if gfind == '':
logging.critical("brew install findutils pleases!")
sys.exit(0)
else:
find = gfind
"""
all vulnerabilities
vulnerabilities_all[vuln_id] = {'name': 'vuln_name', 'third_v_id': 'third_v_id'}
"""
vulnerabilities_all = {}
vulnerabilities = CobraVuls.query.all()
for v in vulnerabilities:
vulnerabilities_all[v.id] = {
'name': v.name,
'third_v_id': v.third_v_id
}
for rule in rules:
rule.regex_location = rule.regex_location.strip()
rule.regex_repair = rule.regex_repair.strip()
logging.info('------------------ RID: {0} {1} {2}'.format(
self.project_id, rule.id, rule.description))
# Filters
for language in languages:
if language.id == rule.language:
extensions = language.extensions.split('|')
if extensions is None:
logging.critical("Rule Language Error")
sys.exit(0)
# White list
white_list = []
ws = CobraWhiteList.query.filter_by(project_id=self.project_id,
rule_id=rule.id,
status=1).all()
if ws is not None:
for w in ws:
white_list.append(w.path)
try:
if rule.regex_location == "":
filters = []
for index, e in enumerate(extensions):
if index > 1:
filters.append('-o')
filters.append('-name')
filters.append('*' + e)
# Find Special Ext Files
param = [find, self.directory, "-type", "f"] + filters
else:
filters = []
for e in extensions:
filters.append('--include=*' + e)
# explode dirs
explode_dirs = ['.svn', '.cvs', '.hg', '.git', '.bzr']
for explode_dir in explode_dirs:
filters.append('--exclude-dir={0}'.format(explode_dir))
# -s suppress error messages / -n Show Line number / -r Recursive / -P Perl regular expression
param = [grep, "-s", "-n", "-r", "-P"] + filters + [
rule.regex_location, self.directory
]
logging.debug(rule.regex_location)
p = subprocess.Popen(param, stdout=subprocess.PIPE)
result = p.communicate()
# Exists result
if len(result[0]):
lines = str(result[0]).strip().split("\n")
for line in lines:
line = line.strip()
if line == '':
continue
# 处理grep结果
if ':' in line:
line_split = line.split(':', 1)
file_path = line_split[0].strip()
code_content = line_split[1].split(':',
1)[1].strip()
line_number = line_split[1].split(':',
1)[0].strip()
else:
# 搜索文件
file_path = line
code_content = ''
line_number = 0
# 核心规则校验
result_info = {
'task_id':
self.task_id,
'project_id':
self.project_id,
'project_directory':
self.directory,
'rule_id':
rule.id,
'result_id':
None,
'file_path':
file_path,
'line_number':
line_number,
'code_content':
code_content,
'third_party_vulnerabilities_name':
vulnerabilities_all[rule.vul_id]['name'],
'third_party_vulnerabilities_type':
vulnerabilities_all[rule.vul_id]['third_v_id']
}
ret_status, ret_result = Core(result_info, rule,
self.project_name,
white_list).scan()
if ret_status is False:
logging.info("扫描 R: False {0}".format(ret_result))
continue
else:
logging.info('Not Found')
except Exception as e:
print(traceback.print_exc())
logging.critical('Error calling grep: ' + str(e))
# Set End Time For Task
t = CobraTaskInfo.query.filter_by(id=self.task_id).first()
t.status = 2
t.file_count = files['file_nums']
t.time_end = int(time.time())
t.time_consume = t.time_end - t.time_start
t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime())
try:
db.session.add(t)
db.session.commit()
except Exception as e:
logging.critical("Set start time failed:" + e.message)
logging.info("Scan Done")
def process_vulnerabilities(self):
"""
Process vulnerabilities
Write vulnerabilities / change the vulnerability status / push third-party vulnerability management platform
:return:
"""
# default status
if self.status is None:
self.status = self.status_init
# Handle relative paths
self.file_path = self.file_path.replace(self.project_directory, '')
# In scan
if self.method == 0:
"""
On Scan mode(method=1) only deal to new not fixed vulnerability(status=0)
"""
if self.status == 0:
# Check whether the unpatched vulnerability has been swept
# Line number 0 when the search for special documents (not to bring the line number)
if self.line_number == 0:
exist_result = CobraResults.query.filter(
CobraResults.project_id == self.project_id,
CobraResults.rule_id == self.rule_id,
CobraResults.file == self.file_path,
).order_by(CobraResults.id.desc()).first()
else:
exist_result = CobraResults.query.filter(
CobraResults.project_id == self.project_id,
CobraResults.rule_id == self.rule_id,
CobraResults.file == self.file_path,
CobraResults.line == self.line_number,
).order_by(CobraResults.id.desc()).first()
# Not fixed vulnerabilities have not been swept before, is written to the vulnerability database
if exist_result is None:
self.insert_vulnerabilities()
else:
logging.debug("This vulnerabilities already exist!")
elif self.method == 1:
"""
On Repair (method=1)
"""
if self.status == self.status_fixed:
# View the original status of the vulnerability
exist_result = CobraResults.query.filter_by(
id=self.result_id).first()
if exist_result is not None:
if exist_result.status < self.status_fixed:
# update vulnerability status is Fixed
exist_result.status = self.status_fixed
exist_result.repair = self.repair_code
db.session.add(exist_result)
db.session.commit()
else:
logging.critical("Vulnerabilities status not < fixed")
else:
logging.critical(
"Not found vulnerabilities on repair method")
else:
logging.critical("Core method not initialize")
def try_find_element(self, *args):
"""尝试定位元素，找不到不报错"""
try:
return self.find_element(*args)
except NoSuchElementException:
logging.debug("元素: {} 未被定位到".format(args))
def type_and_search(self, *args, text=''):
"""输入并按搜索键"""
self.type(*args, text=text)
logging.debug("按搜索键")
self.driver.keyevent(84)
def type_and_enter(self, *args, text=''):
"""输入并按回车键"""
self.type(*args, text=text)
logging.debug("按回车键")
self.driver.keyevent(66)
def wait_click(self, *args):
"""等待并点击元素，定位不到报超时错误"""
logging.debug("等待点击: {}".format(args))
self.wait_element(*args).click()
def click_id(self, *args):
"""点击指定resource_id元素"""
logging.debug("点击指定id: {} 元素".format(args))
self.find_id(*args).click()
def click_text(self, *args):
"""点击指定文本元素"""
logging.debug("点击文本: {}".format(args))
self.find_text(*args).click()
def is_login(self):
if self.try_find_element(*self.MIME_ICO):
logging.debug("定位到用户头像")
return True
else:
logging.debug("未定位到用户头像")