def test_directory(self):
directory = directory.Directory(self.project)
directory.collect_files()
def analyse(self, test=False):
if self.directory is None:
self.log('critical', 'Please set directory')
sys.exit()
files = directory.Directory(self.directory).collect_files(self.task_id)
self.log('info', '**Scan Files**\r\n > Files count: `{files}`\r\n > Time consume: `{consume}s`\r\n'.format(files=files['file_nums'], consume=files['collect_time']))
ext_language = {
# Image
'.jpg': 'image',
'.png': 'image',
'.bmp': 'image',
'.gif': 'image',
'.ico': 'image',
'.cur': 'image',
# Font
'.eot': 'font',
'.otf': 'font',
'.svg': 'font',
'.ttf': 'font',
'.woff': 'font',
# CSS
'.css': 'css',
'.less': 'css',
'.scss': 'css',
'.styl': 'css',
# Media
'.mp3': 'media',
'.swf': 'media',
# Execute
'.exe': 'execute',
'.sh': 'execute',
'.dll': 'execute',
'.so': 'execute',
'.bat': 'execute',
'.pl': 'execute',
# Edit
'.swp': 'tmp',
# Cert
'.crt': 'cert',
# Text
'.txt': 'text',
'.csv': 'text',
'.md': 'markdown',
# Backup
'.zip': 'backup',
'.bak': 'backup',
'.tar': 'backup',
'.rar': 'backup',
'.tar.gz': 'backup',
'.db': 'backup',
# Config
'.xml': 'config',
'.yml': 'config',
'.spf': 'config',
'.iml': 'config',
'.manifest': 'config',
# Source
'.psd': 'source',
'.as': 'source',
# Log
'.log': 'log',
# Template
'.template': 'template',
'.tpl': 'template',
}
for ext in files:
if ext in ext_language:
self.log('info', '{0} - {1}'.format(ext, files[ext]), False)
continue
else:
self.log('info', ext, False)
explode_dirs = ['.svn', '.cvs', '.hg', '.git', '.bzr']
self.log('info', '**Rule Scan**\r\n > Global explode directory: `{dirs}`\r\n'.format(dirs=', '.join(explode_dirs)))
languages = CobraLanguages.query.all()
filter_group = (CobraRules.status == 1,)
if self.rule_id is not None:
filter_group += (CobraRules.id == self.rule_id,)
rules = CobraRules.query.filter(*filter_group).all()
extensions = None
find = tool.find
grep = tool.grep
"""
Vulnerability Types
vulnerability_types[vuln_id] = {'name': 'vuln_name', 'third_v_id': 'third_v_id'}
"""
vulnerability_types = {}
vulnerabilities = CobraVuls.query.all()
for v in vulnerabilities:
vulnerability_types[v.id] = {
'name': v.name,
'third_v_id': v.third_v_id
}
for index, rule in enumerate(rules):
rule.regex_location = rule.regex_location.strip()
rule.regex_repair = rule.regex_repair.strip()
# Filters
for language in languages:
if language.id == rule.language:
extensions = language.extensions.split('|')
if extensions is None:
self.log('critical', 'Rule language error')
sys.exit(0)
# White list
white_list = []
ws = CobraWhiteList.query.filter_by(project_id=self.project_id, rule_id=rule.id, status=1).all()
if ws is not None:
for w in ws:
white_list.append(w.path)
try:
if rule.regex_location == "":
mode = 'Find'
filters = []
for index, e in enumerate(extensions):
if index > 1:
filters.append('-o')
filters.append('-name')
filters.append('*' + e)
# Find Special Ext Files
param = [find, self.directory, "-type", "f"] + filters
else:
mode = 'Grep'
filters = []
for e in extensions:
filters.append('--include=*' + e)
# explode dirs
for explode_dir in explode_dirs:
filters.append('--exclude-dir={0}'.format(explode_dir))
# -s suppress error messages / -n Show Line number / -r Recursive / -P Perl regular expression
param = [grep, "-s", "-n", "-r", "-P"] + filters + [rule.regex_location, self.directory]
self.log('info', '**Rule Info({index})**\r\n > ID: `{rid}` \r\n > Name: `{name}` \r\n > Language: `{language}`\r\n > Rule mode:`{mode}`\r\n > Location: `{location}` \r\n > Repair: `{repair}`\r\n'.format(index=index, rid=rule.id, name=rule.description, language=extensions, mode=mode, location=rule.regex_location, repair=rule.regex_repair))
p = subprocess.Popen(param, stdout=subprocess.PIPE)
result = p.communicate()
# exists result
if len(result[0]):
lines = str(result[0]).strip().split("\n")
self.log('info', '**Founded Vulnerability**\r\n > Vulnerability Count: `{count}`\r\n'.format(count=len(lines)))
for index, line in enumerate(lines):
line = line.strip()
if line == '':
continue
# grep result
if ':' in line:
line_split = line.split(':', 1)
file_path = line_split[0].strip()
code_content = line_split[1].split(':', 1)[1].strip()
line_number = line_split[1].split(':', 1)[0].strip()
else:
# search file
file_path = line
code_content = ''
line_number = 0
# core rule check
result_info = {
'task_id': self.task_id,
'project_id': self.project_id,
'project_directory': self.directory,
'rule_id': rule.id,
'result_id': None,
'file_path': file_path,
'line_number': line_number,
'code_content': code_content,
'third_party_vulnerabilities_name': vulnerability_types[rule.vul_id]['name'],
'third_party_vulnerabilities_type': vulnerability_types[rule.vul_id]['third_v_id']
}
self.data += Core(result_info, rule, self.project_name, white_list, test=True, index=index).scan()
else:
self.log('info', 'Not Found')
except Exception as e:
traceback.print_exc()
self.log('critical', 'Error calling grep: ' + str(e))
if not test:
# set end time for task
t = CobraTaskInfo.query.filter_by(id=self.task_id).first()
t.status = 2
t.file_count = files['file_nums']
t.time_end = int(time.time())
t.time_consume = t.time_end - t.time_start
t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime())
try:
db.session.add(t)
db.session.commit()
except Exception as e:
self.log('critical', "Set start time failed:" + e.message)
self.log('info', "[END] Scan")
return self.data
def analyse(self):
if self.directory is None:
logging.critical("Please set directory")
sys.exit()
logging.info('Start code static analyse...')
d = directory.Directory(self.directory)
files = d.collect_files(self.task_id)
logging.info('Scan Files: {0}, Total Time: {1}s'.format(
files['file_nums'], files['collect_time']))
ext_language = {
# Image
'.jpg': 'image',
'.png': 'image',
'.bmp': 'image',
'.gif': 'image',
'.ico': 'image',
'.cur': 'image',
# Font
'.eot': 'font',
'.otf': 'font',
'.svg': 'font',
'.ttf': 'font',
'.woff': 'font',
# CSS
'.css': 'css',
'.less': 'css',
'.scss': 'css',
'.styl': 'css',
# Media
'.mp3': 'media',
'.swf': 'media',
# Execute
'.exe': 'execute',
'.sh': 'execute',
'.dll': 'execute',
'.so': 'execute',
'.bat': 'execute',
'.pl': 'execute',
# Edit
'.swp': 'tmp',
# Cert
'.crt': 'cert',
# Text
'.txt': 'text',
'.csv': 'text',
'.md': 'markdown',
# Backup
'.zip': 'backup',
'.bak': 'backup',
'.tar': 'backup',
'.rar': 'backup',
'.tar.gz': 'backup',
'.db': 'backup',
# Config
'.xml': 'config',
'.yml': 'config',
'.spf': 'config',
'.iml': 'config',
'.manifest': 'config',
# Source
'.psd': 'source',
'.as': 'source',
# Log
'.log': 'log',
# Template
'.template': 'template',
'.tpl': 'template',
}
for ext in files:
if ext in ext_language:
logging.info('{0} - {1}'.format(ext, files[ext]))
continue
else:
logging.info(ext)
languages = CobraLanguages.query.all()
rules = CobraRules.query.filter_by(status=1).all()
extensions = None
# `grep` (`ggrep` on Mac)
grep = '/bin/grep'
# `find` (`gfind` on Mac)
find = '/bin/find'
if 'darwin' == sys.platform:
ggrep = ''
gfind = ''
for root, dir_names, file_names in os.walk(
'/usr/local/Cellar/grep'):
for filename in file_names:
if 'ggrep' == filename or 'grep' == filename:
ggrep = os.path.join(root, filename)
for root, dir_names, file_names in os.walk(
'/usr/local/Cellar/findutils'):
for filename in file_names:
if 'gfind' == filename:
gfind = os.path.join(root, filename)
if ggrep == '':
logging.critical("brew install ggrep pleases!")
sys.exit(0)
else:
grep = ggrep
if gfind == '':
logging.critical("brew install findutils pleases!")
sys.exit(0)
else:
find = gfind
"""
all vulnerabilities
vulnerabilities_all[vuln_id] = {'name': 'vuln_name', 'third_v_id': 'third_v_id'}
"""
vulnerabilities_all = {}
vulnerabilities = CobraVuls.query.all()
for v in vulnerabilities:
vulnerabilities_all[v.id] = {
'name': v.name,
'third_v_id': v.third_v_id
}
for rule in rules:
rule.regex_location = rule.regex_location.strip()
rule.regex_repair = rule.regex_repair.strip()
logging.info('Scan rule id: {0} {1} {2}'.format(
self.project_id, rule.id, rule.description))
# Filters
for language in languages:
if language.id == rule.language:
extensions = language.extensions.split('|')
if extensions is None:
logging.critical("Rule Language Error")
sys.exit(0)
# White list
white_list = []
ws = CobraWhiteList.query.filter_by(project_id=self.project_id,
rule_id=rule.id,
status=1).all()
if ws is not None:
for w in ws:
white_list.append(w.path)
try:
if rule.regex_location == "":
filters = []
for index, e in enumerate(extensions):
if index > 1:
filters.append('-o')
filters.append('-name')
filters.append('*' + e)
# Find Special Ext Files
param = [find, self.directory, "-type", "f"] + filters
else:
filters = []
for e in extensions:
filters.append('--include=*' + e)
# explode dirs
explode_dirs = ['.svn', '.cvs', '.hg', '.git', '.bzr']
for explode_dir in explode_dirs:
filters.append('--exclude-dir={0}'.format(explode_dir))
# -n Show Line number / -r Recursive / -P Perl regular expression
param = [grep, "-n", "-r", "-P"] + filters + [
rule.regex_location, self.directory
]
# logging.info(' '.join(param))
p = subprocess.Popen(param, stdout=subprocess.PIPE)
result = p.communicate()
# Exists result
if len(result[0]):
lines = str(result[0]).strip().split("\n")
for line in lines:
line = line.strip()
if line == '':
continue
if rule.regex_location == '':
# Find (special file)
file_path = line.strip().replace(
self.directory, '')
logging.debug('File: {0}'.format(file_path))
exist_result = CobraResults.query.filter_by(
project_id=self.project_id,
rule_id=rule.id,
file=file_path).first()
if exist_result is not None:
# push queue
if exist_result.status == 0:
try:
q = Queue(
self.project_name,
vulnerabilities_all[rule.vul_id]
['name'], vulnerabilities_all[
rule.vul_id]['third_v_id'],
file_path, 0, 0, exist_result.id)
q.push()
except Exception as e:
print(traceback.print_exc())
logging.critical(e.message)
logging.warning("Exists Result")
else:
vul = CobraResults(self.task_id,
self.project_id, rule.id,
file_path, 0, '', 0)
db.session.add(vul)
try:
# push queue
q = Queue(
self.project_name, vulnerabilities_all[
rule.vul_id]['name'],
vulnerabilities_all[
rule.vul_id]['third_v_id'],
file_path, 0, 0, vul.id)
q.push()
except Exception as e:
print(traceback.print_exc())
logging.critical(e.message)
else:
# Grep
line_split = line.split(':', 1)
file_path = line_split[0].strip()
if len(line_split) < 2:
logging.info("Line len < 2 {0}".format(line))
continue
code_content = line_split[1].split(':',
1)[1].strip()
line_number = line_split[1].split(':',
1)[0].strip()
if file_path in white_list or ".min.js" in file_path:
logging.info("In white list or min.js")
else:
only_match = rule.regex_location[:
1] == '(' and rule.regex_location[
-1] == ')'
"""
annotation (注释过滤)
# // /* *
Exclude:
- (rule_location) - 当定位规则左右两边为括号时不过滤注释行,比如硬编码密码
"""
match_result = re.match(
"(#)?(//)?(\*)?(/\*)?", code_content)
if match_result.group(
0) is not None and match_result.group(
0
) is not "" and only_match is not True:
logging.info("In Annotation")
else:
param_value = None
# parse file function structure
if only_match:
found_vul = True
else:
if file_path[
-3:] == 'php' and rule.regex_repair.strip(
) != '':
try:
parse_instance = parse.Parse(
rule.regex_location,
file_path, line_number,
code_content)
if parse_instance.is_controllable_param(
):
if parse_instance.is_repair(
rule.regex_repair,
rule.block_repair):
logging.info(
"Static: repaired")
continue
else:
if parse_instance.param_value is not None:
param_value = parse_instance.param_value
found_vul = True
else:
logging.info(
"Static: uncontrollable param"
)
continue
except:
print(traceback.print_exc())
found_vul = False
else:
found_vul = True
file_path = file_path.replace(
self.directory, '')
if found_vul:
logging.info('In Insert')
exist_result = CobraResults.query.filter_by(
project_id=self.project_id,
rule_id=rule.id,
file=file_path,
line=line_number).first()
if exist_result is not None:
logging.info("Exists Result")
# push queue
if exist_result.status == 0:
try:
q = Queue(
self.project_name,
vulnerabilities_all[
rule.vul_id]
['name'],
vulnerabilities_all[
rule.vul_id]
['third_v_id'],
file_path, line_number,
code_content,
exist_result.id)
q.push()
except Exception as e:
print(
traceback.print_exc())
logging.critical(e.message)
else:
code_content = code_content.encode(
'unicode_escape')
if len(code_content) > 512:
code_content = code_content[:
500] + '...'
code_content = '# Trigger\r' + code_content
if param_value is not None:
code_content = '# Param\r' + param_value + '\r//\r// ------ Continue... ------\r//\r' + code_content
logging.debug(
'File: {0}:{1} {2}'.format(
file_path, line_number,
code_content))
vul = CobraResults(
self.task_id, self.project_id,
rule.id, file_path,
line_number, code_content, 0)
db.session.add(vul)
db.session.commit()
try:
q = Queue(
self.project_name,
vulnerabilities_all[
rule.vul_id]['name'],
vulnerabilities_all[
rule.vul_id]
['third_v_id'], file_path,
line_number, code_content,
vul.id)
q.push()
except Exception as e:
print(traceback.print_exc())
logging.critical(e.message)
logging.info(
'Insert Results Success')
else:
logging.info('Not Found')
except Exception as e:
print(traceback.print_exc())
logging.critical('Error calling grep: ' + str(e))
# Set End Time For Task
t = CobraTaskInfo.query.filter_by(id=self.task_id).first()
t.status = 2
t.file_count = files['file_nums']
t.time_end = int(time.time())
t.time_consume = t.time_end - t.time_start
t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime())
try:
db.session.add(t)
db.session.commit()
except Exception as e:
logging.critical("Set start time failed:" + e.message)
logging.info("Scan Done")
def files(self):
d = directory.Directory(self.project_path)
self.target_files = d.collect_files()
return self.target_files
def analyse(self):
if self.directory is None:
logging.critical("Please set directory")
sys.exit()
logging.info('Start code static analyse...')
d = directory.Directory(self.directory)
files = d.collect_files(self.task_id)
logging.info('Scan Files: {0}, Total Time: {1}s'.format(
files['file_nums'], files['collect_time']))
ext_language = {
# Image
'.jpg': 'image',
'.png': 'image',
'.bmp': 'image',
'.gif': 'image',
'.ico': 'image',
'.cur': 'image',
# Font
'.eot': 'font',
'.otf': 'font',
'.svg': 'font',
'.ttf': 'font',
'.woff': 'font',
# CSS
'.css': 'css',
'.less': 'css',
'.scss': 'css',
'.styl': 'css',
# Media
'.mp3': 'media',
'.swf': 'media',
# Execute
'.exe': 'execute',
'.sh': 'execute',
'.dll': 'execute',
'.so': 'execute',
'.bat': 'execute',
'.pl': 'execute',
# Edit
'.swp': 'tmp',
# Cert
'.crt': 'cert',
# Text
'.txt': 'text',
'.csv': 'text',
'.md': 'markdown',
# Backup
'.zip': 'backup',
'.bak': 'backup',
'.tar': 'backup',
'.rar': 'backup',
'.tar.gz': 'backup',
'.db': 'backup',
# Config
'.xml': 'config',
'.yml': 'config',
'.spf': 'config',
'.iml': 'config',
'.manifest': 'config',
# Source
'.psd': 'source',
'.as': 'source',
# Log
'.log': 'log',
# Template
'.template': 'template',
'.tpl': 'template',
}
for ext in files:
if ext in ext_language:
logging.info('{0} - {1}'.format(ext, files[ext]))
continue
else:
logging.info(ext)
languages = CobraLanguages.query.all()
rules = CobraRules.query.filter_by(status=1).all()
extensions = None
# `grep` (`ggrep` on Mac)
grep = '/bin/grep'
# `find` (`gfind` on Mac)
find = '/bin/find'
if 'darwin' == sys.platform:
ggrep = ''
gfind = ''
for root, dir_names, file_names in os.walk(
'/usr/local/Cellar/grep'):
for filename in file_names:
if 'ggrep' == filename or 'grep' == filename:
ggrep = os.path.join(root, filename)
for root, dir_names, file_names in os.walk(
'/usr/local/Cellar/findutils'):
for filename in file_names:
if 'gfind' == filename:
gfind = os.path.join(root, filename)
if ggrep == '':
logging.critical("brew install ggrep pleases!")
sys.exit(0)
else:
grep = ggrep
if gfind == '':
logging.critical("brew install findutils pleases!")
sys.exit(0)
else:
find = gfind
"""
all vulnerabilities
vulnerabilities_all[vuln_id] = {'name': 'vuln_name', 'third_v_id': 'third_v_id'}
"""
vulnerabilities_all = {}
vulnerabilities = CobraVuls.query.all()
for v in vulnerabilities:
vulnerabilities_all[v.id] = {
'name': v.name,
'third_v_id': v.third_v_id
}
for rule in rules:
rule.regex_location = rule.regex_location.strip()
rule.regex_repair = rule.regex_repair.strip()
logging.info(
'------------------\r\nScan rule id: {0} {1} {2}'.format(
self.project_id, rule.id, rule.description))
# Filters
for language in languages:
if language.id == rule.language:
extensions = language.extensions.split('|')
if extensions is None:
logging.critical("Rule Language Error")
sys.exit(0)
# White list
white_list = []
ws = CobraWhiteList.query.filter_by(project_id=self.project_id,
rule_id=rule.id,
status=1).all()
if ws is not None:
for w in ws:
white_list.append(w.path)
try:
if rule.regex_location == "":
filters = []
for index, e in enumerate(extensions):
if index > 1:
filters.append('-o')
filters.append('-name')
filters.append('*' + e)
# Find Special Ext Files
param = [find, self.directory, "-type", "f"] + filters
else:
filters = []
for e in extensions:
filters.append('--include=*' + e)
# explode dirs
explode_dirs = ['.svn', '.cvs', '.hg', '.git', '.bzr']
for explode_dir in explode_dirs:
filters.append('--exclude-dir={0}'.format(explode_dir))
# -n Show Line number / -r Recursive / -P Perl regular expression
param = [grep, "-n", "-r", "-P"] + filters + [
rule.regex_location, self.directory
]
logging.debug(' '.join(param))
p = subprocess.Popen(param, stdout=subprocess.PIPE)
result = p.communicate()
# Exists result
if len(result[0]):
lines = str(result[0]).strip().split("\n")
for line in lines:
line = line.strip()
if line == '':
continue
# 处理grep结果
if ':' in line:
line_split = line.split(':', 1)
file_path = line_split[0].strip()
code_content = line_split[1].split(':',
1)[1].strip()
line_number = line_split[1].split(':',
1)[0].strip()
else:
# 搜索文件
file_path = line
code_content = ''
line_number = 0
# 核心规则校验
result_info = {
'task_id':
self.task_id,
'project_id':
self.project_id,
'project_directory':
self.directory,
'rule_id':
rule.id,
'file_path':
file_path,
'line_number':
line_number,
'code_content':
code_content,
'third_party_vulnerabilities_name':
vulnerabilities_all[rule.vul_id]['name'],
'third_party_vulnerabilities_type':
vulnerabilities_all[rule.vul_id]['third_v_id']
}
ret_status, ret_result = Core(result_info, rule,
self.project_name,
white_list).scan()
if ret_status is False:
logging.info("扫描 R: False {0}".format(ret_result))
continue
else:
logging.info('Not Found')
except Exception as e:
print(traceback.print_exc())
logging.critical('Error calling grep: ' + str(e))
# Set End Time For Task
t = CobraTaskInfo.query.filter_by(id=self.task_id).first()
t.status = 2
t.file_count = files['file_nums']
t.time_end = int(time.time())
t.time_consume = t.time_end - t.time_start
t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime())
try:
db.session.add(t)
db.session.commit()
except Exception as e:
logging.critical("Set start time failed:" + e.message)
logging.info("Scan Done")
def analyse(self):
if self.directory is None:
log.critical("Please set directory")
sys.exit()
log.info('Start code static analyse...')
d = directory.Directory(self.directory)
files = d.collect_files()
log.info('Scan Files: {0}, Total Time: {1}s'.format(
files['file_nums'], files['collect_time']))
ext_language = {
# Image
'.jpg': 'image',
'.png': 'image',
'.bmp': 'image',
'.gif': 'image',
'.ico': 'image',
'.cur': 'image',
# Font
'.eot': 'font',
'.otf': 'font',
'.svg': 'font',
'.ttf': 'font',
'.woff': 'font',
# CSS
'.css': 'css',
'.less': 'css',
'.scss': 'css',
'.styl': 'css',
# Media
'.mp3': 'media',
'.swf': 'media',
# Execute
'.exe': 'execute',
'.sh': 'execute',
'.dll': 'execute',
'.so': 'execute',
'.bat': 'execute',
'.pl': 'execute',
# Edit
'.swp': 'tmp',
# Cert
'.crt': 'cert',
# Text
'.txt': 'text',
'.csv': 'text',
'.md': 'markdown',
# Backup
'.zip': 'backup',
'.bak': 'backup',
'.tar': 'backup',
'.rar': 'backup',
'.tar.gz': 'backup',
'.db': 'backup',
# Config
'.xml': 'config',
'.yml': 'config',
'.spf': 'config',
'.iml': 'config',
'.manifest': 'config',
# Source
'.psd': 'source',
'.as': 'source',
# Log
'.log': 'log',
# Template
'.template': 'template',
'.tpl': 'template',
}
for ext in files:
if ext in ext_language:
log.info('{0} - {1}'.format(ext, files[ext]))
continue
else:
log.info(ext)
languages = CobraLanguages.query.all()
rules = CobraRules.query.filter_by(status=1).all()
extensions = None
for rule in rules:
for language in languages:
if language.id == rule.language:
extensions = language.extensions.split('|')
if extensions is None:
log.warning("Rule Language Error")
# grep name is ggrep on mac
grep = '/bin/grep'
if 'darwin' == sys.platform:
log.info('In Mac OS X System')
for root, dir_names, file_names in os.walk(
'/usr/local/Cellar/grep'):
for filename in file_names:
if 'ggrep' == filename:
grep = os.path.join(root, filename)
filters = []
for e in extensions:
filters.append('--include=*' + e)
# White list
white_list = []
ws = CobraWhiteList.query.filter_by(project_id=self.project_id,
rule_id=rule.id,
status=1).all()
if ws is not None:
for w in ws:
white_list.append(w.path)
try:
log.info('Scan rule id: {0}'.format(rule.id))
# -n Show Line number / -r Recursive / -P Perl regular expression
p = subprocess.Popen([grep, "-n", "-r", "-P"] + filters +
[rule.regex, self.directory],
stdout=subprocess.PIPE)
result = p.communicate()
# Exists result
if len(result[0]):
log.info('Found:')
per_line = str(result[0]).split("\n")
log.debug(per_line)
for r in range(0, len(per_line) - 1):
try:
rr = str(per_line[r]).replace(self.directory,
'').split(':', 1)
code = str(rr[1]).split(':', 1)
if self.task_id is None:
self.task_id = 0
rule_id = rule.id
current_time = datetime.now().strftime(
'%Y-%m-%d %H:%M:%S')
m_file = rr[0].strip()
m_line = code[0]
m_code = str(code[1].strip())
params = [
self.task_id, rule_id, m_file, m_line, m_code,
current_time, current_time
]
try:
if m_file in white_list or ".min.js" in m_file:
log.debug("In White list or min.js")
else:
# # // /* *
match_result = re.match(
"(#)?(//)?(\*)?(/\*)?", m_code)
if match_result.group(
0
) is not None and match_result.group(
0) is not "":
log.debug("In Annotation")
else:
log.debug('In Insert')
if rule.regex == "":
# Didn't filter line when regex is empty
r_content = CobraResults.query.filter_by(
task_id=self.task_id,
rule_id=rule_id,
file=m_file).first()
m_line = 0
else:
r_content = CobraResults.query.filter_by(
task_id=self.task_id,
rule_id=rule_id,
file=m_file,
line=m_line).first()
if r_content is not None:
log.warning("Exists Result")
else:
results = CobraResults(
self.task_id, rule_id, m_file,
m_line, m_code, current_time,
current_time)
db.session.add(results)
db.session.commit()
log.info('Insert Results Success')
except Exception as e:
log.error('Insert Results Failed' +
str(e.message))
log.debug(params)
except Exception as e:
log.critical('Error parsing result: ' +
str(e.message))
else:
log.info('Not Found')
except Exception as e:
log.critical('Error calling grep: ' + str(e))
# Set End Time For Task
t = CobraTaskInfo.query.filter_by(id=self.task_id).first()
t.status = 2
t.file_count = files['file_nums']
t.time_end = int(time.time())
t.time_consume = t.time_end - t.time_start
t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime())
try:
db.session.add(t)
db.session.commit()
except Exception as e:
log.critical("Set start time failed:" + e.message)
log.info("Scan Done")
def analyse(self):
if self.directory is None:
log.critical("Please set directory")
sys.exit()
log.info('Start code static analyse...')
d = directory.Directory(self.directory)
files = d.collect_files(self.task_id)
log.info('Scan Files: {0}, Total Time: {1}s'.format(
files['file_nums'], files['collect_time']))
ext_language = {
# Image
'.jpg': 'image',
'.png': 'image',
'.bmp': 'image',
'.gif': 'image',
'.ico': 'image',
'.cur': 'image',
# Font
'.eot': 'font',
'.otf': 'font',
'.svg': 'font',
'.ttf': 'font',
'.woff': 'font',
# CSS
'.css': 'css',
'.less': 'css',
'.scss': 'css',
'.styl': 'css',
# Media
'.mp3': 'media',
'.swf': 'media',
# Execute
'.exe': 'execute',
'.sh': 'execute',
'.dll': 'execute',
'.so': 'execute',
'.bat': 'execute',
'.pl': 'execute',
# Edit
'.swp': 'tmp',
# Cert
'.crt': 'cert',
# Text
'.txt': 'text',
'.csv': 'text',
'.md': 'markdown',
# Backup
'.zip': 'backup',
'.bak': 'backup',
'.tar': 'backup',
'.rar': 'backup',
'.tar.gz': 'backup',
'.db': 'backup',
# Config
'.xml': 'config',
'.yml': 'config',
'.spf': 'config',
'.iml': 'config',
'.manifest': 'config',
# Source
'.psd': 'source',
'.as': 'source',
# Log
'.log': 'log',
# Template
'.template': 'template',
'.tpl': 'template',
}
for ext in files:
if ext in ext_language:
log.info('{0} - {1}'.format(ext, files[ext]))
continue
else:
log.info(ext)
languages = CobraLanguages.query.all()
rules = CobraRules.query.filter_by(status=1).all()
extensions = None
# `grep` (`ggrep` on Mac)
grep = '/bin/grep'
# `find` (`gfind` on Mac)
find = '/bin/find'
if 'darwin' == sys.platform:
ggrep = ''
gfind = ''
for root, dir_names, file_names in os.walk(
'/usr/local/Cellar/grep'):
for filename in file_names:
if 'ggrep' == filename:
ggrep = os.path.join(root, filename)
for root, dir_names, file_names in os.walk(
'/usr/local/Cellar/findutils'):
for filename in file_names:
if 'gfind' == filename:
gfind = os.path.join(root, filename)
if ggrep == '':
log.critical("brew install ggrep pleases!")
sys.exit(0)
else:
grep = ggrep
if gfind == '':
log.critical("brew install findutils pleases!")
sys.exit(0)
else:
find = gfind
for rule in rules:
log.info('Scan rule id: {0} {1} {2}'.format(
self.project_id, rule.id, rule.description))
# Filters
for language in languages:
if language.id == rule.language:
extensions = language.extensions.split('|')
if extensions is None:
log.critical("Rule Language Error")
sys.exit(0)
# White list
white_list = []
ws = CobraWhiteList.query.filter_by(project_id=self.project_id,
rule_id=rule.id,
status=1).all()
if ws is not None:
for w in ws:
white_list.append(w.path)
try:
if rule.regex.strip() == "":
filters = []
for index, e in enumerate(extensions):
if index > 1:
filters.append('-o')
filters.append('-name')
filters.append('*' + e)
# Find Special Ext Files
param = [find, self.directory, "-type", "f"] + filters
else:
filters = []
for e in extensions:
filters.append('--include=*' + e)
# Explode SVN Dir
filters.append('--exclude-dir=.svn')
filters.append('--exclude-dir=.cvs')
filters.append('--exclude-dir=.hg')
filters.append('--exclude-dir=.git')
filters.append('--exclude-dir=.bzr')
filters.append('--exclude=*.svn-base')
# -n Show Line number / -r Recursive / -P Perl regular expression
param = [grep, "-n", "-r", "-P"
] + filters + [rule.regex, self.directory]
# log.info(' '.join(param))
p = subprocess.Popen(param, stdout=subprocess.PIPE)
result = p.communicate()
# Exists result
if len(result[0]):
lines = str(result[0]).split("\n")
for line in lines:
line = line.strip()
if line == '':
continue
if rule.regex.strip() == '':
# Find
file_path = line.strip().replace(
self.directory, '')
log.debug('File: {0}'.format(file_path))
vul = CobraResults(self.task_id, rule.id,
file_path, 0, '')
db.session.add(vul)
else:
# Grep
line_split = line.replace(self.directory,
'').split(':', 1)
file_path = line_split[0].strip()
code_content = line_split[1].split(':',
1)[1].strip()
line_number = line_split[1].split(':',
1)[0].strip()
if file_path in white_list or ".min.js" in file_path:
log.info("In white list or min.js")
else:
# Annotation
# # // /* *
match_result = re.match(
"(#)?(//)?(\*)?(/\*)?", code_content)
if match_result.group(
0) is not None and match_result.group(
0) is not "":
log.info("In Annotation")
else:
log.info('In Insert')
exist_result = CobraResults.query.filter_by(
task_id=self.task_id,
rule_id=rule.id,
file=file_path,
line=line_number).first()
if exist_result is not None:
log.warning("Exists Result")
else:
log.debug('File: {0}:{1} {2}'.format(
file_path, line_number,
code_content))
vul = CobraResults(
self.task_id, rule.id, file_path,
line_number, code_content)
db.session.add(vul)
log.info('Insert Results Success')
db.session.commit()
else:
log.info('Not Found')
except Exception as e:
log.critical('Error calling grep: ' + str(e))
# Set End Time For Task
t = CobraTaskInfo.query.filter_by(id=self.task_id).first()
t.status = 2
t.file_count = files['file_nums']
t.time_end = int(time.time())
t.time_consume = t.time_end - t.time_start
t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime())
try:
db.session.add(t)
db.session.commit()
except Exception as e:
log.critical("Set start time failed:" + e.message)
log.info("Scan Done")
