def scan(self): """ 扫描漏洞 :return: """ self.method = 0 # 白名单 if self.is_white_list(): logging.info("存在白名单列表中 {0}".format(self.file_path)) return False, 4000 # 特殊文件判断 if self.is_special_file(): logging.info("特殊文件 {0}".format(self.file_path)) return False, 4001 # 注释判断 if self.is_annotation(): logging.info("注释 {0}".format(self.code_content)) return False, 4002 # 仅匹配规则 if self.is_match_only_rule(): logging.info("仅匹配规则 {0}".format(self.rule_location)) found_vul = True else: found_vul = False # 判断参数是否可控 if self.is_can_parse() and self.rule_repair.strip() != '': try: parse_instance = parse.Parse(self.rule_location, self.file_path, self.line_number, self.code_content) if parse_instance.is_controllable_param(): if parse_instance.is_repair(self.rule_repair, self.block_repair): logging.info("Static: repaired") # 标记已修复 self.status = 2 self.process_vulnerabilities() return True, 1003 else: found_vul = True else: logging.info("参数不可控") return False, 4004 except: print(traceback.print_exc()) return False, 4005 if found_vul: self.code_content = self.code_content.encode('unicode_escape') if len(self.code_content) > 512: self.code_content = self.code_content[:500] + '...' self.process_vulnerabilities() return True, 1002 else: logging.critical("Exception core") return False, 4006
def repair(self): """ 验证扫描到的漏洞是否修复 :return: (Status, Result) """ self.method = 1 # 拼接绝对路径 self.file_path = self.project_directory + self.file_path # 定位规则为空时或者行号为0,表示此类型语言(该语言所有后缀)文件都算作漏洞 if self.rule_location == '' or self.line_number == 0: logging.info("Find special files: RID{0}".format(self.rule_id)) # 检查文件是否存在 if os.path.isfile(self.file_path) is False: # 未找到该文件则更新漏洞状态为已修复 logging.info("已删除文件修复完成 {0}".format(self.file_path)) self.status = self.status_fixed self.process_vulnerabilities() return True, 1001 # 文件存在,漏洞还在 return False, 4007 # 取出触发代码(实际文件) trigger_code = File(self.file_path).lines("{0}p".format( self.line_number)) if trigger_code is False: logging.critical("触发代码获取失败 {0}".format(self.code_content)) return False, 4009 self.code_content = trigger_code # 白名单 if self.is_white_list(): self.status = self.status_fixed self.process_vulnerabilities() logging.info("In white list {0}".format(self.file_path)) return False, 4000 # 特殊文件判断 if self.is_special_file(): self.status = self.status_fixed self.process_vulnerabilities() logging.info("Special File: {0}".format(self.file_path)) return False, 4001 # 注释判断 if self.is_annotation(): self.status = self.status_fixed self.process_vulnerabilities() logging.info("In Annotation {0}".format(self.code_content)) return False, 4002 # 仅匹配规则 if self.is_match_only_rule(): logging.info("Only match {0}".format(self.rule_location)) found_vul = True else: found_vul = False # 判断参数是否可控 if self.is_can_parse() and self.rule_repair.strip() != '': try: parse_instance = parse.Parse(self.rule_location, self.file_path, self.line_number, self.code_content) if parse_instance.is_controllable_param(): if parse_instance.is_repair(self.rule_repair, self.block_repair): logging.info("Static: repaired") # 标记已修复 self.status = self.status_fixed self.process_vulnerabilities() return True, 1003 else: found_vul = True else: logging.info("参数不可控") return False, 4004 except: print(traceback.print_exc()) return False, 4005 if found_vul: self.code_content = self.code_content.encode('unicode_escape') if len(self.code_content) > 512: self.code_content = self.code_content[:500] + '...' self.process_vulnerabilities() return True, 1002 else: logging.info("Not found") return False, 4006
def analyse(self): if self.directory is None: logging.critical("Please set directory") sys.exit() logging.info('Start code static analyse...') d = directory.Directory(self.directory) files = d.collect_files(self.task_id) logging.info('Scan Files: {0}, Total Time: {1}s'.format( files['file_nums'], files['collect_time'])) ext_language = { # Image '.jpg': 'image', '.png': 'image', '.bmp': 'image', '.gif': 'image', '.ico': 'image', '.cur': 'image', # Font '.eot': 'font', '.otf': 'font', '.svg': 'font', '.ttf': 'font', '.woff': 'font', # CSS '.css': 'css', '.less': 'css', '.scss': 'css', '.styl': 'css', # Media '.mp3': 'media', '.swf': 'media', # Execute '.exe': 'execute', '.sh': 'execute', '.dll': 'execute', '.so': 'execute', '.bat': 'execute', '.pl': 'execute', # Edit '.swp': 'tmp', # Cert '.crt': 'cert', # Text '.txt': 'text', '.csv': 'text', '.md': 'markdown', # Backup '.zip': 'backup', '.bak': 'backup', '.tar': 'backup', '.rar': 'backup', '.tar.gz': 'backup', '.db': 'backup', # Config '.xml': 'config', '.yml': 'config', '.spf': 'config', '.iml': 'config', '.manifest': 'config', # Source '.psd': 'source', '.as': 'source', # Log '.log': 'log', # Template '.template': 'template', '.tpl': 'template', } for ext in files: if ext in ext_language: logging.info('{0} - {1}'.format(ext, files[ext])) continue else: logging.info(ext) languages = CobraLanguages.query.all() rules = CobraRules.query.filter_by(status=1).all() extensions = None # `grep` (`ggrep` on Mac) grep = '/bin/grep' # `find` (`gfind` on Mac) find = '/bin/find' if 'darwin' == sys.platform: ggrep = '' gfind = '' for root, dir_names, file_names in os.walk( '/usr/local/Cellar/grep'): for filename in file_names: if 'ggrep' == filename or 'grep' == filename: ggrep = os.path.join(root, filename) for root, dir_names, file_names in os.walk( '/usr/local/Cellar/findutils'): for filename in file_names: if 'gfind' == filename: gfind = os.path.join(root, filename) if ggrep == '': logging.critical("brew install ggrep pleases!") sys.exit(0) else: grep = ggrep if gfind == '': logging.critical("brew install findutils pleases!") sys.exit(0) else: find = gfind """ all vulnerabilities vulnerabilities_all[vuln_id] = {'name': 'vuln_name', 'third_v_id': 'third_v_id'} """ vulnerabilities_all = {} vulnerabilities = CobraVuls.query.all() for v in vulnerabilities: vulnerabilities_all[v.id] = { 'name': v.name, 'third_v_id': v.third_v_id } for rule in rules: rule.regex_location = rule.regex_location.strip() rule.regex_repair = rule.regex_repair.strip() logging.info('Scan rule id: {0} {1} {2}'.format( self.project_id, rule.id, rule.description)) # Filters for language in languages: if language.id == rule.language: extensions = language.extensions.split('|') if extensions is None: logging.critical("Rule Language Error") sys.exit(0) # White list white_list = [] ws = CobraWhiteList.query.filter_by(project_id=self.project_id, rule_id=rule.id, status=1).all() if ws is not None: for w in ws: white_list.append(w.path) try: if rule.regex_location == "": filters = [] for index, e in enumerate(extensions): if index > 1: filters.append('-o') filters.append('-name') filters.append('*' + e) # Find Special Ext Files param = [find, self.directory, "-type", "f"] + filters else: filters = [] for e in extensions: filters.append('--include=*' + e) # explode dirs explode_dirs = ['.svn', '.cvs', '.hg', '.git', '.bzr'] for explode_dir in explode_dirs: filters.append('--exclude-dir={0}'.format(explode_dir)) # -n Show Line number / -r Recursive / -P Perl regular expression param = [grep, "-n", "-r", "-P"] + filters + [ rule.regex_location, self.directory ] # logging.info(' '.join(param)) p = subprocess.Popen(param, stdout=subprocess.PIPE) result = p.communicate() # Exists result if len(result[0]): lines = str(result[0]).strip().split("\n") for line in lines: line = line.strip() if line == '': continue if rule.regex_location == '': # Find (special file) file_path = line.strip().replace( self.directory, '') logging.debug('File: {0}'.format(file_path)) exist_result = CobraResults.query.filter_by( project_id=self.project_id, rule_id=rule.id, file=file_path).first() if exist_result is not None: # push queue if exist_result.status == 0: try: q = Queue( self.project_name, vulnerabilities_all[rule.vul_id] ['name'], vulnerabilities_all[ rule.vul_id]['third_v_id'], file_path, 0, 0, exist_result.id) q.push() except Exception as e: print(traceback.print_exc()) logging.critical(e.message) logging.warning("Exists Result") else: vul = CobraResults(self.task_id, self.project_id, rule.id, file_path, 0, '', 0) db.session.add(vul) try: # push queue q = Queue( self.project_name, vulnerabilities_all[ rule.vul_id]['name'], vulnerabilities_all[ rule.vul_id]['third_v_id'], file_path, 0, 0, vul.id) q.push() except Exception as e: print(traceback.print_exc()) logging.critical(e.message) else: # Grep line_split = line.split(':', 1) file_path = line_split[0].strip() if len(line_split) < 2: logging.info("Line len < 2 {0}".format(line)) continue code_content = line_split[1].split(':', 1)[1].strip() line_number = line_split[1].split(':', 1)[0].strip() if file_path in white_list or ".min.js" in file_path: logging.info("In white list or min.js") else: only_match = rule.regex_location[: 1] == '(' and rule.regex_location[ -1] == ')' """ annotation (注释过滤) # // /* * Exclude: - (rule_location) - 当定位规则左右两边为括号时不过滤注释行,比如硬编码密码 """ match_result = re.match( "(#)?(//)?(\*)?(/\*)?", code_content) if match_result.group( 0) is not None and match_result.group( 0 ) is not "" and only_match is not True: logging.info("In Annotation") else: param_value = None # parse file function structure if only_match: found_vul = True else: if file_path[ -3:] == 'php' and rule.regex_repair.strip( ) != '': try: parse_instance = parse.Parse( rule.regex_location, file_path, line_number, code_content) if parse_instance.is_controllable_param( ): if parse_instance.is_repair( rule.regex_repair, rule.block_repair): logging.info( "Static: repaired") continue else: if parse_instance.param_value is not None: param_value = parse_instance.param_value found_vul = True else: logging.info( "Static: uncontrollable param" ) continue except: print(traceback.print_exc()) found_vul = False else: found_vul = True file_path = file_path.replace( self.directory, '') if found_vul: logging.info('In Insert') exist_result = CobraResults.query.filter_by( project_id=self.project_id, rule_id=rule.id, file=file_path, line=line_number).first() if exist_result is not None: logging.info("Exists Result") # push queue if exist_result.status == 0: try: q = Queue( self.project_name, vulnerabilities_all[ rule.vul_id] ['name'], vulnerabilities_all[ rule.vul_id] ['third_v_id'], file_path, line_number, code_content, exist_result.id) q.push() except Exception as e: print( traceback.print_exc()) logging.critical(e.message) else: code_content = code_content.encode( 'unicode_escape') if len(code_content) > 512: code_content = code_content[: 500] + '...' code_content = '# Trigger\r' + code_content if param_value is not None: code_content = '# Param\r' + param_value + '\r//\r// ------ Continue... ------\r//\r' + code_content logging.debug( 'File: {0}:{1} {2}'.format( file_path, line_number, code_content)) vul = CobraResults( self.task_id, self.project_id, rule.id, file_path, line_number, code_content, 0) db.session.add(vul) db.session.commit() try: q = Queue( self.project_name, vulnerabilities_all[ rule.vul_id]['name'], vulnerabilities_all[ rule.vul_id] ['third_v_id'], file_path, line_number, code_content, vul.id) q.push() except Exception as e: print(traceback.print_exc()) logging.critical(e.message) logging.info( 'Insert Results Success') else: logging.info('Not Found') except Exception as e: print(traceback.print_exc()) logging.critical('Error calling grep: ' + str(e)) # Set End Time For Task t = CobraTaskInfo.query.filter_by(id=self.task_id).first() t.status = 2 t.file_count = files['file_nums'] t.time_end = int(time.time()) t.time_consume = t.time_end - t.time_start t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime()) try: db.session.add(t) db.session.commit() except Exception as e: logging.critical("Set start time failed:" + e.message) logging.info("Scan Done")
def repair(self): """ Scan vulnerabilities is repair :flow: - exist file [add] - test file - whitelist file - special file - annotation - rule :return: (Status, Result) """ self.method = 1 # Full path self.file_path = self.project_directory + self.file_path """ When the targeting rule is empty or the line number is 0, it means that this type of language (all suffixes in that language) is counted as a vulnerability Their repair method is only one: delete the file """ if self.rule_location == '' or self.line_number == 0: logging.info("Find special files: RID{0}".format(self.rule_id)) # Check if the file exists if os.path.isfile(self.file_path) is False: # If the file is not found, the vulnerability state is fixed logging.info("Deleted file repair is complete {0}".format(self.file_path)) self.status = self.status_fixed self.repair_code = self.repair_code_not_exist_file self.process_vulnerabilities() return else: return # Not exist file if os.path.isfile(self.file_path) is False: self.status = self.status_fixed self.repair_code = self.repair_code_not_exist_file self.process_vulnerabilities() return # Test file if self.is_test_file(): self.status = self.status_fixed self.repair_code = self.repair_code_test_file self.process_vulnerabilities() return """ Cobra Skip @cobra const `@[cC][oO][bB][rR][aA]\s*[cC][oO][nN][sS][tT]` """ file_content = File(self.file_path).read_file() ret_regex_const = re.findall(r'@[cC][oO][bB][rR][aA]\s*[cC][oO][nN][sS][tT]', file_content) if len(ret_regex_const) > 0: self.status = self.status_fixed self.repair_code = self.repair_code_const_file self.process_vulnerabilities() return """ @cobra third-party `@[cC][oO][bB][rR][aA]\s*[tT][hH][iI][rR][dD]-[pP][aA][rR][tT][yY]` """ ret_regex_third_party = re.findall(r'@[cC][oO][bB][rR][aA]\s*[tT][hH][iI][rR][dD]-[pP][aA][rR][tT][yY]', file_content) if len(ret_regex_third_party) > 0: self.status = self.status_fixed self.repair_code = self.repair_code_third_party self.process_vulnerabilities() return # Remove the trigger code (actual file) trigger_code = File(self.file_path).lines("{0}p".format(self.line_number)) if trigger_code is False: logging.critical("Failed to fetch the trigger code {0}".format(self.code_content)) self.status = self.status_fixed self.repair_code = self.repair_code_empty_code self.process_vulnerabilities() return self.code_content = trigger_code # Whitelist if self.is_white_list(): self.status = self.status_fixed self.repair_code = self.repair_code_whitelist self.process_vulnerabilities() logging.info("In white list {0}".format(self.file_path)) return # Special file if self.is_special_file(): self.status = self.status_fixed self.repair_code = self.repair_code_special_file self.process_vulnerabilities() logging.info("Special File: {0}".format(self.file_path)) return # Annotation if self.is_annotation(): self.status = self.status_fixed self.repair_code = self.repair_code_annotation self.process_vulnerabilities() logging.info("In Annotation {0}".format(self.code_content)) return # Modify ret_regex = re.findall(self.rule_location, trigger_code.strip()) if len(ret_regex) == 0: self.status = self.status_fixed self.repair_code = self.repair_code_modify self.process_vulnerabilities() return # Fixed if self.is_can_parse() and self.rule_repair.strip() != '': try: parse_instance = parse.Parse(self.rule_location, self.file_path, self.line_number, self.code_content) if parse_instance.is_repair(self.rule_repair, self.block_repair): logging.info("Static: repaired") # Fixed self.status = self.status_fixed self.repair_code = self.repair_code_fixed self.process_vulnerabilities() return else: logging.critical("[repair] not fixed") return except: logging.info(traceback.print_exc()) return
def scan(self): """ Scan vulnerabilities :flow: - whitelist file - special file - test file - annotation - rule :return: """ self.method = 0 if self.is_white_list(): logging.info("Whitelist file {0}".format(self.file_path)) return False, 4000 if self.is_special_file(): logging.info("Special file: {0}".format(self.file_path)) return False, 4001 if self.is_test_file(): logging.info("Test file: {0}".format(self.file_path)) return False, 4007 if self.is_annotation(): logging.info("Annotation {0}".format(self.code_content)) return False, 4002 if self.is_match_only_rule(): logging.info("Only Rule: {0}".format(self.rule_location)) found_vul = True else: found_vul = False # parameter is controllable if self.is_can_parse() and self.rule_repair.strip() != '': try: parse_instance = parse.Parse(self.rule_location, self.file_path, self.line_number, self.code_content) if parse_instance.is_controllable_param(): if parse_instance.is_repair(self.rule_repair, self.block_repair): # fixed return True, 1003 else: found_vul = True else: logging.info("parameter is not controllable") return False, 4004 except: print(traceback.print_exc()) return False, 4005 if found_vul: self.code_content = self.code_content.encode('unicode_escape') if len(self.code_content) > 512: self.code_content = self.code_content[:500] self.status = self.status_init self.repair_code = self.repair_code_init self.process_vulnerabilities() return True, 1002 else: logging.info("Not found vulnerabilities") return False, 4006
def repair(self): """ Scan vulnerabilities is repair :flow: - exist file [add] - test file - whitelist file - special file - annotation - rule :return: (Status, Result) """ self.method = 1 # Full path self.file_path = self.project_directory + self.file_path """ 定位规则为空时或者行号为0,表示此类型语言(该语言所有后缀)文件都算作漏洞 他们的修复方法只有一个:删除文件 """ if self.rule_location == '' or self.line_number == 0: logging.info("Find special files: RID{0}".format(self.rule_id)) # 检查文件是否存在 if os.path.isfile(self.file_path) is False: # 未找到该文件则更新漏洞状态为已修复 logging.info("已删除文件修复完成 {0}".format(self.file_path)) self.status = self.status_fixed self.repair_code = self.repair_code_not_exist_file self.process_vulnerabilities() return else: return # Not exist file if os.path.isfile(self.file_path) is False: self.status = self.status_fixed self.repair_code = self.repair_code_not_exist_file self.process_vulnerabilities() return # Test file if self.is_test_file(): self.status = self.status_fixed self.repair_code = self.repair_code_test_file self.process_vulnerabilities() return """ Cobra Skip @cobra const `@[cC][oO][bB][rR][aA]\s*[cC][oO][nN][sS][tT]` """ file_content = File(self.file_path).read_file() ret_regex_const = re.findall( r'@[cC][oO][bB][rR][aA]\s*[cC][oO][nN][sS][tT]', file_content) if len(ret_regex_const) > 0: self.status = self.status_fixed self.repair_code = self.repair_code_const_file self.process_vulnerabilities() return """ @cobra third-party `@[cC][oO][bB][rR][aA]\s*[tT][hH][iI][rR][dD]-[pP][aA][rR][tT][yY]` """ ret_regex_third_party = re.findall( r'@[cC][oO][bB][rR][aA]\s*[tT][hH][iI][rR][dD]-[pP][aA][rR][tT][yY]', file_content) if len(ret_regex_third_party) > 0: self.status = self.status_fixed self.repair_code = self.repair_code_third_party self.process_vulnerabilities() return # 取出触发代码(实际文件) trigger_code = File(self.file_path).lines("{0}p".format( self.line_number)) if trigger_code is False: logging.critical("触发代码获取失败 {0}".format(self.code_content)) self.status = self.status_fixed self.repair_code = self.repair_code_empty_code self.process_vulnerabilities() return self.code_content = trigger_code # Whitelist if self.is_white_list(): self.status = self.status_fixed self.repair_code = self.repair_code_whitelist self.process_vulnerabilities() logging.info("In white list {0}".format(self.file_path)) return # Special file if self.is_special_file(): self.status = self.status_fixed self.repair_code = self.repair_code_special_file self.process_vulnerabilities() logging.info("Special File: {0}".format(self.file_path)) return # Annotation if self.is_annotation(): self.status = self.status_fixed self.repair_code = self.repair_code_annotation self.process_vulnerabilities() logging.info("In Annotation {0}".format(self.code_content)) return # Modify ret_regex = re.findall(self.rule_location, trigger_code.strip()) if len(ret_regex) == 0: self.status = self.status_fixed self.repair_code = self.repair_code_modify self.process_vulnerabilities() return # Fixed if self.is_can_parse() and self.rule_repair.strip() != '': try: parse_instance = parse.Parse(self.rule_location, self.file_path, self.line_number, self.code_content) if parse_instance.is_repair(self.rule_repair, self.block_repair): logging.info("Static: repaired") # Fixed self.status = self.status_fixed self.repair_code = self.repair_code_fixed self.process_vulnerabilities() return else: logging.critical("[repair] not fixed") return except: logging.info(traceback.print_exc()) return
def scan(self): """ Scan vulnerabilities :flow: - whitelist file - special file - test file - annotation - rule :return: """ self.method = 0 if self.is_white_list(): self.log('info', "[RET] Whitelist\r\n") return self.data if self.is_special_file(): self.log('info', "[RET] Special File\r\n") return self.data if self.is_test_file(): self.log('info', "[RET] Test File\r\n") return self.data if self.is_annotation(): self.log('info', "[RET] Annotation\r\n") return self.data if self.is_match_only_rule(): self.log('info', "Only Rule: {0}".format(self.rule_location)) found_vul = True else: found_vul = False # parameter is controllable if self.is_can_parse() and self.rule_repair.strip() != '': try: parse_instance = parse.Parse(self.rule_location, self.file_path, self.line_number, self.code_content) param_is_controllable, data = parse_instance.is_controllable_param( ) if param_is_controllable: self.data += data self.log('info', '[RET] Param is controllable\r\n') is_repair, data = parse_instance.is_repair( self.rule_repair, self.block_repair) self.data += data if is_repair: # fixed self.log('info', '[RET] Vulnerability Fixed\r\n') return self.data else: self.log('info', 'Repair: Not fixed') found_vul = True else: self.data += data self.log('info', '[RET] Param Not Controllable\r\n') return self.data except: traceback.print_exc() return self.data if found_vul: self.code_content = self.code_content.encode('unicode_escape') if len(self.code_content) > 512: self.code_content = self.code_content[:500] self.status = self.status_init self.repair_code = self.repair_code_init self.process_vulnerabilities() return self.data else: self.log('info', "[RET] Not found vulnerability\r\n") return self.data