def test_directory(self): directory = directory.Directory(self.project) directory.collect_files()
def analyse(self, test=False): if self.directory is None: self.log('critical', 'Please set directory') sys.exit() files = directory.Directory(self.directory).collect_files(self.task_id) self.log('info', '**Scan Files**\r\n > Files count: `{files}`\r\n > Time consume: `{consume}s`\r\n'.format(files=files['file_nums'], consume=files['collect_time'])) ext_language = { # Image '.jpg': 'image', '.png': 'image', '.bmp': 'image', '.gif': 'image', '.ico': 'image', '.cur': 'image', # Font '.eot': 'font', '.otf': 'font', '.svg': 'font', '.ttf': 'font', '.woff': 'font', # CSS '.css': 'css', '.less': 'css', '.scss': 'css', '.styl': 'css', # Media '.mp3': 'media', '.swf': 'media', # Execute '.exe': 'execute', '.sh': 'execute', '.dll': 'execute', '.so': 'execute', '.bat': 'execute', '.pl': 'execute', # Edit '.swp': 'tmp', # Cert '.crt': 'cert', # Text '.txt': 'text', '.csv': 'text', '.md': 'markdown', # Backup '.zip': 'backup', '.bak': 'backup', '.tar': 'backup', '.rar': 'backup', '.tar.gz': 'backup', '.db': 'backup', # Config '.xml': 'config', '.yml': 'config', '.spf': 'config', '.iml': 'config', '.manifest': 'config', # Source '.psd': 'source', '.as': 'source', # Log '.log': 'log', # Template '.template': 'template', '.tpl': 'template', } for ext in files: if ext in ext_language: self.log('info', '{0} - {1}'.format(ext, files[ext]), False) continue else: self.log('info', ext, False) explode_dirs = ['.svn', '.cvs', '.hg', '.git', '.bzr'] self.log('info', '**Rule Scan**\r\n > Global explode directory: `{dirs}`\r\n'.format(dirs=', '.join(explode_dirs))) languages = CobraLanguages.query.all() filter_group = (CobraRules.status == 1,) if self.rule_id is not None: filter_group += (CobraRules.id == self.rule_id,) rules = CobraRules.query.filter(*filter_group).all() extensions = None find = tool.find grep = tool.grep """ Vulnerability Types vulnerability_types[vuln_id] = {'name': 'vuln_name', 'third_v_id': 'third_v_id'} """ vulnerability_types = {} vulnerabilities = CobraVuls.query.all() for v in vulnerabilities: vulnerability_types[v.id] = { 'name': v.name, 'third_v_id': v.third_v_id } for index, rule in enumerate(rules): rule.regex_location = rule.regex_location.strip() rule.regex_repair = rule.regex_repair.strip() # Filters for language in languages: if language.id == rule.language: extensions = language.extensions.split('|') if extensions is None: self.log('critical', 'Rule language error') sys.exit(0) # White list white_list = [] ws = CobraWhiteList.query.filter_by(project_id=self.project_id, rule_id=rule.id, status=1).all() if ws is not None: for w in ws: white_list.append(w.path) try: if rule.regex_location == "": mode = 'Find' filters = [] for index, e in enumerate(extensions): if index > 1: filters.append('-o') filters.append('-name') filters.append('*' + e) # Find Special Ext Files param = [find, self.directory, "-type", "f"] + filters else: mode = 'Grep' filters = [] for e in extensions: filters.append('--include=*' + e) # explode dirs for explode_dir in explode_dirs: filters.append('--exclude-dir={0}'.format(explode_dir)) # -s suppress error messages / -n Show Line number / -r Recursive / -P Perl regular expression param = [grep, "-s", "-n", "-r", "-P"] + filters + [rule.regex_location, self.directory] self.log('info', '**Rule Info({index})**\r\n > ID: `{rid}` \r\n > Name: `{name}` \r\n > Language: `{language}`\r\n > Rule mode:`{mode}`\r\n > Location: `{location}` \r\n > Repair: `{repair}`\r\n'.format(index=index, rid=rule.id, name=rule.description, language=extensions, mode=mode, location=rule.regex_location, repair=rule.regex_repair)) p = subprocess.Popen(param, stdout=subprocess.PIPE) result = p.communicate() # exists result if len(result[0]): lines = str(result[0]).strip().split("\n") self.log('info', '**Founded Vulnerability**\r\n > Vulnerability Count: `{count}`\r\n'.format(count=len(lines))) for index, line in enumerate(lines): line = line.strip() if line == '': continue # grep result if ':' in line: line_split = line.split(':', 1) file_path = line_split[0].strip() code_content = line_split[1].split(':', 1)[1].strip() line_number = line_split[1].split(':', 1)[0].strip() else: # search file file_path = line code_content = '' line_number = 0 # core rule check result_info = { 'task_id': self.task_id, 'project_id': self.project_id, 'project_directory': self.directory, 'rule_id': rule.id, 'result_id': None, 'file_path': file_path, 'line_number': line_number, 'code_content': code_content, 'third_party_vulnerabilities_name': vulnerability_types[rule.vul_id]['name'], 'third_party_vulnerabilities_type': vulnerability_types[rule.vul_id]['third_v_id'] } self.data += Core(result_info, rule, self.project_name, white_list, test=True, index=index).scan() else: self.log('info', 'Not Found') except Exception as e: traceback.print_exc() self.log('critical', 'Error calling grep: ' + str(e)) if not test: # set end time for task t = CobraTaskInfo.query.filter_by(id=self.task_id).first() t.status = 2 t.file_count = files['file_nums'] t.time_end = int(time.time()) t.time_consume = t.time_end - t.time_start t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime()) try: db.session.add(t) db.session.commit() except Exception as e: self.log('critical', "Set start time failed:" + e.message) self.log('info', "[END] Scan") return self.data
def analyse(self): if self.directory is None: logging.critical("Please set directory") sys.exit() logging.info('Start code static analyse...') d = directory.Directory(self.directory) files = d.collect_files(self.task_id) logging.info('Scan Files: {0}, Total Time: {1}s'.format( files['file_nums'], files['collect_time'])) ext_language = { # Image '.jpg': 'image', '.png': 'image', '.bmp': 'image', '.gif': 'image', '.ico': 'image', '.cur': 'image', # Font '.eot': 'font', '.otf': 'font', '.svg': 'font', '.ttf': 'font', '.woff': 'font', # CSS '.css': 'css', '.less': 'css', '.scss': 'css', '.styl': 'css', # Media '.mp3': 'media', '.swf': 'media', # Execute '.exe': 'execute', '.sh': 'execute', '.dll': 'execute', '.so': 'execute', '.bat': 'execute', '.pl': 'execute', # Edit '.swp': 'tmp', # Cert '.crt': 'cert', # Text '.txt': 'text', '.csv': 'text', '.md': 'markdown', # Backup '.zip': 'backup', '.bak': 'backup', '.tar': 'backup', '.rar': 'backup', '.tar.gz': 'backup', '.db': 'backup', # Config '.xml': 'config', '.yml': 'config', '.spf': 'config', '.iml': 'config', '.manifest': 'config', # Source '.psd': 'source', '.as': 'source', # Log '.log': 'log', # Template '.template': 'template', '.tpl': 'template', } for ext in files: if ext in ext_language: logging.info('{0} - {1}'.format(ext, files[ext])) continue else: logging.info(ext) languages = CobraLanguages.query.all() rules = CobraRules.query.filter_by(status=1).all() extensions = None # `grep` (`ggrep` on Mac) grep = '/bin/grep' # `find` (`gfind` on Mac) find = '/bin/find' if 'darwin' == sys.platform: ggrep = '' gfind = '' for root, dir_names, file_names in os.walk( '/usr/local/Cellar/grep'): for filename in file_names: if 'ggrep' == filename or 'grep' == filename: ggrep = os.path.join(root, filename) for root, dir_names, file_names in os.walk( '/usr/local/Cellar/findutils'): for filename in file_names: if 'gfind' == filename: gfind = os.path.join(root, filename) if ggrep == '': logging.critical("brew install ggrep pleases!") sys.exit(0) else: grep = ggrep if gfind == '': logging.critical("brew install findutils pleases!") sys.exit(0) else: find = gfind """ all vulnerabilities vulnerabilities_all[vuln_id] = {'name': 'vuln_name', 'third_v_id': 'third_v_id'} """ vulnerabilities_all = {} vulnerabilities = CobraVuls.query.all() for v in vulnerabilities: vulnerabilities_all[v.id] = { 'name': v.name, 'third_v_id': v.third_v_id } for rule in rules: rule.regex_location = rule.regex_location.strip() rule.regex_repair = rule.regex_repair.strip() logging.info('Scan rule id: {0} {1} {2}'.format( self.project_id, rule.id, rule.description)) # Filters for language in languages: if language.id == rule.language: extensions = language.extensions.split('|') if extensions is None: logging.critical("Rule Language Error") sys.exit(0) # White list white_list = [] ws = CobraWhiteList.query.filter_by(project_id=self.project_id, rule_id=rule.id, status=1).all() if ws is not None: for w in ws: white_list.append(w.path) try: if rule.regex_location == "": filters = [] for index, e in enumerate(extensions): if index > 1: filters.append('-o') filters.append('-name') filters.append('*' + e) # Find Special Ext Files param = [find, self.directory, "-type", "f"] + filters else: filters = [] for e in extensions: filters.append('--include=*' + e) # explode dirs explode_dirs = ['.svn', '.cvs', '.hg', '.git', '.bzr'] for explode_dir in explode_dirs: filters.append('--exclude-dir={0}'.format(explode_dir)) # -n Show Line number / -r Recursive / -P Perl regular expression param = [grep, "-n", "-r", "-P"] + filters + [ rule.regex_location, self.directory ] # logging.info(' '.join(param)) p = subprocess.Popen(param, stdout=subprocess.PIPE) result = p.communicate() # Exists result if len(result[0]): lines = str(result[0]).strip().split("\n") for line in lines: line = line.strip() if line == '': continue if rule.regex_location == '': # Find (special file) file_path = line.strip().replace( self.directory, '') logging.debug('File: {0}'.format(file_path)) exist_result = CobraResults.query.filter_by( project_id=self.project_id, rule_id=rule.id, file=file_path).first() if exist_result is not None: # push queue if exist_result.status == 0: try: q = Queue( self.project_name, vulnerabilities_all[rule.vul_id] ['name'], vulnerabilities_all[ rule.vul_id]['third_v_id'], file_path, 0, 0, exist_result.id) q.push() except Exception as e: print(traceback.print_exc()) logging.critical(e.message) logging.warning("Exists Result") else: vul = CobraResults(self.task_id, self.project_id, rule.id, file_path, 0, '', 0) db.session.add(vul) try: # push queue q = Queue( self.project_name, vulnerabilities_all[ rule.vul_id]['name'], vulnerabilities_all[ rule.vul_id]['third_v_id'], file_path, 0, 0, vul.id) q.push() except Exception as e: print(traceback.print_exc()) logging.critical(e.message) else: # Grep line_split = line.split(':', 1) file_path = line_split[0].strip() if len(line_split) < 2: logging.info("Line len < 2 {0}".format(line)) continue code_content = line_split[1].split(':', 1)[1].strip() line_number = line_split[1].split(':', 1)[0].strip() if file_path in white_list or ".min.js" in file_path: logging.info("In white list or min.js") else: only_match = rule.regex_location[: 1] == '(' and rule.regex_location[ -1] == ')' """ annotation (注释过滤) # // /* * Exclude: - (rule_location) - 当定位规则左右两边为括号时不过滤注释行,比如硬编码密码 """ match_result = re.match( "(#)?(//)?(\*)?(/\*)?", code_content) if match_result.group( 0) is not None and match_result.group( 0 ) is not "" and only_match is not True: logging.info("In Annotation") else: param_value = None # parse file function structure if only_match: found_vul = True else: if file_path[ -3:] == 'php' and rule.regex_repair.strip( ) != '': try: parse_instance = parse.Parse( rule.regex_location, file_path, line_number, code_content) if parse_instance.is_controllable_param( ): if parse_instance.is_repair( rule.regex_repair, rule.block_repair): logging.info( "Static: repaired") continue else: if parse_instance.param_value is not None: param_value = parse_instance.param_value found_vul = True else: logging.info( "Static: uncontrollable param" ) continue except: print(traceback.print_exc()) found_vul = False else: found_vul = True file_path = file_path.replace( self.directory, '') if found_vul: logging.info('In Insert') exist_result = CobraResults.query.filter_by( project_id=self.project_id, rule_id=rule.id, file=file_path, line=line_number).first() if exist_result is not None: logging.info("Exists Result") # push queue if exist_result.status == 0: try: q = Queue( self.project_name, vulnerabilities_all[ rule.vul_id] ['name'], vulnerabilities_all[ rule.vul_id] ['third_v_id'], file_path, line_number, code_content, exist_result.id) q.push() except Exception as e: print( traceback.print_exc()) logging.critical(e.message) else: code_content = code_content.encode( 'unicode_escape') if len(code_content) > 512: code_content = code_content[: 500] + '...' code_content = '# Trigger\r' + code_content if param_value is not None: code_content = '# Param\r' + param_value + '\r//\r// ------ Continue... ------\r//\r' + code_content logging.debug( 'File: {0}:{1} {2}'.format( file_path, line_number, code_content)) vul = CobraResults( self.task_id, self.project_id, rule.id, file_path, line_number, code_content, 0) db.session.add(vul) db.session.commit() try: q = Queue( self.project_name, vulnerabilities_all[ rule.vul_id]['name'], vulnerabilities_all[ rule.vul_id] ['third_v_id'], file_path, line_number, code_content, vul.id) q.push() except Exception as e: print(traceback.print_exc()) logging.critical(e.message) logging.info( 'Insert Results Success') else: logging.info('Not Found') except Exception as e: print(traceback.print_exc()) logging.critical('Error calling grep: ' + str(e)) # Set End Time For Task t = CobraTaskInfo.query.filter_by(id=self.task_id).first() t.status = 2 t.file_count = files['file_nums'] t.time_end = int(time.time()) t.time_consume = t.time_end - t.time_start t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime()) try: db.session.add(t) db.session.commit() except Exception as e: logging.critical("Set start time failed:" + e.message) logging.info("Scan Done")
def files(self): d = directory.Directory(self.project_path) self.target_files = d.collect_files() return self.target_files
def analyse(self): if self.directory is None: logging.critical("Please set directory") sys.exit() logging.info('Start code static analyse...') d = directory.Directory(self.directory) files = d.collect_files(self.task_id) logging.info('Scan Files: {0}, Total Time: {1}s'.format( files['file_nums'], files['collect_time'])) ext_language = { # Image '.jpg': 'image', '.png': 'image', '.bmp': 'image', '.gif': 'image', '.ico': 'image', '.cur': 'image', # Font '.eot': 'font', '.otf': 'font', '.svg': 'font', '.ttf': 'font', '.woff': 'font', # CSS '.css': 'css', '.less': 'css', '.scss': 'css', '.styl': 'css', # Media '.mp3': 'media', '.swf': 'media', # Execute '.exe': 'execute', '.sh': 'execute', '.dll': 'execute', '.so': 'execute', '.bat': 'execute', '.pl': 'execute', # Edit '.swp': 'tmp', # Cert '.crt': 'cert', # Text '.txt': 'text', '.csv': 'text', '.md': 'markdown', # Backup '.zip': 'backup', '.bak': 'backup', '.tar': 'backup', '.rar': 'backup', '.tar.gz': 'backup', '.db': 'backup', # Config '.xml': 'config', '.yml': 'config', '.spf': 'config', '.iml': 'config', '.manifest': 'config', # Source '.psd': 'source', '.as': 'source', # Log '.log': 'log', # Template '.template': 'template', '.tpl': 'template', } for ext in files: if ext in ext_language: logging.info('{0} - {1}'.format(ext, files[ext])) continue else: logging.info(ext) languages = CobraLanguages.query.all() rules = CobraRules.query.filter_by(status=1).all() extensions = None # `grep` (`ggrep` on Mac) grep = '/bin/grep' # `find` (`gfind` on Mac) find = '/bin/find' if 'darwin' == sys.platform: ggrep = '' gfind = '' for root, dir_names, file_names in os.walk( '/usr/local/Cellar/grep'): for filename in file_names: if 'ggrep' == filename or 'grep' == filename: ggrep = os.path.join(root, filename) for root, dir_names, file_names in os.walk( '/usr/local/Cellar/findutils'): for filename in file_names: if 'gfind' == filename: gfind = os.path.join(root, filename) if ggrep == '': logging.critical("brew install ggrep pleases!") sys.exit(0) else: grep = ggrep if gfind == '': logging.critical("brew install findutils pleases!") sys.exit(0) else: find = gfind """ all vulnerabilities vulnerabilities_all[vuln_id] = {'name': 'vuln_name', 'third_v_id': 'third_v_id'} """ vulnerabilities_all = {} vulnerabilities = CobraVuls.query.all() for v in vulnerabilities: vulnerabilities_all[v.id] = { 'name': v.name, 'third_v_id': v.third_v_id } for rule in rules: rule.regex_location = rule.regex_location.strip() rule.regex_repair = rule.regex_repair.strip() logging.info( '------------------\r\nScan rule id: {0} {1} {2}'.format( self.project_id, rule.id, rule.description)) # Filters for language in languages: if language.id == rule.language: extensions = language.extensions.split('|') if extensions is None: logging.critical("Rule Language Error") sys.exit(0) # White list white_list = [] ws = CobraWhiteList.query.filter_by(project_id=self.project_id, rule_id=rule.id, status=1).all() if ws is not None: for w in ws: white_list.append(w.path) try: if rule.regex_location == "": filters = [] for index, e in enumerate(extensions): if index > 1: filters.append('-o') filters.append('-name') filters.append('*' + e) # Find Special Ext Files param = [find, self.directory, "-type", "f"] + filters else: filters = [] for e in extensions: filters.append('--include=*' + e) # explode dirs explode_dirs = ['.svn', '.cvs', '.hg', '.git', '.bzr'] for explode_dir in explode_dirs: filters.append('--exclude-dir={0}'.format(explode_dir)) # -n Show Line number / -r Recursive / -P Perl regular expression param = [grep, "-n", "-r", "-P"] + filters + [ rule.regex_location, self.directory ] logging.debug(' '.join(param)) p = subprocess.Popen(param, stdout=subprocess.PIPE) result = p.communicate() # Exists result if len(result[0]): lines = str(result[0]).strip().split("\n") for line in lines: line = line.strip() if line == '': continue # 处理grep结果 if ':' in line: line_split = line.split(':', 1) file_path = line_split[0].strip() code_content = line_split[1].split(':', 1)[1].strip() line_number = line_split[1].split(':', 1)[0].strip() else: # 搜索文件 file_path = line code_content = '' line_number = 0 # 核心规则校验 result_info = { 'task_id': self.task_id, 'project_id': self.project_id, 'project_directory': self.directory, 'rule_id': rule.id, 'file_path': file_path, 'line_number': line_number, 'code_content': code_content, 'third_party_vulnerabilities_name': vulnerabilities_all[rule.vul_id]['name'], 'third_party_vulnerabilities_type': vulnerabilities_all[rule.vul_id]['third_v_id'] } ret_status, ret_result = Core(result_info, rule, self.project_name, white_list).scan() if ret_status is False: logging.info("扫描 R: False {0}".format(ret_result)) continue else: logging.info('Not Found') except Exception as e: print(traceback.print_exc()) logging.critical('Error calling grep: ' + str(e)) # Set End Time For Task t = CobraTaskInfo.query.filter_by(id=self.task_id).first() t.status = 2 t.file_count = files['file_nums'] t.time_end = int(time.time()) t.time_consume = t.time_end - t.time_start t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime()) try: db.session.add(t) db.session.commit() except Exception as e: logging.critical("Set start time failed:" + e.message) logging.info("Scan Done")
def analyse(self): if self.directory is None: log.critical("Please set directory") sys.exit() log.info('Start code static analyse...') d = directory.Directory(self.directory) files = d.collect_files() log.info('Scan Files: {0}, Total Time: {1}s'.format( files['file_nums'], files['collect_time'])) ext_language = { # Image '.jpg': 'image', '.png': 'image', '.bmp': 'image', '.gif': 'image', '.ico': 'image', '.cur': 'image', # Font '.eot': 'font', '.otf': 'font', '.svg': 'font', '.ttf': 'font', '.woff': 'font', # CSS '.css': 'css', '.less': 'css', '.scss': 'css', '.styl': 'css', # Media '.mp3': 'media', '.swf': 'media', # Execute '.exe': 'execute', '.sh': 'execute', '.dll': 'execute', '.so': 'execute', '.bat': 'execute', '.pl': 'execute', # Edit '.swp': 'tmp', # Cert '.crt': 'cert', # Text '.txt': 'text', '.csv': 'text', '.md': 'markdown', # Backup '.zip': 'backup', '.bak': 'backup', '.tar': 'backup', '.rar': 'backup', '.tar.gz': 'backup', '.db': 'backup', # Config '.xml': 'config', '.yml': 'config', '.spf': 'config', '.iml': 'config', '.manifest': 'config', # Source '.psd': 'source', '.as': 'source', # Log '.log': 'log', # Template '.template': 'template', '.tpl': 'template', } for ext in files: if ext in ext_language: log.info('{0} - {1}'.format(ext, files[ext])) continue else: log.info(ext) languages = CobraLanguages.query.all() rules = CobraRules.query.filter_by(status=1).all() extensions = None for rule in rules: for language in languages: if language.id == rule.language: extensions = language.extensions.split('|') if extensions is None: log.warning("Rule Language Error") # grep name is ggrep on mac grep = '/bin/grep' if 'darwin' == sys.platform: log.info('In Mac OS X System') for root, dir_names, file_names in os.walk( '/usr/local/Cellar/grep'): for filename in file_names: if 'ggrep' == filename: grep = os.path.join(root, filename) filters = [] for e in extensions: filters.append('--include=*' + e) # White list white_list = [] ws = CobraWhiteList.query.filter_by(project_id=self.project_id, rule_id=rule.id, status=1).all() if ws is not None: for w in ws: white_list.append(w.path) try: log.info('Scan rule id: {0}'.format(rule.id)) # -n Show Line number / -r Recursive / -P Perl regular expression p = subprocess.Popen([grep, "-n", "-r", "-P"] + filters + [rule.regex, self.directory], stdout=subprocess.PIPE) result = p.communicate() # Exists result if len(result[0]): log.info('Found:') per_line = str(result[0]).split("\n") log.debug(per_line) for r in range(0, len(per_line) - 1): try: rr = str(per_line[r]).replace(self.directory, '').split(':', 1) code = str(rr[1]).split(':', 1) if self.task_id is None: self.task_id = 0 rule_id = rule.id current_time = datetime.now().strftime( '%Y-%m-%d %H:%M:%S') m_file = rr[0].strip() m_line = code[0] m_code = str(code[1].strip()) params = [ self.task_id, rule_id, m_file, m_line, m_code, current_time, current_time ] try: if m_file in white_list or ".min.js" in m_file: log.debug("In White list or min.js") else: # # // /* * match_result = re.match( "(#)?(//)?(\*)?(/\*)?", m_code) if match_result.group( 0 ) is not None and match_result.group( 0) is not "": log.debug("In Annotation") else: log.debug('In Insert') if rule.regex == "": # Didn't filter line when regex is empty r_content = CobraResults.query.filter_by( task_id=self.task_id, rule_id=rule_id, file=m_file).first() m_line = 0 else: r_content = CobraResults.query.filter_by( task_id=self.task_id, rule_id=rule_id, file=m_file, line=m_line).first() if r_content is not None: log.warning("Exists Result") else: results = CobraResults( self.task_id, rule_id, m_file, m_line, m_code, current_time, current_time) db.session.add(results) db.session.commit() log.info('Insert Results Success') except Exception as e: log.error('Insert Results Failed' + str(e.message)) log.debug(params) except Exception as e: log.critical('Error parsing result: ' + str(e.message)) else: log.info('Not Found') except Exception as e: log.critical('Error calling grep: ' + str(e)) # Set End Time For Task t = CobraTaskInfo.query.filter_by(id=self.task_id).first() t.status = 2 t.file_count = files['file_nums'] t.time_end = int(time.time()) t.time_consume = t.time_end - t.time_start t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime()) try: db.session.add(t) db.session.commit() except Exception as e: log.critical("Set start time failed:" + e.message) log.info("Scan Done")
def analyse(self): if self.directory is None: log.critical("Please set directory") sys.exit() log.info('Start code static analyse...') d = directory.Directory(self.directory) files = d.collect_files(self.task_id) log.info('Scan Files: {0}, Total Time: {1}s'.format( files['file_nums'], files['collect_time'])) ext_language = { # Image '.jpg': 'image', '.png': 'image', '.bmp': 'image', '.gif': 'image', '.ico': 'image', '.cur': 'image', # Font '.eot': 'font', '.otf': 'font', '.svg': 'font', '.ttf': 'font', '.woff': 'font', # CSS '.css': 'css', '.less': 'css', '.scss': 'css', '.styl': 'css', # Media '.mp3': 'media', '.swf': 'media', # Execute '.exe': 'execute', '.sh': 'execute', '.dll': 'execute', '.so': 'execute', '.bat': 'execute', '.pl': 'execute', # Edit '.swp': 'tmp', # Cert '.crt': 'cert', # Text '.txt': 'text', '.csv': 'text', '.md': 'markdown', # Backup '.zip': 'backup', '.bak': 'backup', '.tar': 'backup', '.rar': 'backup', '.tar.gz': 'backup', '.db': 'backup', # Config '.xml': 'config', '.yml': 'config', '.spf': 'config', '.iml': 'config', '.manifest': 'config', # Source '.psd': 'source', '.as': 'source', # Log '.log': 'log', # Template '.template': 'template', '.tpl': 'template', } for ext in files: if ext in ext_language: log.info('{0} - {1}'.format(ext, files[ext])) continue else: log.info(ext) languages = CobraLanguages.query.all() rules = CobraRules.query.filter_by(status=1).all() extensions = None # `grep` (`ggrep` on Mac) grep = '/bin/grep' # `find` (`gfind` on Mac) find = '/bin/find' if 'darwin' == sys.platform: ggrep = '' gfind = '' for root, dir_names, file_names in os.walk( '/usr/local/Cellar/grep'): for filename in file_names: if 'ggrep' == filename: ggrep = os.path.join(root, filename) for root, dir_names, file_names in os.walk( '/usr/local/Cellar/findutils'): for filename in file_names: if 'gfind' == filename: gfind = os.path.join(root, filename) if ggrep == '': log.critical("brew install ggrep pleases!") sys.exit(0) else: grep = ggrep if gfind == '': log.critical("brew install findutils pleases!") sys.exit(0) else: find = gfind for rule in rules: log.info('Scan rule id: {0} {1} {2}'.format( self.project_id, rule.id, rule.description)) # Filters for language in languages: if language.id == rule.language: extensions = language.extensions.split('|') if extensions is None: log.critical("Rule Language Error") sys.exit(0) # White list white_list = [] ws = CobraWhiteList.query.filter_by(project_id=self.project_id, rule_id=rule.id, status=1).all() if ws is not None: for w in ws: white_list.append(w.path) try: if rule.regex.strip() == "": filters = [] for index, e in enumerate(extensions): if index > 1: filters.append('-o') filters.append('-name') filters.append('*' + e) # Find Special Ext Files param = [find, self.directory, "-type", "f"] + filters else: filters = [] for e in extensions: filters.append('--include=*' + e) # Explode SVN Dir filters.append('--exclude-dir=.svn') filters.append('--exclude-dir=.cvs') filters.append('--exclude-dir=.hg') filters.append('--exclude-dir=.git') filters.append('--exclude-dir=.bzr') filters.append('--exclude=*.svn-base') # -n Show Line number / -r Recursive / -P Perl regular expression param = [grep, "-n", "-r", "-P" ] + filters + [rule.regex, self.directory] # log.info(' '.join(param)) p = subprocess.Popen(param, stdout=subprocess.PIPE) result = p.communicate() # Exists result if len(result[0]): lines = str(result[0]).split("\n") for line in lines: line = line.strip() if line == '': continue if rule.regex.strip() == '': # Find file_path = line.strip().replace( self.directory, '') log.debug('File: {0}'.format(file_path)) vul = CobraResults(self.task_id, rule.id, file_path, 0, '') db.session.add(vul) else: # Grep line_split = line.replace(self.directory, '').split(':', 1) file_path = line_split[0].strip() code_content = line_split[1].split(':', 1)[1].strip() line_number = line_split[1].split(':', 1)[0].strip() if file_path in white_list or ".min.js" in file_path: log.info("In white list or min.js") else: # Annotation # # // /* * match_result = re.match( "(#)?(//)?(\*)?(/\*)?", code_content) if match_result.group( 0) is not None and match_result.group( 0) is not "": log.info("In Annotation") else: log.info('In Insert') exist_result = CobraResults.query.filter_by( task_id=self.task_id, rule_id=rule.id, file=file_path, line=line_number).first() if exist_result is not None: log.warning("Exists Result") else: log.debug('File: {0}:{1} {2}'.format( file_path, line_number, code_content)) vul = CobraResults( self.task_id, rule.id, file_path, line_number, code_content) db.session.add(vul) log.info('Insert Results Success') db.session.commit() else: log.info('Not Found') except Exception as e: log.critical('Error calling grep: ' + str(e)) # Set End Time For Task t = CobraTaskInfo.query.filter_by(id=self.task_id).first() t.status = 2 t.file_count = files['file_nums'] t.time_end = int(time.time()) t.time_consume = t.time_end - t.time_start t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime()) try: db.session.add(t) db.session.commit() except Exception as e: log.critical("Set start time failed:" + e.message) log.info("Scan Done")