def collect_files(self, task_id=None): t1 = time.clock() self.files(self.path) self.result['no_extension'] = {'file_count': 0, 'file_list': []} for extension, values in self.type_nums.items(): extension = extension.strip() self.result[extension] = {'file_count': len(values), 'file_list': []} # .php : 123 logging.debug('{0} : {1}'.format(extension, len(values))) if task_id is not None: # Store ext = CobraExt(task_id, extension, len(values)) db.session.add(ext) for f in self.file: es = f.split(os.extsep) if len(es) >= 2: # Exists Extension # os.extsep + es[len(es) - 1] if f.endswith(extension): self.result[extension]['file_list'].append(f) else: # Didn't have extension self.result['no_extension']['file_count'] = int(self.result['no_extension']['file_count']) + 1 self.result['no_extension']['file_list'].append(f) if task_id is not None: db.session.commit() t2 = time.clock() self.result['file_nums'] = self.file_id self.result['collect_time'] = t2 - t1 return self.result
def framework(self): """ Detection framework for project :param: framework | language :return: self.rules['name'] """ for rule in self.rules: rules_types = ['file', 'directory'] rules_count = len(rule['rules']) rules_completed = 0 logging.info("------ {0} (C: {1})".format(rule['name'], rules_count)) for rule_type in rules_types: if rule_type in rule['rules']: target = os.path.join(self.project_directory, rule['rules'][rule_type]) logging.debug('{0}: {1}'.format(rule_type, target)) if rule_type == 'file': if os.path.isfile(target): rules_completed += 1 elif rule_type == 'directory': if os.path.isdir(target): rules_completed += 1 if rules_completed == rules_count: logging.info("Framework: {0}".format(rule['name'])) return rule['name'], rule['language'] return '', ''
def pull(self): """Pull a repo from repo_address and repo_directory""" logging.info('Start Pull Repo...') if not self.__check_exist(): return False, 'No local repo exist. Please clone first.' # change work directory to the repo repo_dir = self.repo_directory logging.debug('cd directory: {0}'.format(repo_dir)) os.chdir(repo_dir) cmd = 'git pull origin master' p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) (pull_out, pull_err) = p.communicate() logging.info(pull_out) logging.info(pull_err) self.parse_err(pull_err) pull_err = pull_err.replace('{0}:{1}'.format(self.repo_username, self.repo_password), '') # change work directory back. os.chdir(repo_dir) if 'Updating' in pull_out or 'up-to-date' in pull_out: logging.info('Pull done.') return True, None else: return False, pull_err
def log(self, level, message, test=True): if test: self.data.append('[{0}] {1}'.format(level.upper(), message)) if level == 'critical': logging.critical(message) elif level == 'warning': logging.warning(message) elif level == 'info': logging.info(message) elif level == 'debug': logging.debug(message) elif level == 'error': logging.error(message)
def files(self, directory, level=1): if level == 1: logging.debug(directory) for filename in os.listdir(directory): path = os.path.join(directory, filename) # Directory Structure # logging.debug('| ' * (level - 1) + '|--' + filename) if os.path.isdir(path): self.files(path, level + 1) if os.path.isfile(path): # Statistic File Type Count file_name, file_extension = os.path.splitext(path) self.type_nums.setdefault(file_extension.lower(), []).append(filename) path = path.replace(self.path, '') self.file.append(path) self.file_id += 1 logging.debug("{0}, {1}".format(self.file_id, path))
def count_by_time(start, end): count = db.session.query( func.count(CobraResults.id).label('count'), CobraResults.status ).filter( CobraResults.created_at >= '{start} 00:00:00'.format(start=start), CobraResults.created_at <= '{end} 23:59:59'.format(end=end), # Active project CobraProjects.status > 0, CobraResults.project_id == CobraProjects.id ).group_by(CobraResults.status).all() logging.debug('VT {start} {end} {count}'.format(start=start, end=end, count=count)) c_dict = {} for ci in count: count, status = ci c_dict[status] = count if 0 not in c_dict: c_dict[0] = 0 if 1 not in c_dict: c_dict[1] = 0 if 2 not in c_dict: c_dict[2] = 0 return c_dict
def count_by_time(start, end, t='task'): filter_group = ( CobraTaskInfo.created_at >= '{0} 00:00:00'.format(start), CobraTaskInfo.created_at <= '{0} 23:59:59'.format(end), # Active project CobraProjects.status > 0, CobraProjects.repository == CobraTaskInfo.target ) count = 0 if t == 'task': count = db.session.query( func.count(CobraTaskInfo.id).label('count') ).filter( *filter_group ).first() elif t == 'project': count = db.session.query( func.count(func.distinct(CobraTaskInfo.target)).label('count') ).filter( *filter_group ).first() elif t == 'line': count = db.session.query( func.sum(CobraTaskInfo.code_number).label('count') ).filter( *filter_group ).first() elif t == 'file': count = db.session.query( func.sum(CobraTaskInfo.file_count).label('count') ).filter( *filter_group ).first() if count[0] is None: return 0 else: logging.debug('SD {t} {start} {end} {count}'.format(start=start, end=end, t=t, count=int(count[0]))) return int(count[0])
def __init__(self, filename, current_version=None, online_version=None): self.filename = filename self.current_version = current_version self.online_version = online_version self.username = config.Config('svn', 'username').value self.password = config.Config('svn', 'password').value # Test SVN cmd = self.svn + " info --no-auth-cache --non-interactive --username='******' --password='******' %s" % ( self.username, self.password, self.filename) p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) (diff_out, diff_err) = p.communicate() if len(diff_err) == 0: logging.debug("svn diff success") elif 'authorization failed' in diff_err: logging.warning("svn diff auth failed") sys.exit(1) elif 'Not a valid URL' in diff_err: logging.warning("svn diff url not a valid") sys.exit(1)
def __init__(self, filename, current_version=None, online_version=None): self.filename = filename self.current_version = current_version self.online_version = online_version self.username = config.Config('svn', 'username').value self.password = config.Config('svn', 'password').value # Test SVN cmd = self.svn + " info --no-auth-cache --non-interactive --username='******' --password='******' %s" % ( self.username, self.password, self.filename ) p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) (diff_out, diff_err) = p.communicate() if len(diff_err) == 0: logging.debug("svn diff success") elif 'authorization failed' in diff_err: logging.warning("svn diff auth failed") sys.exit(1) elif 'Not a valid URL' in diff_err: logging.warning("svn diff url not a valid") sys.exit(1)
def send_email(report_file=''): report_file = report_file or get_last_report() config = Config() email_config = config.get_email_config() smtp_server = email_config.get("smtp_server") or 'mail.163.com' smtp_user = email_config.get('smtp_user') or '*****@*****.**' smtp_password = email_config.get('smtp_password') or 'hanzhichao123' subject = email_config.get('subject') or 'Test Report' body = email_config.get('body') or 'hi, all<br>Test complete, please see the attachments' receivers = email_config.get('receivers') msg = MIMEMultipart() msg.attach(MIMEText(body, 'html', 'utf-8')) msg['From'] = smtp_user msg['To'] = '' msg['Subject'] = Header(subject, 'utf-8') att1 = MIMEText(open(report_file, 'rb').read(), 'base64', 'utf-8') att1["Content-Type"] = 'application/octet-stream' att1["Content-Disposition"] = 'attachment; filename="{}"'.format(os.path.basename(report_file)) msg.attach(att1) try: logging.debug("连接SMTP服务器: {}".format(smtp_server)) smtp = smtplib.SMTP(smtp_server) # 从配置文件中读取 logging.debug("登录SMTP服务器 用户名: {} 密码: {}".format(smtp_user, smtp_password)) smtp.login(smtp_user, smtp_password) # 从配置文件中读取 for receiver in receivers: msg['To'] = receiver logging.debug("由{}发送邮件给: {}".format(smtp_user, receiver)) smtp.sendmail(smtp_user, receiver, msg.as_string()) logging.info("邮件发送完成!") except Exception as e: logging.error(str(e)) finally: smtp.quit()
def click(self, *args): """点击元素,支持id,class name,xpath, text, msg, 点击不到则截图并报错""" logging.debug("点击: {}".format(args)) self.find(*args).click()
def is_controllable_param(self): """ 参数是否可控 :return: """ param_name = re.findall(self.rule, self.code) if len(param_name) == 1: param_name = param_name[0].strip() self.param_name = param_name logging.debug('参数: `{0}`'.format(param_name)) # 固定字符串判断 regex_string = self.regex[self.language]['string'] string = re.findall(regex_string, param_name) if len(string) >= 1 and string[0] != '': logging.debug("是否字符串: 是") logging.info("返回: 不可控 (字符串)") return False logging.debug("是否字符串: 否") # 变量判断 if param_name[:1] == '$': logging.debug("参数是否变量: 是") # 获取参数赋值代码块 param_block_code = self.block_code(0) if param_block_code is False: logging.debug("向上搜索参数区块代码: 未找到") logging.info("返回: 可控 (代码未找到)") return True logging.debug("向上搜索参数区块代码: {0}".format(param_block_code)) # 外部取参赋值 """ # Need match $url = $_GET['test']; $url = $_POST['test']; $url = $_REQUEST['test']; $url = $_SERVER['user_agent']; # Don't match $url = $_SERVER $url = $testsdf; """ regex_get_param = r'({0}\s*=\s*\$_[GET|POST|REQUEST|SERVER]+(?:\[))'.format( re.escape(param_name)) regex_get_param_result = re.findall(regex_get_param, param_block_code) if len(regex_get_param_result) >= 1: self.param_value = regex_get_param_result[0] logging.debug("参数是否直接取自外部: 是") logging.info("返回: 可控(取外部入参)") return True logging.debug("参数是否直接取自外部入参: 否") # 函数入参 regex_function_param = r'(function\s*\w+\s*\(.*{0})'.format( re.escape(param_name)) regex_function_param_result = re.findall( regex_function_param, param_block_code) if len(regex_function_param_result) >= 1: self.param_value = regex_function_param_result[0] logging.debug("参数是否函数入参: 是") logging.info("返回: 可控(函数入参)") return True logging.debug("参数是否直接函数入参: 否") # 常量赋值 uc_rule = r'{0}\s?=\s?([A-Z_]*)'.format(re.escape(param_name)) uc_rule_result = re.findall(uc_rule, param_block_code) if len(uc_rule_result) >= 1: logging.debug("参数变量是否直接赋值常量: 是 `{0} = {1}`".format( param_name, uc_rule_result[0])) logging.info("返回: 不可控") return False logging.debug("参数变量是否直接赋值常量: 否") # 固定字符串判断 regex_assign_string = self.regex[ self.language]['assign_string'].format( re.escape(param_name)) string = re.findall(regex_assign_string, param_block_code) if len(string) >= 1 and string[0] != '': logging.debug("是否赋值字符串: 是") logging.info("返回: 不可控 (字符串)") return False logging.debug("是否赋值字符串: 否") logging.info("返回: 可控(默认情况)") return True else: if self.language == 'java': # Java 变量就是没有$ param_block_code = self.block_code(0) if param_block_code is False: logging.debug("向上搜索参数区块代码: 未找到") logging.info("返回: 可控 (代码未找到)") return True logging.debug("向上搜索参数区块代码: {0}".format(param_block_code)) regex_assign_string = self.regex[ self.language]['assign_string'].format( re.escape(param_name)) string = re.findall(regex_assign_string, param_block_code) if len(string) >= 1 and string[0] != '': logging.debug("是否赋值字符串: 是") logging.info("返回: 不可控 (字符串)") return False logging.debug("是否赋值字符串: 否") # 是否取外部参数 regex_get_param = r'String\s{0}\s=\s\w+\.getParameter(.*)'.format( re.escape(param_name)) get_param = re.findall(regex_get_param, param_block_code) if len(get_param) >= 1 and get_param[0] != '': logging.debug("是否赋值外部取参: 是") logging.info("返回: 不可控 (外部取参)") return False logging.debug("是否赋值外部取参: 否") logging.info("返回: 可控 (变量赋值)") return True logging.debug("参数是否变量: 否 (没有包含$)") logging.info("返回: 不可控(参数不为变量)") return False else: logging.warning("未获取到参数名,请检查定位规则")
def analyse(self): if self.directory is None: logging.critical("Please set directory") sys.exit() logging.info('Start code static analyse...') d = directory.Directory(self.directory) files = d.collect_files(self.task_id) logging.info('Scan Files: {0}, Total Time: {1}s'.format( files['file_nums'], files['collect_time'])) ext_language = { # Image '.jpg': 'image', '.png': 'image', '.bmp': 'image', '.gif': 'image', '.ico': 'image', '.cur': 'image', # Font '.eot': 'font', '.otf': 'font', '.svg': 'font', '.ttf': 'font', '.woff': 'font', # CSS '.css': 'css', '.less': 'css', '.scss': 'css', '.styl': 'css', # Media '.mp3': 'media', '.swf': 'media', # Execute '.exe': 'execute', '.sh': 'execute', '.dll': 'execute', '.so': 'execute', '.bat': 'execute', '.pl': 'execute', # Edit '.swp': 'tmp', # Cert '.crt': 'cert', # Text '.txt': 'text', '.csv': 'text', '.md': 'markdown', # Backup '.zip': 'backup', '.bak': 'backup', '.tar': 'backup', '.rar': 'backup', '.tar.gz': 'backup', '.db': 'backup', # Config '.xml': 'config', '.yml': 'config', '.spf': 'config', '.iml': 'config', '.manifest': 'config', # Source '.psd': 'source', '.as': 'source', # Log '.log': 'log', # Template '.template': 'template', '.tpl': 'template', } for ext in files: if ext in ext_language: logging.info('{0} - {1}'.format(ext, files[ext])) continue else: logging.info(ext) languages = CobraLanguages.query.all() rules = CobraRules.query.filter_by(status=1).all() extensions = None # `grep` (`ggrep` on Mac) grep = '/bin/grep' # `find` (`gfind` on Mac) find = '/bin/find' if 'darwin' == sys.platform: ggrep = '' gfind = '' for root, dir_names, file_names in os.walk( '/usr/local/Cellar/grep'): for filename in file_names: if 'ggrep' == filename or 'grep' == filename: ggrep = os.path.join(root, filename) for root, dir_names, file_names in os.walk( '/usr/local/Cellar/findutils'): for filename in file_names: if 'gfind' == filename: gfind = os.path.join(root, filename) if ggrep == '': logging.critical("brew install ggrep pleases!") sys.exit(0) else: grep = ggrep if gfind == '': logging.critical("brew install findutils pleases!") sys.exit(0) else: find = gfind """ all vulnerabilities vulnerabilities_all[vuln_id] = {'name': 'vuln_name', 'third_v_id': 'third_v_id'} """ vulnerabilities_all = {} vulnerabilities = CobraVuls.query.all() for v in vulnerabilities: vulnerabilities_all[v.id] = { 'name': v.name, 'third_v_id': v.third_v_id } for rule in rules: rule.regex_location = rule.regex_location.strip() rule.regex_repair = rule.regex_repair.strip() logging.info('------------------ RID: {0} {1} {2}'.format( self.project_id, rule.id, rule.description)) # Filters for language in languages: if language.id == rule.language: extensions = language.extensions.split('|') if extensions is None: logging.critical("Rule Language Error") sys.exit(0) # White list white_list = [] ws = CobraWhiteList.query.filter_by(project_id=self.project_id, rule_id=rule.id, status=1).all() if ws is not None: for w in ws: white_list.append(w.path) try: if rule.regex_location == "": filters = [] for index, e in enumerate(extensions): if index > 1: filters.append('-o') filters.append('-name') filters.append('*' + e) # Find Special Ext Files param = [find, self.directory, "-type", "f"] + filters else: filters = [] for e in extensions: filters.append('--include=*' + e) # explode dirs explode_dirs = ['.svn', '.cvs', '.hg', '.git', '.bzr'] for explode_dir in explode_dirs: filters.append('--exclude-dir={0}'.format(explode_dir)) # -s suppress error messages / -n Show Line number / -r Recursive / -P Perl regular expression param = [grep, "-s", "-n", "-r", "-P"] + filters + [ rule.regex_location, self.directory ] logging.debug(rule.regex_location) p = subprocess.Popen(param, stdout=subprocess.PIPE) result = p.communicate() # Exists result if len(result[0]): lines = str(result[0]).strip().split("\n") for line in lines: line = line.strip() if line == '': continue # 处理grep结果 if ':' in line: line_split = line.split(':', 1) file_path = line_split[0].strip() code_content = line_split[1].split(':', 1)[1].strip() line_number = line_split[1].split(':', 1)[0].strip() else: # 搜索文件 file_path = line code_content = '' line_number = 0 # 核心规则校验 result_info = { 'task_id': self.task_id, 'project_id': self.project_id, 'project_directory': self.directory, 'rule_id': rule.id, 'result_id': None, 'file_path': file_path, 'line_number': line_number, 'code_content': code_content, 'third_party_vulnerabilities_name': vulnerabilities_all[rule.vul_id]['name'], 'third_party_vulnerabilities_type': vulnerabilities_all[rule.vul_id]['third_v_id'] } ret_status, ret_result = Core(result_info, rule, self.project_name, white_list).scan() if ret_status is False: logging.info("扫描 R: False {0}".format(ret_result)) continue else: logging.info('Not Found') except Exception as e: print(traceback.print_exc()) logging.critical('Error calling grep: ' + str(e)) # Set End Time For Task t = CobraTaskInfo.query.filter_by(id=self.task_id).first() t.status = 2 t.file_count = files['file_nums'] t.time_end = int(time.time()) t.time_consume = t.time_end - t.time_start t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime()) try: db.session.add(t) db.session.commit() except Exception as e: logging.critical("Set start time failed:" + e.message) logging.info("Scan Done")
def process_vulnerabilities(self): """ Process vulnerabilities Write vulnerabilities / change the vulnerability status / push third-party vulnerability management platform :return: """ # default status if self.status is None: self.status = self.status_init # Handle relative paths self.file_path = self.file_path.replace(self.project_directory, '') # In scan if self.method == 0: """ On Scan mode(method=1) only deal to new not fixed vulnerability(status=0) """ if self.status == 0: # Check whether the unpatched vulnerability has been swept # Line number 0 when the search for special documents (not to bring the line number) if self.line_number == 0: exist_result = CobraResults.query.filter( CobraResults.project_id == self.project_id, CobraResults.rule_id == self.rule_id, CobraResults.file == self.file_path, ).order_by(CobraResults.id.desc()).first() else: exist_result = CobraResults.query.filter( CobraResults.project_id == self.project_id, CobraResults.rule_id == self.rule_id, CobraResults.file == self.file_path, CobraResults.line == self.line_number, ).order_by(CobraResults.id.desc()).first() # Not fixed vulnerabilities have not been swept before, is written to the vulnerability database if exist_result is None: self.insert_vulnerabilities() else: logging.debug("This vulnerabilities already exist!") elif self.method == 1: """ On Repair (method=1) """ if self.status == self.status_fixed: # View the original status of the vulnerability exist_result = CobraResults.query.filter_by( id=self.result_id).first() if exist_result is not None: if exist_result.status < self.status_fixed: # update vulnerability status is Fixed exist_result.status = self.status_fixed exist_result.repair = self.repair_code db.session.add(exist_result) db.session.commit() else: logging.critical("Vulnerabilities status not < fixed") else: logging.critical( "Not found vulnerabilities on repair method") else: logging.critical("Core method not initialize")
def try_find_element(self, *args): """尝试定位元素,找不到不报错""" try: return self.find_element(*args) except NoSuchElementException: logging.debug("元素: {} 未被定位到".format(args))
def type_and_search(self, *args, text=''): """输入并按搜索键""" self.type(*args, text=text) logging.debug("按搜索键") self.driver.keyevent(84)
def type_and_enter(self, *args, text=''): """输入并按回车键""" self.type(*args, text=text) logging.debug("按回车键") self.driver.keyevent(66)
def wait_click(self, *args): """等待并点击元素,定位不到报超时错误""" logging.debug("等待点击: {}".format(args)) self.wait_element(*args).click()
def click_id(self, *args): """点击指定resource_id元素""" logging.debug("点击指定id: {} 元素".format(args)) self.find_id(*args).click()
def click_text(self, *args): """点击指定文本元素""" logging.debug("点击文本: {}".format(args)) self.find_text(*args).click()
def is_login(self): if self.try_find_element(*self.MIME_ICO): logging.debug("定位到用户头像") return True else: logging.debug("未定位到用户头像")