def framework(self): """ Detection framework for project :param: framework | language :return: self.rules['name'] """ for rule in self.rules: rules_types = ['file', 'directory'] rules_count = len(rule['rules']) rules_completed = 0 logging.info("------ {0} (C: {1})".format(rule['name'], rules_count)) for rule_type in rules_types: if rule_type in rule['rules']: target = os.path.join(self.project_directory, rule['rules'][rule_type]) logging.debug('{0}: {1}'.format(rule_type, target)) if rule_type == 'file': if os.path.isfile(target): rules_completed += 1 elif rule_type == 'directory': if os.path.isdir(target): rules_completed += 1 if rules_completed == rules_count: logging.info("Framework: {0}".format(rule['name'])) return rule['name'], rule['language'] return '', ''
def checkout(self, branch): """ Checkout to special branch. :param branch: branch name :return: True-checkout success or already on special branch False-checkout failed. Maybe no branch name. """ if not self.__check_exist(): logging.info('No repo directory.') return False current_dir = os.getcwd() os.chdir(self.repo_directory) cmd = "git checkout " + branch p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) (checkout_out, checkout_err) = p.communicate() logging.info(checkout_err) # Already on # did not match # Switched to a new branch if 'did not match' in checkout_err: os.chdir(current_dir) return False else: os.chdir(current_dir) return True
def run(self, is_all=None, target=None, tid=None, pid=None): if bool(is_all) is True: logging.info('[START] Scan all projects') scan.Scan().all() logging.info('[END] Scan all projects') else: if target is None: logging.critical("Please set --target param") sys.exit() if tid is not None: task_id = tid # Start Time For Task t = CobraTaskInfo.query.filter_by(id=tid).first() if t is None: logging.critical("Task id doesn't exists.") sys.exit() if t.status not in [0, 1]: logging.critical("Task Already Scan.") sys.exit() t.status = 1 t.time_start = int(time.time()) t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime()) try: db.session.add(t) db.session.commit() except Exception as e: logging.error("Set start time failed" + str(e.message)) else: task_id = None if os.path.isdir(target) is not True: logging.critical('Target is not directory') sys.exit() from engine import static static.Static(target, task_id=task_id, project_id=pid).analyse()
def push(self): try: # 为杜绝前面环节问题导致输出重复,所以推送前先检查是否已经推送过 exist_vuln = CobraResults.query.filter_by(id=self.vuln_id, status=2).count() if exist_vuln == 0: logging.info("已经推送过") return False vulns = {'info': json.dumps(self.vulnerabilities)} response = requests.post(self.api, data=vulns) if response.text == 'done': logging.info('推送漏洞到第三方漏洞管理平台成功') """ 更新漏洞状态 1. 漏洞状态是初始化(0) -> 更新(1) 2. 漏洞状态是已推送(1) -> 不更新 3. 漏洞状态是已修复(2) -> 不更新 """ if self.vuln_id is None: logging.warning("漏洞ID不能为空") else: vuln = CobraResults.query.filter_by(id=self.vuln_id).first() if vuln.status == 0: vuln.status = 1 db.session.add(vuln) db.session.commit() return True else: logging.critical('推送第三方漏洞管理平台失败 \r\n{0}'.format(response.text)) return False except (requests.ConnectionError, requests.HTTPError) as e: logging.warning("推送第三方漏洞管理平台出现异常: {0}".format(e)) return False
def game(self): """猜数字游戏""" data=number1.get() try: number3=int(data) if number>number3: number_small=tk.Toplevel() number_small.geometry("220x120") number_small.title("错误提示") Label(number_small,text = '您猜的数字小了哟').pack() Button(number_small, text="返回",command=number_small.destroy).pack() elif number<number3: number_largr=tk.Toplevel() number_largr.geometry("220x120") number_largr.title("错误提示") Label(number_largr,text = '您猜的数字大了哟').pack() Button(number_largr, text="返回",command=number_largr.destroy).pack() else: number_success=tk.Toplevel() number_success.geometry("220x120") number_success.title("错误提示") Label(number_success,text = '恭喜您猜对了哟').pack() Button(number_success, text="返回",command=number_success.destroy).pack() except Exception as e: number_fial=tk.Toplevel() number_fial.geometry("220x120") number_fial.title("错误提示") Label(number_fial,text = '请输入数字').pack() Button(number_fial, text="返回",command=number_fial.destroy).pack() logging.info('请输入数字:{}'.format(e)) print('请输入数字:{}'.format(e))
def toLogin(self): """登录窗体页面""" global number global number1 global loginsucces name=entry_name.get() #获取文本输入框的get方法 pwd=entry_pwd.get() try: pwd1=db.query("select password from test_user where name =%s",(name)) res=pwd1[0][0] # print(res) if pwd==res: print("恭喜您登录成功!!!") loginsucces = tk.Toplevel() loginsucces.geometry("600x600") loginsucces.title("登录成功") # menu1=UserMenu(loginsucces) menu=Menu(loginsucces) loginsucces.config(menu=menu) file1=Menu(menu) file1.add_command(label="Save") file1.add_command(label="Exit",command=menu1.exit) menu.add_cascade(label='File',menu=file1) edit=Menu(menu) edit.add_command(label="Undo") edit.add_command(label="Show Image",command=self.showImg) menu.add_cascade(label='Edit',menu=edit) user=Menu(menu) user.add_command(label="update",command=self.update_user_ui) # user.add_command(label="userinfo",command=self.show_userinfo) menu.add_cascade(label='User',menu=user) data_number=tk.StringVar() Entry(loginsucces,textvariable=data_number).grid(row=1,column=0) number1=Entry(loginsucces,textvariable=data_number) number=random.randint(0,100) print(number) Button(loginsucces, text="确定", command=self.game).grid(row=1,column=1) dataText=Text(loginsucces,width=67, height=60) dataText.grid(row=2, column=0) # dataText.pack() else: loginfial=tk.Toplevel() loginfial.geometry("220x120") loginfial.title("密码错误") Label(loginfial,text = '密码错误').pack() Button(loginfial, text="返回",command=loginfial.destroy).pack() print("密码错误,请重试!!!") except Exception as e: loginfial1=tk.Toplevel() loginfial1.geometry("220x120") loginfial1.title("密码错误") Label(loginfial1,text = '您输入的账号或密码错误').pack() Button(loginfial1, text="返回",command=loginfial1.destroy).pack() logging.info('您输入的账号或密码错误:{}'.format(e))
def committer(file, path, line_number, length=1): """ git blame -L21,+1 -- git.py 362d5798 (wufeifei 2016-09-10 12:19:44 +0800 21) logging = logging.getLogger(__name__) (?:.{8}\s\()(.*)\s(\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}) group #1: wufeifei group #2: 2016-09-10 12:19:44 :param file: :param path: :param line_number: :param length: :return: group#1, group#2 """ os.chdir(path) cmd = "git blame -L{0},+{1} -- {2}".format(line_number, length, file) p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) (checkout_out, checkout_err) = p.communicate() logging.info(checkout_err) if len(checkout_out) != 0: group = re.findall( r'(?:.{8}\s\()(.*)\s(\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2})', checkout_out) return True, group[0][0], group[0][1] else: return False, None, None
def run(self, target=None, tid=None): if target is None: logging.critical("Please set --target param") sys.exit() if tid is None: logging.critical("Please set --tid param") sys.exit() # Statistic Code p = subprocess.Popen(['cloc', target], stdout=subprocess.PIPE) (output, err) = p.communicate() rs = output.split("\n") for r in rs: r_e = r.split() if len(r_e) > 3 and r_e[0] == 'SUM:': t = CobraTaskInfo.query.filter_by(id=tid).first() if t is not None: t.code_number = r_e[4] try: db.session.add(t) db.session.commit() logging.info("Statistic code number done") except Exception as e: logging.error("Statistic code number failed" + str(e.message))
def test_search_normal(self): """正常搜索测试""" keyword = 'lv' logging.info("正常搜索测试,关键词:{}".format(keyword)) user = SearchView(self.driver) user.search(keyword) self.assertEqual(keyword, user.get_search_text())
def push(self): try: # 为杜绝前面环节问题导致输出重复,所以推送前先检查是否已经推送过 exist_vuln = CobraResults.query.filter_by(id=self.vuln_id, status=2).count() if exist_vuln == 0: logging.info("已经推送过") return False vulns = {'info': json.dumps(self.vulnerabilities)} response = requests.post(self.api, data=vulns) if response.text == 'done': logging.info('推送漏洞到第三方漏洞管理平台成功') """ 更新漏洞状态 1. 漏洞状态是初始化(0) -> 更新(1) 2. 漏洞状态是已推送(1) -> 不更新 3. 漏洞状态是已修复(2) -> 不更新 """ if self.vuln_id is None: logging.warning("漏洞ID不能为空") else: vuln = CobraResults.query.filter_by( id=self.vuln_id).first() if vuln.status == 0: vuln.status = 1 db.session.add(vuln) db.session.commit() return True else: logging.critical('推送第三方漏洞管理平台失败 \r\n{0}'.format(response.text)) return False except (requests.ConnectionError, requests.HTTPError) as e: logging.warning("推送第三方漏洞管理平台出现异常: {0}".format(e)) return False
def pull_code(self, branch='master'): logging.info('Gitlab project') # Git if 'gitlab' in self.target: username = config.Config('git', 'username').value password = config.Config('git', 'password').value else: username = None password = None gg = git.Git(self.target, branch=branch, username=username, password=password) # Git Clone Error try: clone_ret, clone_err = gg.clone() if clone_ret is False: return 4001, 'Clone Failed ({0})'.format(clone_err), gg except NotExistError: # update project status p = CobraProjects.query.filter_by(repository=self.target).first() if p is not None: if p.status == CobraProjects.get_status('on'): p.status = CobraProjects.get_status('off') db.session.add(p) db.session.commit() return 4001, 'Repository Does not exist!', gg except AuthError: logging.critical('Git Authentication Failed') return 4001, 'Repository Authentication Failed', gg return 1001, 'Success', gg
def framework(self): """ Detection framework for project :param: framework | language :return: self.rules['name'] """ for rule in self.rules: rules_types = ['file', 'directory'] rules_count = len(rule['rules']) rules_completed = 0 logging.info("------ {0} COUNT: {1}".format(rule['name'], rules_count)) for rule_type in rules_types: if rule_type in rule['rules']: target = os.path.join(self.project_directory, rule['rules'][rule_type]) logging.debug('{0}: {1}'.format(rule_type, target)) if rule_type == 'file': if os.path.isfile(target): rules_completed += 1 elif rule_type == 'directory': if os.path.isdir(target): rules_completed += 1 if rules_completed == rules_count: logging.info("Framework: {0}".format(rule['name'])) return rule['name'], rule['language'] return '', ''
def click_watch_banner(self): self.boot_home() logging.info("等待点击腕表推荐Banner") self.wait_click(*self.BANNER) logging.debug("所有上下文: {}".format(self.driver.contexts)) logging.info("切换上下文到:{}".format("WEBVIEW_com.secoo")) self.driver.switch_to.context('WEBVIEW_com.secoo')
def compress(self): dc = decompress.Decompress(self.target) ret, result_d = dc.decompress() if ret is False: return 1002, result_d else: directory = result_d logging.info("Scan directory: {0}".format(directory)) current_time = time.strftime('%Y-%m-%d %X', time.localtime()) p = CobraProjects.query.filter_by(repository=directory).first() # detection framework for project framework, language = detection.Detection(directory).framework() if framework != '' or language != '': project_framework = '{0} ({1})'.format(framework, language) else: project_framework = '' if not p: # insert into project table. repo_name = directory.split('/')[-1] project = CobraProjects(directory, '', repo_name, 'Upload', project_framework, '', '', 1, current_time) db.session.add(project) db.session.commit() project_id = project.id else: project_id = p.id # update project's framework p.framework = project_framework db.session.add(p) task = CobraTaskInfo(directory, '', 3, '', '', 0, 0, 0, 1, 0, 0, current_time, current_time) db.session.add(task) db.session.commit() cobra_path = os.path.join(config.Config().project_directory, 'cobra.py') if os.path.isfile(cobra_path) is not True: return 1004, 'Cobra Not Found' # 扫描漏洞 subprocess.Popen([ 'python', cobra_path, "scan", "-p", str(project_id), "-i", str(task.id), "-t", directory ]) # 统计代码行数 subprocess.Popen([ 'python', cobra_path, "statistic", "-i", str(task.id), "-t", directory ]) # 检测漏洞修复状况 subprocess.Popen( ['python', cobra_path, "repair", "-p", str(project_id)]) result = dict() result['scan_id'] = task.id result['project_id'] = project_id result['msg'] = u'success' return 1001, result
def __init__(self, repo_address, branch='master', username=None, password=None): # get upload directory self.upload_directory = os.path.join( config.Config('upload', 'directory').value, 'versions') if os.path.isdir(self.upload_directory) is False: os.makedirs(self.upload_directory) self.repo_address = repo_address self.repo_username = username self.repo_password = password self.repo_branch = branch repo_user = self.repo_address.split('/')[-2] repo_name = self.repo_address.split('/')[-1].replace('.git', '') self.repo_author = repo_user self.repo_name = repo_name self.repo_directory = os.path.join( os.path.join(self.upload_directory, repo_user), repo_name) logging.info('Git class init.')
def boot_app(cls): config = Config() run_config = config.get_run_config() app = Config.app or run_config.get('app') device = Config.device or run_config.get('device') hub = Config.hub or run_config.get('hub') data = load_yaml('conf', 'devices') common = data.get('common') or {} if not common or not isinstance(common, dict): raise ValueError("app配置错误!") app = data.get('apps').get(app) if not app or not isinstance(app, dict): raise ValueError("app配置错误!") device = data.get('devices').get(device) if not device or not isinstance(device, dict): raise ValueError("device配置错误!") hub = data.get('hubs').get(hub) if not hub or not isinstance(hub, str): raise ValueError("hub配置错误!") time_out = run_config.get("implicitly_wait") or 5 desired_caps = dict(common, **app, **device) logging.debug("当前desired_caps: {}".format(desired_caps)) logging.debug("当前hub: {}".format(hub)) logging.info("开始启动app") dr = webdriver.Remote(hub, desired_caps) dr.implicitly_wait(time_out) return dr
def setUp(self): warnings.simplefilter("ignore", ResourceWarning) warnings.simplefilter("ignore", DeprecationWarning) warnings.simplefilter("ignore", InsecureRequestWarning) logging.info("{} setUp {}".format("-" * 5, "-" * 5)) self.driver = Device.boot_app()
def cancel_all_orders(self): logging.info("取消所有订单") orders = self.find_elements(*self.ORDERS) for order in orders: order.click() self.click(*self.CANCEL_ORDER_BTN) self.click(*self.SURE_CANCLE_BTN) self.driver.keyevent(4)
def logout_action(self): logging.info("============== 退出登录 ===============") logging.info("点击设置图标") self.click(*self.MIME_SETTINGS) logging.info("点击退出登录按钮") self.click(*self.LOGOUT_BTN) logging.info("点击确认") self.click(*self.SURE_BTN) logging.info("退出登录结束")
def login_action(self, username, password): logging.info("登录操作") logging.info('用户名: {}'.format(username)) self.type(*self.USERNAME_IPT, text=username) logging.info('密码: {}'.format(password)) self.find_elements(*self.PASSWORD_IPT)[1].send_keys(password) # self.type(*self.PASSWORD_IPT, text=password) logging.info('点击登录按钮') self.click(*self.LOGIN_BTN) logging.info("登录结束")
def test_login_normal(self): """ 正常登录测试 smoke """ username, password = "******", "hanzhichao123" logging.info("正常登录测试,用户名:{}, 密码: {}".format(username, password)) user = LoginView(self.driver) user.login(username, password) self.assertTrue(user.is_login())
def search(self, keyword): self.load_search_page() logging.info("输入框输入'{}'".format(keyword)) # import os;os.system("adb shell ime set com.baidu.input/.ImeService") # import time; time.sleep(5) self.type(*self.SEARCH, text=keyword) # time.sleep(5) # self.driver.keyevent(66) # self.driver.keyevent(84) self.RECOMMEND[1] = self.RECOMMEND[1].format(keyword) # 有时输入了没触发搜索推荐 logging.info("点击搜索推荐'{}'".format(keyword)) self.click(*self.RECOMMEND)
def regist(self): """注册窗体页面""" try: name=entry_name.get() #获取文本输入框的get方法 pwd=entry_pwd.get() if name is '': insertnull=tk.Toplevel() insertnull.geometry("220x120") insertnull.title("注册失败") Label(insertnull,text = '用户名或密码为空').pack() Button(insertnull, text="返回",command=insertnull.destroy).pack() else: name2=db.query("select name from test_user where name =%s",(name)) # print(name2) if name2 is (): # name_res=name2[0][0] # print(name) # print(pwd) # print(name_res) if pwd is '': insertnull=tk.Toplevel() insertnull.geometry("220x120") insertnull.title("注册失败") Label(insertnull,text = '密码为空').pack() Button(insertnull, text="返回",command=insertnull.destroy).pack() print("密码为空") else: value=(name,pwd) db.commit("INSERT INTO test_user ( NAME, PASSWORD ) VALUES(%s,%s)",value) insertsucces=tk.Toplevel() insertsucces.geometry("220x120") insertsucces.title("注册成功") Label(insertsucces,text = '恭喜您注册成功').pack() print("恭喜您注册成功") handler = lambda: self.onCloseRegisterFrame(insertsucces) btn = tk.Button(insertsucces, text="登录", command=handler) btn.pack() Button(insertsucces, text="返回",command=insertsucces.destroy).pack() else: inserts=tk.Toplevel() inserts.geometry("220x120") inserts.title("注册失败") Label(inserts,text = '用户已注册').pack() Button(inserts, text="返回",command=inserts.destroy).pack() print("用户已注册") except Exception as e: insertnull=tk.Toplevel() insertnull.geometry("220x120") insertnull.title("注册失败") Label(insertnull,text = '用户名或密码为空').pack() Button(insertnull, text="返回",command=insertnull.destroy).pack() logging.info('用户名或密码为空{}'.format(e))
def get_repo(self): """ clone or pull the special repo. If the repo already exist in the "uploads" folder, it will pull the repo. If there is no repo in "uploads" folder, it will clone the repo. :return: """ if self.__check_exist(): logging.info('repo already exist. Try to pull the repo') return self.pull() else: return self.clone()
def login(self, username, password, project='测试项目'): self.driver.get("https://sensorsdata.secoo.com") # js = 'document.querySelector(".selector>select").removeAttribute("style")' # self.driver.execute_script(js) # project = Select(self.driver.find_element_by_css_selector(".selector>select")) # project.select_by_visible_text("测试项目") logging.info("[神策]登录({})".format(project)) self.driver.find_element_by_id("userName").send_keys(username) self.driver.find_element_by_id("password").send_keys(password) self.driver.find_element_by_id('rememberme').click() self.driver.find_element_by_id("submit").click()
def log(self, level, message, test=True): if test: self.data.append('[{0}] {1}'.format(level.upper(), message)) if level == 'critical': logging.critical(message) elif level == 'warning': logging.warning(message) elif level == 'info': logging.info(message) elif level == 'debug': logging.debug(message) elif level == 'error': logging.error(message)
def insert_vulnerabilities(self): """ Write a new vulnerability to the database :return: """ vul = CobraResults(self.task_id, self.project_id, self.rule_id, self.file_path, self.line_number, self.code_content, self.repair_code, self.status) db.session.add(vul) db.session.commit() logging.info("insert new vulnerabilities Method: {0} VID: {1}".format( self.method, vul.id)) self.push_third_party_vulnerabilities(vul.id)
def update_user(self): """修改个人信息""" user_name=username.get() user_password1=user_password.get() res1=sex.get() res2=age.get() res3=phone.get() res4=pwd3.get() # print(res1) # print(res2) # print(res3) # print(res) try: if user_name is '' or user_password1 is '': update_user_fial=tk.Toplevel() update_user_fial.geometry("220x120") update_user_fial.title("错误提示") Label(update_user_fial,text = '用户名或密码为空').pack() Button(update_user_fial, text="返回",command=update_user_fial.destroy).pack() else: name4=db.query("select name from test_user where name =%s",(user_name)) if name4 is (): update_user_fial1=tk.Toplevel() update_user_fial1.geometry("220x120") update_user_fial1.title("错误提示") Label(update_user_fial1,text = '用户名不存在').pack() Button(update_user_fial1, text="返回",command=update_user_fial1.destroy).pack() else: password3=db.query("select password from test_user where name =%s",(user_name)) res=password3[0][0] if user_password1==res: db.commit("UPDATE test_user SET sex=%s,age=%s,phone=%s,passwords=%s WHERE name=%s",(res1,res2,res3,res4,user_name)) update_user_success=tk.Toplevel() update_user_success.geometry("220x120") update_user_success.title("修改个人信息") Label(update_user_success,text = '修改个人信息成功').pack() Button(update_user_success, text="返回",command=update_user_success.destroy).pack() print("修改个人信息成功") else: update_user_fial4=tk.Toplevel() update_user_fial4.geometry("220x120") update_user_fial4.title("错误提示") Label(update_user_fial4,text = '密码错误').pack() Button(update_user_fial4, text="返回",command=update_user_fial4.destroy).pack() except Exception as e: updatefial=tk.Toplevel() updatefial.geometry("220x120") updatefial.title("错误提示") Label(updatefial,text = '用户名为空').pack() Button(updatefial, text="返回",command=updatefial.destroy).pack() logging.info('用户名为空{}'.format(e))
def pull(self): """Pull a repo from repo_address and repo_directory""" logging.info('Start Pull Repo...') if not self.__check_exist(): return False, 'No local repo exist. Please clone first.' # change work directory to the repo repo_dir = self.repo_directory logging.debug('cd directory: {0}'.format(repo_dir)) os.chdir(repo_dir) cmd = 'git pull origin master' p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) (pull_out, pull_err) = p.communicate() logging.info(pull_out) logging.info(pull_err) self.parse_err(pull_err) pull_err = pull_err.replace('{0}:{1}'.format(self.repo_username, self.repo_password), '') # change work directory back. os.chdir(repo_dir) if 'Updating' in pull_out or 'up-to-date' in pull_out: logging.info('Pull done.') return True, None else: return False, pull_err
def get_data(self): logging.info("[神策]检测数据") wait = WebDriverWait(self.driver, 60) try: data = wait.until( EC.presence_of_element_located( (By.XPATH, '//div[@id="imported-content-container"]' '/div[@class="content-item"]' '/div[@class="content-item-json"]' '/div[@class="data-json"]'))) except TimeoutException: return None else: return json.loads(data.text)
def block_code(self, block_position): """ 获取搜索区块代码 :param block_position: 0:up 上 1:down 下 2:location_line 当前行 :return: """ if block_position == 2: if self.line is None or self.line == 0: logging.critical("行号异常: {0}".format(self.line)) return False line_rule = '{0}p'.format(self.line) code = File(self.file_path).lines(line_rule) code = code.strip() return code else: block_start = 1 block_end = 0 functions = self.functions() if functions: for function_name, function_value in functions.items(): if int(function_value['start']) < int(self.line) < int( function_value['end']): in_this_function = '<---- {0}'.format(self.line) if block_position == 0: block_start = function_value['start'] block_end = int(self.line) - 1 elif block_position == 1: block_start = int(self.line) block_end = int(function_value['end']) - 1 logging.debug("触发行所在函数: {0} ({1} - {2}) {3}".format( function_name, function_value['start'], function_value['end'], in_this_function)) else: # 没有functions时,以触发行来分割整个文件 if block_position == 0: block_start = 1 block_end = int(self.line) - 1 elif block_position == 1: block_start = int(self.line) + 1 block_end = sum(1 for l in open(self.file_path)) logging.debug("没有找到任何方法,将以整个文件分割.") # get param block code line_rule = "{0},{1}p".format(block_start, block_end) code = File(self.file_path).lines(line_rule) logging.info('取出代码: {0} - {1}p'.format(block_start, block_end)) return code
def all(self): projects = CobraProjects.query.with_entities(CobraProjects.repository).filter(CobraProjects.status == CobraProjects.get_status('on')).all() for project in projects: payload = json.dumps({ "key": self.key, "target": project.repository, "branch": self.branch }) try: response = requests.post(self.api.format('add'), data=payload, headers=self.headers) response_json = response.json() logging.info(project.repository, response_json) except (requests.ConnectionError, requests.HTTPError) as e: logging.critical("API Add failed: {0}".format(e))
def clone(self): """Clone a repo from repo_address :return: True - clone success, False - clone error. """ logging.info('Start Clone Repo...') if self.__check_exist(): logging.info('Repo Already Exist. Stop Clone.') logging.debug('Directory exist, pull...') return self.pull() # call(['rm', '-rf', self.repo_directory]) # if no username or password provide, it may be a public repo. if self.repo_username is None or self.repo_password is None: # public repo clone_address = self.repo_address else: # private repo clone_address = self.repo_address.split('://')[0] + '://' + quote(self.repo_username) + ':' + \ self.repo_password + '@' + self.repo_address.split('://')[1] # clone repo with username and password # "http[s]://username:[email protected]/username/reponame" # !!! if add password in the url, .git/config will log your url with password cmd = 'git clone ' + clone_address + ' "' + self.repo_directory + '" -b master' p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) (clone_out, clone_err) = p.communicate() logging.info(clone_out) logging.info(clone_err) if 'not found' in clone_err or 'Not found' in clone_err: return False, 'repo doesn\'t exist.' elif 'already exists' in clone_err: return False, 'repo has already cloned.' elif 'Authentication failed' in clone_err: return False, 'Authentication failed.' clone_err = clone_err.replace( '{0}:{1}'.format(self.repo_username, self.repo_password), '') logging.info('clone done. Switching to branch ' + self.repo_branch) # check out to special branch if self.checkout(self.repo_branch): return True, None else: return False, clone_err
def compress(self): dc = decompress.Decompress(self.target) ret, result_d = dc.decompress() if ret is False: return 1002, result_d else: directory = result_d logging.info("Scan directory: {0}".format(directory)) current_time = time.strftime('%Y-%m-%d %X', time.localtime()) p = CobraProjects.query.filter_by(repository=directory).first() # detection framework for project framework, language = detection.Detection(directory).framework() if framework != '' or language != '': project_framework = '{0} ({1})'.format(framework, language) else: project_framework = '' if not p: # insert into project table. repo_name = directory.split('/')[-1] project = CobraProjects(directory, '', repo_name, 'Upload', project_framework, '', '', 1, current_time) db.session.add(project) db.session.commit() project_id = project.id else: project_id = p.id # update project's framework p.framework = project_framework db.session.add(p) task = CobraTaskInfo(directory, '', 3, '', '', 0, 0, 0, 1, 0, 0, current_time, current_time) db.session.add(task) db.session.commit() cobra_path = os.path.join(config.Config().project_directory, 'cobra.py') if os.path.isfile(cobra_path) is not True: return 1004, 'Cobra Not Found' # 扫描漏洞 subprocess.Popen(['python', cobra_path, "scan", "-p", str(project_id), "-i", str(task.id), "-t", directory]) # 统计代码行数 subprocess.Popen(['python', cobra_path, "statistic", "-i", str(task.id), "-t", directory]) # 检测漏洞修复状况 subprocess.Popen(['python', cobra_path, "repair", "-p", str(project_id)]) result = dict() result['scan_id'] = task.id result['project_id'] = project_id result['msg'] = u'success' return 1001, result
def get_real_directory(self): """ get real directory /path/project-v1.2/project-v1.2 -> /path/project-v1.2/ :param: :return: """ directory = os.path.join(self.upload_directory, self.dir_name) file_count = 0 directory_path = None for filename in os.listdir(directory): directory_path = os.path.join(directory, filename) file_count += 1 logging.info("Decompress path count: {0}, directory path: {1}".format(file_count, directory_path)) if file_count == 1 and os.path.isdir(directory_path): return directory_path else: return directory
def __init__(self, repo_address, branch='master', username=None, password=None): # get upload directory self.upload_directory = os.path.join(config.Config('upload', 'directory').value, 'versions') if os.path.isdir(self.upload_directory) is False: os.makedirs(self.upload_directory) self.repo_address = repo_address self.repo_username = username self.repo_password = password self.repo_branch = branch repo_user = self.repo_address.split('/')[-2] repo_name = self.repo_address.split('/')[-1].replace('.git', '') self.repo_author = repo_user self.repo_name = repo_name self.repo_directory = os.path.join(os.path.join(self.upload_directory, repo_user), repo_name) logging.info('Git class init.')
def run(self, is_all=None, pid=None): if bool(is_all) is True: message = '[START] Pull all projects code' print(message) logging.info(message) projects = CobraProjects.query.with_entities(CobraProjects.repository).filter(CobraProjects.status == CobraProjects.get_status('on')).all() for project in projects: if '.git' not in project.repository: continue code, msg, gg = scan.Scan(project.repository).pull_code() message = 'Pull code: {msg} {directory}'.format(msg=msg, directory=gg.repo_directory) if code == 1001: logging.info(message) else: logging.warning(message) print(message) message = '[END] Scan all projects' print(message) logging.info(message) elif pid is not None: project = CobraProjects.query.filter_by(id=pid).first() if project is None: message = 'Project not found' print(message) logging.critical(message) else: if '.git' not in project.repository: message = 'Not git repository' print(message) logging.info(message) code, msg, gg = scan.Scan(project.repository).pull_code() message = 'Pull code: {msg} {directory}'.format(msg=msg, directory=gg.repo_directory) if code == 1001: logging.info(message) else: logging.warning(message) print(message) else: message = 'Please set --target param' print(message) logging.critical(message) sys.exit()
def diff(self, new_version, old_version, raw_output=False): """ Diff between two version, in SHA hex. :param new_version: the new version in SHA hex. :param old_version: the old version in SHA hex. :param raw_output: True-return raw git diff result, False-return parsed result, only add content :return: the diff result in str, raw or formatted. """ if not self.__check_exist(): logging.info('No local repo exist. Please clone it first.') return False # change the work directory to the repo. current_dir = os.getcwd() + os.sep repo_dir = current_dir + self.repo_directory os.chdir(repo_dir) cmd = 'git diff ' + old_version + ' ' + new_version p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) (diff_out, diff_err) = p.communicate() logging.info(diff_out) # change the work directory back. os.chdir(current_dir) logging.info('diff done.') if raw_output: return diff_out else: return self.__parse_diff_result(diff_out)
def clone(self): """Clone a repo from repo_address :return: True - clone success, False - clone error. """ logging.info('Start Clone Repo...') if self.__check_exist(): logging.info('Repo Already Exist. Start Pull.') return self.pull() # call(['rm', '-rf', self.repo_directory]) # if no username or password provide, it may be a public repo. if self.repo_username is None or self.repo_password is None: # public repo clone_address = self.repo_address else: # private repo clone_address = self.repo_address.split('://')[0] + '://' + quote(self.repo_username) + ':' + \ self.repo_password + '@' + self.repo_address.split('://')[1] # clone repo with username and password # "http[s]://username:[email protected]/username/reponame" # !!! if add password in the url, .git/config will log your url with password cmd = 'git clone ' + clone_address + ' "' + self.repo_directory + '" -b master' p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) (clone_out, clone_err) = p.communicate() logging.info(clone_out) logging.info(clone_err) self.parse_err(clone_err) clone_err = clone_err.replace('{0}:{1}'.format(self.repo_username, self.repo_password), '') logging.info('clone done. Switching to branch ' + self.repo_branch) # check out to special branch if self.checkout(self.repo_branch): return True, None else: return False, clone_err
def repair(self): """ Scan vulnerabilities is repair :flow: - exist file [add] - test file - whitelist file - special file - annotation - rule :return: (Status, Result) """ self.method = 1 # Full path self.file_path = self.project_directory + self.file_path """ When the targeting rule is empty or the line number is 0, it means that this type of language (all suffixes in that language) is counted as a vulnerability Their repair method is only one: delete the file """ if self.rule_location == '' or self.line_number == 0: logging.info("Find special files: RID{0}".format(self.rule_id)) # Check if the file exists if os.path.isfile(self.file_path) is False: # If the file is not found, the vulnerability state is fixed logging.info("Deleted file repair is complete {0}".format(self.file_path)) self.status = self.status_fixed self.repair_code = self.repair_code_not_exist_file self.process_vulnerabilities() return else: return # Not exist file if os.path.isfile(self.file_path) is False: self.status = self.status_fixed self.repair_code = self.repair_code_not_exist_file self.process_vulnerabilities() return # Test file if self.is_test_file(): self.status = self.status_fixed self.repair_code = self.repair_code_test_file self.process_vulnerabilities() return """ Cobra Skip @cobra const `@[cC][oO][bB][rR][aA]\s*[cC][oO][nN][sS][tT]` """ file_content = File(self.file_path).read_file() ret_regex_const = re.findall(r'@[cC][oO][bB][rR][aA]\s*[cC][oO][nN][sS][tT]', file_content) if len(ret_regex_const) > 0: self.status = self.status_fixed self.repair_code = self.repair_code_const_file self.process_vulnerabilities() return """ @cobra third-party `@[cC][oO][bB][rR][aA]\s*[tT][hH][iI][rR][dD]-[pP][aA][rR][tT][yY]` """ ret_regex_third_party = re.findall(r'@[cC][oO][bB][rR][aA]\s*[tT][hH][iI][rR][dD]-[pP][aA][rR][tT][yY]', file_content) if len(ret_regex_third_party) > 0: self.status = self.status_fixed self.repair_code = self.repair_code_third_party self.process_vulnerabilities() return # Remove the trigger code (actual file) trigger_code = File(self.file_path).lines("{0}p".format(self.line_number)) if trigger_code is False: logging.critical("Failed to fetch the trigger code {0}".format(self.code_content)) self.status = self.status_fixed self.repair_code = self.repair_code_empty_code self.process_vulnerabilities() return self.code_content = trigger_code # Whitelist if self.is_white_list(): self.status = self.status_fixed self.repair_code = self.repair_code_whitelist self.process_vulnerabilities() logging.info("In white list {0}".format(self.file_path)) return # Special file if self.is_special_file(): self.status = self.status_fixed self.repair_code = self.repair_code_special_file self.process_vulnerabilities() logging.info("Special File: {0}".format(self.file_path)) return # Annotation if self.is_annotation(): self.status = self.status_fixed self.repair_code = self.repair_code_annotation self.process_vulnerabilities() logging.info("In Annotation {0}".format(self.code_content)) return # Modify ret_regex = re.findall(self.rule_location, trigger_code.strip()) if len(ret_regex) == 0: self.status = self.status_fixed self.repair_code = self.repair_code_modify self.process_vulnerabilities() return # Fixed if self.is_can_parse() and self.rule_repair.strip() != '': try: parse_instance = parse.Parse(self.rule_location, self.file_path, self.line_number, self.code_content) if parse_instance.is_repair(self.rule_repair, self.block_repair): logging.info("Static: repaired") # Fixed self.status = self.status_fixed self.repair_code = self.repair_code_fixed self.process_vulnerabilities() return else: logging.critical("[repair] not fixed") return except: logging.info(traceback.print_exc()) return
def reports(vid, start_time, end_time): projects = CobraProjects.query.order_by(CobraProjects.id.asc()).all() rank = [] count_project_not_fixed = 0 count_project_fixed = 0 count_vulnerability_not_fixed = 0 count_vulnerability_fixed = 0 special_rules_ids = [] if vid is 0: vulnerability_fixed_week = CobraResults.query.with_entities(CobraResults.id).filter(CobraResults.updated_at > '2016-11-28 00:00:00', CobraResults.updated_at < '2016-11-04 23:59:59', CobraResults.status == 2).count() vulnerability_not_fixed_week = CobraResults.query.with_entities(CobraResults.id).filter(CobraResults.updated_at > '2016-11-28 00:00:00', CobraResults.updated_at < '2016-11-04 23:59:59', CobraResults.status < 2).count() else: rules = CobraRules.query.with_entities(CobraRules.id).filter(CobraRules.vul_id == vid).all() for rule in rules: special_rules_ids.append(rule.id) vulnerability_fixed_week = CobraResults.query.filter(CobraResults.rule_id.in_(special_rules_ids), CobraResults.created_at > '2016-11-28 00:00:00', CobraResults.created_at < '2016-11-04 23:59:59', CobraResults.status == 2).count() vulnerability_not_fixed_week = CobraResults.query.with_entities(CobraResults.id).filter(CobraResults.updated_at > '2016-11-28 00:00:00', CobraResults.updated_at < '2016-11-04 23:59:59', CobraResults.status < 2).count() filter_group = (CobraResults.created_at > '{0} 00:00:00'.format(start_time), CobraResults.created_at < '{0} 23:59:59'.format(end_time),) for project in projects: if vid is 0: filter_group_total_base = (CobraResults.project_id == project.id,) if start_time == '0' and end_time == '0': filter_group_total = filter_group_total_base + () count_total = CobraResults.query.filter(*filter_group_total).count() else: filter_group_total = filter_group + filter_group_total_base + () count_total = CobraResults.query.filter(*filter_group_total).count() else: filter_group_total_base = (CobraResults.project_id == project.id, CobraResults.rule_id.in_(special_rules_ids),) if start_time == '0' and end_time == '0': filter_group_total = filter_group_total_base + () count_total = CobraResults.query.filter(*filter_group_total).count() else: filter_group_total = filter_group + filter_group_total_base + () count_total = CobraResults.query.filter(*filter_group_total).count() # detect project Cobra configuration file if project.repository[0] == '/': project_directory = project.repository else: project_directory = Git(project.repository).repo_directory cobra_properties = config.properties(os.path.join(project_directory, 'cobra')) need_scan = True if 'scan' in cobra_properties: need_scan = common.to_bool(cobra_properties['scan']) if need_scan: if vid is 0: filter_group_fixed_base = (CobraResults.project_id == project.id, CobraResults.status == 2,) if start_time == '0' and end_time == '0': filter_group_fixed = filter_group_fixed_base + () count_fixed = CobraResults.query.filter(*filter_group_fixed).count() else: filter_group_fixed = filter_group + filter_group_fixed_base + () count_fixed = CobraResults.query.filter(*filter_group_fixed).count() else: filter_group_fixed_base = (CobraResults.project_id == project.id, CobraResults.status == 2, CobraResults.rule_id.in_(special_rules_ids)) if start_time == '0' and end_time == '0': filter_group_fixed = filter_group_fixed_base + () count_fixed = CobraResults.query.filter(*filter_group_fixed).count() else: filter_group_fixed = filter_group + filter_group_fixed_base + () count_fixed = CobraResults.query.filter(*filter_group_fixed).count() if project.status == 1: count_not_fixed = count_total - count_fixed remark = '' else: count_fixed = count_total count_not_fixed = 0 remark = 'deleted' else: count_fixed = count_total count_not_fixed = 0 remark = 'offline' # update project status if project.status == CobraProjects.get_status('on'): project.status = CobraProjects.get_status('off') db.session.add(project) db.session.commit() logging.info('Update project status (./cobra) {project}'.format(project=project.repository)) if count_total != 0: if need_scan: if project.status == 1: if count_not_fixed == 0: count_project_fixed += 1 count_vulnerability_fixed += count_fixed ret_whole = 'fixed' else: count_project_not_fixed += 1 count_vulnerability_fixed += count_fixed count_vulnerability_not_fixed += count_not_fixed ret_whole = 'not_fixed' else: # deleted project count_project_fixed += 1 count_vulnerability_fixed += count_fixed ret_whole = 'fixed' else: count_project_fixed += 1 count_vulnerability_fixed += count_fixed ret_whole = 'fixed' report = 'http://' + config.Config('cobra', 'domain').value + '/report/' + str(project.id) s = { 'name': project.name, 'id': project.id, 'not_fixed': count_not_fixed, 'fixed': count_fixed, 'total': count_total, 'remark': remark, 'author': project.author, 'report': report, 'class': ret_whole } rank.append(s) rank = sorted(rank, key=lambda x: x['not_fixed'], reverse=True) vulnerabilities_types = CobraVuls.query.all() if start_time == '0': start_time = '' if end_time == '0': end_time = '' data = { 'rank': rank, 'vulnerabilities_types': vulnerabilities_types, 'vid': vid, 'count': { 'vulnerability': { 'not_fixed': count_vulnerability_not_fixed, 'fixed': count_vulnerability_fixed, 'total': count_vulnerability_not_fixed + count_vulnerability_fixed }, 'project': { 'not_fixed': count_project_not_fixed, 'fixed': count_project_fixed, 'total': count_project_not_fixed + count_project_fixed }, 'week': { 'fixed': "{0}({1})".format(vulnerability_fixed_week, common.percent(vulnerability_fixed_week, count_vulnerability_fixed)), 'not_fixed': "{0}({1})".format(vulnerability_not_fixed_week, common.percent(vulnerability_not_fixed_week, count_vulnerability_not_fixed)) } }, 'filter': { 'start': start_time, 'end': end_time } } return render_template("backend/report/report.html", data=data)