def case_test_task_timing_executor(project_id, case_testplan_uid): """ 定时任务 执行case task :param project_id: 项目的project ID :param case_testplan_uid: case testplan的 UUID :return: """ case_test_plan = CaseTestPlanModel.objects.filter( project_id=project_id, plan_id=case_testplan_uid).first() if not case_test_plan: logger.error("case testplan uid: {} not found") return False case_paths = case_test_plan.case_paths case_task_uid = "CASETASKTIMED_" + str(IdWorker(0, 1).get_id()) case_test_plan_task = CaseTestPlanTaskModel.objects.create( test_plan_uid=case_testplan_uid, case_task_uid=case_task_uid, state=CaseTestPlanTaskState.WAITING, case_job_number=len(case_paths), finish_num=0) CaseRunner.distributor(case_test_plan_task) # 根据是否并行执行case选择不用的触发器 if case_test_plan.parallel: '''并行执行''' case_jobs_id = CaseJobModel.objects.filter( case_task_id=case_test_plan_task.id).values_list('id', flat=True) for case_job_id in case_jobs_id: case_test_job_executor.delay(case_job_id, project_id, case_test_plan.plan_id, case_test_plan_task.id) else: '''串行执行''' case_test_task_executor(case_test_plan_task.id)
def assert_delimiter(key_str, response): """ 分隔符处理 """ hierarchy = key_str.split('.')[1:] try: if hierarchy[0] == 'status_code': result = response.status_code return result result = json.loads(response.text) for tier in hierarchy: try: tier = int(tier) except ValueError: tier = tier if isinstance(tier, int): try: result = result[tier] except IndexError as es: logger.error(es) return "EXCEPTION" else: result = result.get(tier, dict()) return result except Exception as es: logger.error(str(es)) return "EXCEPTION"
def get(self, request, *args, **kwargs): """ 【获取case信息】 """ gitlab_url = request.query_params.dict().get('gitlab_url') project_name = request.query_params.dict().get('project_name') branch_name = request.query_params.dict().get('branch_name', 'master') case_name = request.query_params.dict().get('label') case_path = request.query_params.dict().get('path') if case_path: # 有指定文件 表示要获取case详情 try: with open(os.path.join(settings.BASE_DIR, 'case_house', case_path), 'r', encoding='utf-8') as f: case_data = f.read() return HttpResponse(case_data, content_type='text/plain') except FileNotFoundError as es: logger.error(str(es)) return JsonResponse({"error": "case:{}不存在".format(case_name)}) else: # 没有指定到case路径 则返回case目录树 path_tree_instance = PathTree(branch_name) gitlab_path = gitlab_url.replace(':', '-').replace('.', '-').replace( '/', '') tree = path_tree_instance.path_tree( os.path.join(settings.BASE_DIR, 'case_house', gitlab_path, branch_name, project_name)) refine_tree = path_tree_instance.empty_json_data(tree) return JsonResponse({ "branch": branch_name, "case_tree": [refine_tree] })
def validate(self, attrs): for item in attrs: if item in ['headers', 'formData', 'urlencoded', 'raw']: try: json.loads(attrs[item]) except Exception as es: logger.error(es) raise serializers.ValidationError( '字段{}不是json格式'.format(item)) return attrs
def create(self, request, *args, **kwargs): serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) self.perform_create(serializer) if request.data.get('timer_enable') and request.data.get('crontab'): crontab = request.data.get('crontab').replace('?', '*') crontab = crontab.split(' ') if len(crontab) == 7: # 包含秒 包含年 crontab = crontab[1:-1] elif len(crontab) == 6: # 包含秒 不包含年 crontab = crontab[1:] crontab_kwargs = { 'minute': crontab[0], 'hour': crontab[1], 'day_of_week': crontab[4], 'day_of_month': crontab[2], 'month_of_year': crontab[3] } with transaction.atomic(): save_id = transaction.savepoint() try: schedule, _ = CrontabSchedule.objects.get_or_create( minute=crontab_kwargs.get('minute', '*'), hour=crontab_kwargs.get('hour', '*'), day_of_week=crontab_kwargs.get('day_of_week', '*'), day_of_month=crontab_kwargs.get('day_of_month', '*'), month_of_year=crontab_kwargs.get('month_of_year', '*'), timezone=pytz.timezone(TIME_ZONE)) dt = datetime.now().strftime('%Y%m%d%H%M%S') _periodic_task = PeriodicTask.objects.create( name=dt + '-' + 'case测试计划定时任务' + '-' + serializer.data.get('plan_id'), task='case_test_task_timing_executor', args=json.dumps([ serializer.data.get('project_id'), serializer.data.get('plan_id') ]), enabled=True, crontab=schedule) logger.info( "timing case testplan:{} create success!".format( serializer.data.get('plan_id'))) except Exception as e: transaction.savepoint_rollback(save_id) logger.error( 'timing case testplan:{} create failed!,error:{}'. format(serializer.data.get('plan_id'), e)) headers = self.get_success_headers(serializer.data) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
def isRegular(expression): """ 判断是否为正则表达式 """ if not isinstance(expression, str): return False try: term = re.compile(expression) except Exception as es: logger.error(es) return False else: return term
def case_test_job_executor(case_job_id, project_id, test_plan_uid, task_id): """ 【case job执行处理器】 :param case_job_id: case job的id project_id: 项目id test_plan_uid: 测试计划uid task_id: CaseTestPlanTaskModel模型的id主键 """ # 根据case testPlan task id获取模型并更新状态为RUNNING CaseTestPlanTaskModel.objects.filter(id=task_id).update( state=CaseTestPlanTaskState.RUNNING) case_job = CaseJobModel.objects.get(id=case_job_id) # 执行case job(运行case并生成报告) result = CaseRunner.executor(case_job=case_job, project_id=project_id, test_plan_uid=test_plan_uid, task_id=task_id) # 因为使用celery进行异步执行case,会出现在同一时刻多个case同时完成时一起修改finish num导致数据出错的问题,因此使用分布式锁保证数据一致性 # 用testplanid与taskid组成redis锁的key,可以唯一定位一个case testplan lock_key = test_plan_uid + str(task_id) # 从redis连接池获取一个连接实例 r = redis.Redis(connection_pool=RedisPoll().instance) while True: if r.setnx(lock_key, 1): # 只有在key不存在的情况下才能设置成功 条件成立 try: # 修改已完成数 case_task = CaseTestPlanTaskModel.objects.get(id=task_id) case_task.finish_num += 1 case_task.save() if case_task.finish_num == case_task.case_job_number: # 已完成数等于case总数 那整个case test plan全部完成 case_task.state = CaseTestPlanTaskState.FINISH case_task.save() used_time = case_task.update_date - case_task.create_date case_task.used_time = used_time.total_seconds() case_task.save() except Exception as es: logger.error( "Update CaseTestPlanTask finish_num Fail, es:{}, testplan id:{}" .format(es, task_id)) finally: # 为了防止造成死锁 无论成功与否都需要释放锁 r.delete(lock_key) break else: # 已经被其他celery任务上锁 等待0.5s后重试 time.sleep(0.2) continue return result
def executor(cls, case_job, project_id, test_plan_uid, task_id): """ 执行器 :param case_job: :param project_id: :param test_plan_uid: :param task_id: :return: """ case_job.state = CaseJobState.RUNNING case_job.save() report_name = case_job.case_path.split('/')[-1].split('.')[0] + '.html' result_log_name = case_job.case_path.split('/')[-1].split('.')[0] + '.log' xml_name = case_job.case_path.split('/')[-1].split('.')[0] + '.xml' report_path = os.path.join(MEDIA_ROOT, 'html-report', str(project_id), test_plan_uid, str(task_id)) try: if not os.path.exists(report_path): os.makedirs(report_path) except FileExistsError: pass try: p = subprocess.Popen( 'pytest {} -vv -s --html={} --self-contained-html --result-log={} --junit-xml={}'.format( os.path.join(settings.BASE_DIR, 'case_house', case_job.case_path), os.path.join(report_path, report_name), os.path.join(report_path, result_log_name), os.path.join(report_path, xml_name)), shell=True, stdout=subprocess.PIPE) out = p.stdout read_data = out.read().decode("utf-8", "ignore") # case_job.log = read_data case_job.report_path = '/media/html-report/{}/{}/{}/{}'.format(project_id, test_plan_uid, task_id, report_name) case_result = read_data.split('\n')[-2] case_job.result = case_result.replace('=', '').strip(' ') case_job.state = CaseJobState.FINISH case_job.save() return True except Exception as es: logger.error("case job excepted:{}".format(es)) case_job.state = CaseJobState.FAILED case_job.save() return False
def executor(cls, case_job, project_id, test_plan_uid, task_id): """ 执行器 :param case_job: :param project_id: :param test_plan_uid: :param task_id: :return: """ default_venv = ConfigParser.get_config('python_venv', 'venv_path') case_job.state = CaseJobState.RUNNING case_job.save() case_path = case_job.case_path case_dir = '/'.join(case_path.split('/')[0:3]) case_venv = os.path.join(settings.BASE_DIR, 'case_house', case_dir, 'venv') cmd_path = os.path.join(settings.BASE_DIR, 'case_house', case_dir) if os.path.exists(case_venv): venv_path = case_venv else: venv_path = default_venv file_path, file_name = os.path.split(case_path) report_name = case_path.split('/')[-1].split('.')[0] + '.html' result_log_name = case_path.split('/')[-1].split('.')[0] + '.log' xml_name = case_path.split('/')[-1].split('.')[0] + '.xml' report_path = os.path.join(MEDIA_ROOT, 'html-report', str(project_id), test_plan_uid, str(task_id)) try: if not os.path.exists(report_path): os.makedirs(report_path) except FileExistsError: pass if 'test' in os.path.splitext(file_name)[0] and os.path.splitext(file_name)[1].startswith('.py'): # 是pytest 脚本 try: cmd = '{} {} -vv -s --html={} --self-contained-html --result-log={} --junit-xml={}'.format( os.path.join(venv_path, 'bin/pytest'), os.path.join(settings.BASE_DIR, 'case_house', case_path), os.path.join(report_path, report_name), os.path.join(report_path, result_log_name), os.path.join(report_path, xml_name)) logger.info(cmd) p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, cwd=cmd_path) out = p.stdout read_data = out.read().decode("utf-8", "ignore") case_job.log = read_data case_job.report_path = '/media/html-report/{}/{}/{}/{}'.format(project_id, test_plan_uid, task_id, report_name) case_result = read_data.split('\n')[-2] case_job.result = case_result.replace('=', '').strip(' ') case_job.state = CaseJobState.FINISH case_job.save() return True except Exception as es: logger.error("case job excepted:{}".format(es)) case_job.state = CaseJobState.FAILED case_job.save() return False elif os.path.splitext(file_name)[1] == '.py': # 普通python脚本 try: p = subprocess.Popen( '{} {}'.format( os.path.join(venv_path, 'bin/python'), os.path.join(settings.BASE_DIR, 'case_house', case_path)), shell=True, stdout=subprocess.PIPE, cwd=cmd_path) out = p.stdout read_data = out.read().decode("utf-8", "ignore") case_job.log = read_data case_job.state = CaseJobState.FINISH case_job.save() return True except Exception as es: logger.error("case job excepted:{}".format(es)) case_job.state = CaseJobState.FAILED case_job.save() return False elif os.path.splitext(file_name)[1] == '.yml': # HttpRunner脚本 try: case_task_id = case_job.case_task_id test_plan_uid = CaseTestPlanTaskModel.objects.get(id=case_task_id).test_plan_uid env_file = CaseTestPlanModel.objects.get(plan_id=test_plan_uid).env_file if env_file: p = subprocess.Popen( '{} {} --dot-env-path {} --report-file {}'.format( os.path.join(venv_path, 'bin/hrun'), os.path.join(settings.BASE_DIR, 'case_house', case_path), os.path.join(settings.BASE_DIR, 'case_house', env_file), os.path.join(report_path, report_name)), shell=True, stdout=subprocess.PIPE, cwd=cmd_path) else: p = subprocess.Popen('{} {} --report-file {}'.format( os.path.join(venv_path, 'bin/hrun'), os.path.join(settings.BASE_DIR, 'case_house', case_path), os.path.join(report_path, report_name)), shell=True, stdout=subprocess.PIPE, cwd=cmd_path) out = p.stdout read_data = out.read().decode("utf-8", "ignore") case_job.report_path = '/media/html-report/{}/{}/{}/{}'.format(project_id, test_plan_uid, task_id, report_name) case_job.log = read_data # case_job.result = case_result.replace('=', '').strip(' ') case_job.state = CaseJobState.FINISH case_job.save() return True except Exception as es: logger.error("case job excepted:{}".format(es)) case_job.state = CaseJobState.FAILED case_job.save() return False
def handler(self, interface_job): """ 测试计划处理 """ # 获取interface对象 if interface_job.interfaceType == InterFaceType.INSTANCE.value: interface = InterfaceModel.objects.get(id=interface_job.interface_id) else: interface = InterfaceCacheModel.objects.get(id=interface_job.interface_id) extracts_dict = get_all_extracts(interface_job.test_plan_id, interface_job.api_test_plan_task_id) project_id = ApiTestPlanModel.objects.filter(plan_id=interface_job.test_plan_id).first().project_id # 添加全局环境变量到变量池 global_env_variable = ProjectModel.objects.get(id=project_id).env_variable if global_env_variable: for k, v in global_env_variable.items(): extracts_dict[k] = v # 添加变量函数到变量池 extracts_dict['ts_10'] = ts_10() extracts_dict['ts_13'] = ts_13() for key, value in extracts_dict.items(): if '$%s' % key in interface.addr: interface.addr = interface.addr.replace('$%s' % key, str(value), 10) if '$%s' % key in interface.name: interface.name = interface.name.replace('$%s' % key, str(value), 10) if '$%s' % key in interface.desc: interface.desc = interface.desc.replace('$%s' % key, str(value), 10) if '$%s' % key in json.dumps(interface.headers): interface.headers = json.loads(json.dumps(interface.headers).replace('$%s' % key, str(value), 10)) if '$%s' % key in json.dumps(interface.params): interface.params = json.loads(json.dumps(interface.params).replace('$%s' % key, str(value), 10)) if '$%s' % key in json.dumps(interface.formData): interface.formData = json.loads(json.dumps(interface.formData).replace('$%s' % key, str(value), 10)) if '$%s' % key in json.dumps(interface.urlencoded): interface.urlencoded = json.loads(json.dumps(interface.urlencoded).replace('$%s' % key, str(value), 10)) if '$%s' % key in json.dumps(interface.raw): interface.raw = json.loads(json.dumps(interface.raw).replace('$%s' % key, str(value), 10)) if '$%s' % key in json.dumps(interface.asserts): interface.asserts = json.loads(json.dumps(interface.asserts).replace('$%s' % key, str(value), 10)) interface.save() # 将api job状态更新为RUNNING InterfaceJobModel.objects.filter(id=interface_job.id).update(state=ApiJobState.RUNNING) headers = interface.headers # 请求头 # 根据请求方式动态选择requests的请求方法 try: requests_fun = getattr(self.session, interface.get_request_mode_display().lower()) if interface.request_mode == "GET": data = interface.params response = requests_fun(url=interface.addr, headers=headers, params=data) else: if interface.formData: # form-data文件请求 data = interface.formData response = requests_fun(url=interface.addr, headers=headers, data=data) elif interface.urlencoded: # form 表单 data = interface.urlencoded response = requests_fun(url=interface.addr, headers=headers, data=data) elif interface.raw: # json请求 data = json.dumps(interface.raw).encode("utf-8") response = requests_fun(url=interface.addr, headers=headers, data=data) else: response = requests_fun(url=interface.addr, headers=headers) extracts_result = self.dispose_response(interface=interface, response=response) InterfaceJobModel.objects.filter(id=interface_job.id).update(extracts=extracts_result) except Exception as es: logger.error("interface job:{} Except: {}".format(interface_job.id, str(es))) InterfaceJobModel.objects.filter(id=interface_job.id).update(state=ApiJobState.FAILED)
def dispose_response(self, interface, response): """ 请求处理器 """ # 处理断言 for _assert in interface.asserts: if _assert['assertType'] == "regular": # 正则判断逻辑 if not isRegular(_assert['expressions']): logger.error( 'interface Id:{}, testPlan Id:{} expressions:{} is not regular'.format(interface.id, self.test_plan_id, _assert['expressions'])) update_api_job_fail(self.test_plan_id, interface.id, response) break else: pattern = isRegular(_assert['expressions']) re_result = assert_regular(pattern, response.text) logger.info("正则匹配结果:{}".format(re_result)) if not re_result: # 断言失败 update_api_job_fail(self.test_plan_id, interface.id, response) # 跟新interfaceJob状态失败 break else: try: calculate_fun = getattr(operation, _assert['calculate']) except AttributeError as es: logger.error("calculate rule {} is not exist!".format(_assert['calculate'])) update_api_job_fail(self.test_plan_id, interface.id, response) break if not calculate_fun(re_result, _assert['expect']): update_api_job_fail(self.test_plan_id, interface.id, response) # 跟新interfaceJob状态失败 break elif _assert['assertType'] == "delimiter": # 分隔符取值 delimiter_result = assert_delimiter(_assert['expressions'], response) logger.info("分隔符匹配结果: {}".format(delimiter_result)) if delimiter_result == 'EXCEPTION': logger.error( "delimiter error:{}, interfaceJobId: {}, test_plan Id:{}".format(_assert['expressions'], interface.id, self.test_plan_id)) update_api_job_fail(self.test_plan_id, interface.id, response) break elif delimiter_result == dict(): update_api_job_fail(self.test_plan_id, interface.id, response) break else: try: calculate_fun = getattr(operation, _assert['calculate']) except AttributeError as es: logger.error("calculate rule {} is not exist!".format(_assert['calculate'])) update_api_job_fail(self.test_plan_id, interface.id, response) break if not calculate_fun(delimiter_result, _assert['expect']): update_api_job_fail(self.test_plan_id, interface.id, response) break else: # 所有断言验证通过 update_api_job_success(self.test_plan_id, interface.id, response) # 处理提取规则 extracts_result = {} # 提取结果的集合 for extract in interface.extract: if extract['extractType'] == "regular": if not isRegular(extract['expressions']): logger.error( 'interface Id:{}, testPlan Id:{} expressions:{} is not regular'.format(interface.id, self.test_plan_id, extract['expressions'])) pattern = isRegular(extract['expressions']) re_result = assert_regular(pattern, response.text) if not re_result: # 没有匹配到结果 extracts_result[extract['variable_name']] = '' else: extracts_result[extract['variable_name']] = re_result elif extract['extractType'] == "delimiter": delimiter_result = assert_delimiter(extract['expressions'], response) if delimiter_result == 'EXCEPTION': logger.error( "delimiter error:{}, interfaceJobId: {}, test_plan Id:{}".format(extract['expressions'], interface.id, self.test_plan_id)) extracts_result[extract['variable_name']] = '' else: extracts_result[extract['variable_name']] = delimiter_result return extracts_result
async def receive(self, text_data): try: text_data_json = json.loads(text_data) print(text_data_json) except json.decoder.JSONDecodeError: await self.send(text_data=json.dumps({ "success": False, "error": "Illegal data type" })) return False while True: try: mode_type = text_data_json.get('mode_type') task_or_job = text_data_json.get('task_or_job') value = text_data_json.get('value') except AttributeError as es: logger.error(str(es)) send_msg = {"success": False, "error": str(es)} return send_msg try: if mode_type not in ['api', 'case'] or task_or_job not in [ 'task', 'job' ] or not value: error_data = 'Illegal parameter value: {}, {}'.format( mode_type, task_or_job) logger.error(error_data) send_msg = {"success": False, "error": error_data} return send_msg elif mode_type == "api": if task_or_job == "task": result = await ApiStatusManager().get_task_result(value ) msg = { "success": True, "mode": "task", "data": list(result) } elif task_or_job == "job": result = await ApiStatusManager().get_job_result(value) msg = { "success": True, "mode": "job", "data": list(result) } elif mode_type == "case": if task_or_job == "task": result = await CaseStatusManager().get_task_result( value) msg = { "success": True, "mode": "task", "data": list(result) } elif task_or_job == "job": result = await CaseStatusManager().get_job_result(value ) msg = { "success": True, "mode": "job", "data": list(result) } else: error_data = 'Illegal parameter value: {}'.format( mode_type) logger.error(error_data) msg = {"success": False, "error": error_data} except Exception as es: logger.error(str(es)) msg = {"success": False, "error": str(es)} await self.send(json.dumps(msg, cls=DateEncoder)) await asyncio.sleep(2)
def branch_pull(gitlab_info, project_id, branch_name): """ 录取gitlab指定分支代码 """ instance = GitlabAPI(gitlab_url=gitlab_info.get('gitlab_url'), private_token=gitlab_info.get('private_token')) project = instance.gl.projects.get(project_id) obj_tuple = GitCaseModel.objects.update_or_create( gitlab_url=gitlab_info.get('gitlab_url'), gitlab_project_name=project.name, gitlab_project_id=project.id, branch_name=branch_name) obj_tuple[0].status = BranchState.PULLING obj_tuple[0].save() cache_key = "private_token:" + gitlab_info.get( 'private_token') + "-" + "project_id:" + str( project_id) + "-" + "branch_name:" + branch_name cache.set(cache_key, BranchState.PULLING) try: info = project.repository_tree(ref=branch_name, all=True, recursive=True, as_list=True) file_list = [] root_path = os.path.join( BASE_DIR, 'case_house', gitlab_info.get('gitlab_url').replace(':', '-').replace( '.', '-').replace('/', ''), branch_name, project.name) if not os.path.isdir(root_path): os.makedirs(root_path) os.chdir(root_path) # 调用创建目录的函数并生成文件名列表 for info_dir in range(len(info)): if info[info_dir]['type'] == 'tree': dir_name = info[info_dir]['path'] create_dir(dir_name) else: file_name = info[info_dir]['path'] file_list.append(file_name) file_list_len = len(file_list) for index, info_file in enumerate(range(file_list_len)): # 开始下载 getf = project.files.get(file_path=file_list[info_file], ref=branch_name) content = getf.decode() with open(file_list[info_file], 'wb') as code: logger.info("\033[0;32;40m开始下载文件: \033[0m{0}".format( file_list[info_file])) code.write(content) finish_progress = float('%.2f' % ((index + 1) / file_list_len)) * 100 cache.set(cache_key + "progress", finish_progress, 3) branch_obj = GitCaseModel.objects.get( gitlab_url=gitlab_info.get('gitlab_url'), gitlab_project_id=project.id, gitlab_project_name=project.name, branch_name=branch_name, ) branch_obj.status = BranchState.DONE branch_obj.save() cache.set(cache_key, BranchState.DONE) return True except Exception as es: logger.error(str(es)) branch_obj = GitCaseModel.objects.get( gitlab_url=gitlab_info.get('gitlab_url'), gitlab_project_id=project.id, gitlab_project_name=project.name, branch_name=branch_name, ) branch_obj.status = BranchState.FAILED branch_obj.save() cache.set(cache_key, BranchState.FAILED) return False
def get(self, request, *args, **kwargs): """ 【获取指定case中的条目信息】 :return """ case_path = request.query_params.dict().get('path') if case_path: # 有指定文件 表示要获取case详情 try: relative_path_dir, script_name = os.path.split(case_path) if "test" in script_name.lower() and os.path.splitext( case_path)[1] == ".py": absolute_path = os.path.join(settings.BASE_DIR, 'case_house', case_path) p_dir, p_file = os.path.split(absolute_path) if platform.system().lower() == "linux": # Linux系统 p = subprocess.Popen( "pytest {} --collect-only -q | head -n -2".format( p_file), cwd=p_dir, shell=True, stdout=subprocess.PIPE) elif platform.system().lower() == "darwin": # Mac系统 p = subprocess.Popen( "pytest {} --collect-only -q | ghead -n -2".format( p_file), cwd=p_dir, shell=True, stdout=subprocess.PIPE) out = p.stdout read_data = out.read().decode("utf-8", "ignore") if "ERROR" in read_data: subCasesList = [] else: subCasesList = [{ "label": case, "filepath": os.path.join(relative_path_dir, case) } for case in read_data.split('\n')[:-1]] return JsonResponse({ "success": True, "subCaseList": subCasesList }) else: return JsonResponse({"success": True, "subCaseList": []}) except FileNotFoundError as es: logger.error(str(es)) return JsonResponse({ "success": False, "error": "case:{} not find".format(case_path) }) else: return Response(data={ "success": False, "error": "Lack of necessary parameters:case_path}" }, status=status.HTTP_404_NOT_FOUND)