class JobsMonitor(object): def __init__(self): self.setting = SettingModule() # 监控任务状态 @gen.coroutine def jobs_status(self): try: s_setting, total = yield self.setting.get_settings_list( s_type='job', status=0, pj_status=1, limit=None) for row in s_setting: if int(time.time()) > int(float(row.name)): yield self.setting.edit_setting(sid=row.id, status=1) except Exception as e: log.warning(e) # 执行定时任务 @gen.coroutine def run_jobs(self): try: s_setting, total = yield self.setting.get_settings_list( s_type='job', status=1, pj_status=1, limit=None) for row in s_setting: test = TestRunner(row.project_id, row.id) test.run_job() except Exception as e: log.warning(e)
def __init__(self): self.project = ProjectModule() self.setting = SettingModule() self.option_func = OptionsFunction() self.common_func = CommonFunction() self.thread_func = ThreadFunction() self.jenkins_server = jenkins.Jenkins(url=jenkins_url, username=jenkins_user, password=jenkins_password, timeout=5)
def __init__(self, pid=0, jid=0): self.default_headers = { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36', 'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4', 'Cookie': '' } self.headers = dict() self.setting = SettingModule() self.common_func = CommonFunction() self.option_func = OptionsFunction() self.pid = pid self.jid = jid
def prepare(self): self.user = UserModule() self.project = ProjectModule() self.setting = SettingModule() self.option = OptionModule() self.msg = MessagesModule() self.statistics = StatisticsModule() self.common_func = CommonFunction() self.option_func = OptionsFunction() self.thread_func = ThreadFunction()
def prepare(self): self.common_func = CommonFunction() self.user = UserModule() self.project = ProjectModule() self.setting = SettingModule() self.option = OptionModule() self.option_func = OptionsFunction() self.company = yield self.option_func.get_option_by_name(name='company') self.limit = yield self.option_func.get_option_by_name(name='page_limit') self.company = self.company or '开源接口测试平台' self.argv = dict(company=self.company)
def prepare(self): self.func = Function() self.user = UserModule() self.project = ProjectModule() self.setting = SettingModule() self.option = OptionModule() self.func_option = OptionsFunc() self.company = yield self.func_option.get_option_by_name( name='company') self.limit = yield self.func_option.get_option_by_name( name='page_limit') self.company = self.company if self.company else 'BST开源测试平台' self.argv = dict(company=self.company)
class JobsMonitor(object): def __init__(self): self.project = ProjectModule() self.setting = SettingModule() self.option_func = OptionsFunction() self.common_func = CommonFunction() self.thread_func = ThreadFunction() self.jenkins_server = jenkins.Jenkins(url=jenkins_url, username=jenkins_user, password=jenkins_password, timeout=5) # 监控任务状态 @gen.coroutine def jobs_status(self): try: job_list, total = yield self.setting.get_settings_list( s_type=['job', 'jobG', 'jobA'], status=[0, 2], pj_status=1) for job in job_list: desc = json.loads(job.value) if (desc.get('buildEnv') or desc.get('dayBuild')) and job.status == 0 and int( time.time()) > int(job.createTime.timestamp()): yield self.setting.edit_setting(sid=job.id, status=1) continue job_name = desc.get( 'jobName') if job.type != 'job' else jenkins_jacoco if job_name and job.status == 2: if job.type != 'job': last_build_number = self.jenkins_server.get_job_info( job_name).get('lastBuild').get('number') last_build = self.jenkins_server.get_build_info( job_name, last_build_number) status = None if last_build.get('building') else 0 else: status = None if (job.type != 'job' and last_build.get('queueId') == desc.get('queueId')) or (job_name == jenkins_jacoco): if (job.type != 'job' and status is None and desc['url'] == last_build.get('url')) or ( job_name == jenkins_jacoco): if desc.get('jacocoId'): jacoco_job = self.jenkins_server.get_job_info( jenkins_jacoco) first_build_number = jacoco_job.get( 'firstBuild').get('number') last_build_number = jacoco_job.get( 'lastBuild').get('number') for num in range(first_build_number, last_build_number + 1): try: jacoco = self.jenkins_server.get_build_info( jenkins_jacoco, num) if jacoco.get('queueId') != desc.get( 'jacocoId'): continue if not jacoco.get( 'building') and jacoco.get( 'result') == 'FAILURE': day_build_env = desc.get( 'buildEnv') or desc.get( 'dayBuild') apps = yield self.option_func.get_env_info( eid=day_build_env) stop_time = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() + 3600 * 24)) parameters = [ ('ACTION', 'Monitoring'), ('STOPTIME', stop_time), ('CLEAN', 'No'), ('JOB_ID', job.name) ] for app in [ a.get('details') for a in apps ]: for ap in app: if ap.get('title').lower( ).find('linux_app') != -1: parameters.append( ('ENV', ap.get('ip'))) if len(parameters) > 4: version_file = os.path.join( static_path, 'diffAPP', job.name) if not os.path.isdir( version_file): os.makedirs(version_file) version_file = os.path.join( version_file, 'jacoco_app_version.txt') if os.path.isfile( version_file): os.remove(version_file) for app in (desc.get('runApps') or []): parameters.append( ('APP', app)) jacoco_id = self.jenkins_server.build_job( jenkins_jacoco, parameters=parameters) desc['jacocoId'] = jacoco_id yield self.setting.edit_setting( sid=job.id, value=desc) except Exception as e: log.error(e) continue continue desc['url'] = last_build.get('url') cycle = desc.get('cycle') if cycle == 'hour': next_time = job.createTime + timedelta(hours=1) elif cycle == 'day': next_time = job.createTime + timedelta(days=1) elif cycle == 'week': next_time = job.createTime + timedelta(weeks=1) elif cycle == 'mouth': next_time = job.createTime + timedelta(days=30) elif cycle == 'year': next_time = job.createTime + timedelta(days=365) else: next_time = job.createTime if status == 0: status = 3 if status in [0, 3] and desc.get('jacocoId'): jacoco_job = self.jenkins_server.get_job_info( jenkins_jacoco) first_build_number = jacoco_job.get( 'firstBuild').get('number') last_build_number = jacoco_job.get( 'lastBuild').get('number') for num in range(first_build_number, last_build_number + 1): try: jacoco = self.jenkins_server.get_build_info( jenkins_jacoco, num) if jacoco.get('queueId') != desc.get( 'jacocoId'): continue self.jenkins_server.stop_build( jenkins_jacoco, num) except Exception as e: log.error(e) continue apps = yield self.option_func.get_env_info( eid=desc.get('buildEnv') or desc.get('dayBuild')) desc['buildEnv'] = '' parameters = [('ACTION', 'Reporting'), ('HIS_BUILD_ID', num), ('CLEAN', 'Yes'), ('JOB_ID', job.name)] for app in [a.get('details') for a in apps]: for ap in app: if ap.get('title').lower().find( 'linux_app') != -1: parameters.append( ('ENV', ap.get('ip'))) if len(parameters) > 4: for app in (desc.get('runApps') or []): parameters.append(('APP', app)) self.jenkins_server.build_job( jenkins_jacoco, parameters=parameters) yield self.setting.edit_setting( sid=job.id, value=desc, status=status, create_time=next_time.strftime('%Y-%m-%d %H:%M:%S') if status in [0, 3] else None) except Exception as e: log.warning(e) # 执行定时任务 @gen.coroutine def run_jobs(self): try: job_list, total = yield self.setting.get_settings_list( s_type=['jobG', 'jobA'], status=1, pj_status=1) for job in job_list: desc = json.loads(job.value) job_name = desc.get('jobName') day_build_env = desc.get('buildEnv') or desc.get('dayBuild') if not job_name or not day_build_env: continue data = munchify( dict(env=day_build_env, type='api' if job.type == 'jobA' else 'gui', exec=job_name)) yield self.option_func.get_hosts_info(data) yield self.option_func.get_mysql_jdbc(eid=day_build_env, data=data) yield self.option_func.get_cases_info(job, data) queue_id = self.jenkins_server.build_job( job_name, parameters=dict(JOB_ID=job.name)) if queue_id: apps = yield self.option_func.get_env_info( eid=day_build_env) stop_time = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() + 3600 * 24)) parameters = [('ACTION', 'Monitoring'), ('STOPTIME', stop_time), ('CLEAN', 'Yes'), ('JOB_ID', job.name)] for app in [a.get('details') for a in apps]: for ap in app: if ap.get('title').lower().find('linux_app') != -1: parameters.append(('ENV', ap.get('ip'))) if len(parameters) > 4: version_file = os.path.join(static_path, 'diffAPP', job.name) if not os.path.isdir(version_file): os.makedirs(version_file) version_file = os.path.join(version_file, 'jacoco_app_version.txt') if os.path.isfile(version_file): os.remove(version_file) for app in (desc.get('runApps') or []): parameters.append(('APP', app)) jacoco_id = self.jenkins_server.build_job( jenkins_jacoco, parameters=parameters) desc['jacocoId'] = jacoco_id desc['queueId'] = queue_id desc['url'] = self.jenkins_server.get_job_info( job_name).get('url') yield self.setting.edit_setting(sid=job.id, value=desc, status=2) except Exception as e: log.warning(e) # 申请关闭外网权限 @gen.coroutine def close_network(self): try: now_date = time.strftime('%Y-%m-%d') now_time = time.strftime('%H:%M:%S') if now_time > '08:30:00': server, total = yield self.setting.get_settings_list( s_type='env', status=1) ips = list() for svr in server: desc = json.loads(svr.value) if (svr.createTime.strftime('%Y-%m-%d') < now_date or (svr.createTime.strftime('%Y-%m-%d') == now_date and now_time > svr.createTime.strftime('%H:%M:%S')) ) and desc.get('network') == 'yes' and desc.get('ip'): ips.append(desc.get('ip').strip()) if set(ips): title = '测试环境申请关闭外网权限' mail_content = ''' <p>Hi 你好!</p> <p style="padding-left:30px;">测试人员已完成测试任务,申请关闭以下测试环境服务器外网权限。请帮忙处理一下,3ks~</p> <p style="padding-left:30px;">1、服务器</p> <p style="padding-left:60px;">{}</p> '''.format('</br>'.join(set(ips))) res, msg = yield self.common_func.send_email( subject=title, content=mail_content, to=net_mail_to, cc=net_mail_cc) if res: log.info(msg) for svr in server: desc = json.loads(svr.value) if (svr.createTime.strftime('%Y-%m-%d') < now_date or (svr.createTime.strftime('%Y-%m-%d') == now_date and now_time > svr.createTime.strftime('%H:%M:%S')) ) and desc.get( 'network') == 'yes' and desc.get('ip'): desc['network'] = 'no' yield self.setting.edit_setting(sid=svr.id, value=desc) else: log.warn(msg) except Exception as e: log.error(e) # 同步准生产数据库表 @gen.coroutine def sync_db_tables(self): dumps = dict() db_path = os.path.join(static_path, 'syncDB', 'lasTables') if not os.path.isdir(db_path): os.makedirs(db_path) flag_file = os.path.join(db_path, 'TAG') old_date = '' if os.path.isfile(flag_file): with open(flag_file, 'r') as fp: old_date = fp.read() now_date = time.strftime('%Y%m%d') if old_date != now_date: env_list, total = yield self.project.get_projects_list( p_type='env', status=1, search='准生产') for env in env_list: details, total = yield self.setting.get_settings_list( s_type='env', name=env.name) for detail in details: desc = json.loads(detail.value) if desc.get('type') != 'APPLICATION' and desc.get( 'title').upper().find('MYSQL') == -1: continue dumps[detail.id] = dict( ip=desc.get('ip').strip(), port=desc.get('port').strip(), user=desc.get('user').strip(), password=desc.get('password').strip(), dbs=desc.get('description').split(',')) break with open(flag_file, 'w') as fp: fp.write(time.strftime('%Y%m%d')) for key in dumps: for db in dumps[key]['dbs']: tmp_path = os.path.join(db_path, '{}_tmp.txt'.format(db)) shell_dump = '''cd {} /opt/lampp/bin/mysqldump -h{} -P{} -u{} -p{} -d {} > {}'''.format( root_160, dumps[key]['ip'], dumps[key]['port'], dumps[key]['user'], dumps[key]['password'], db, 'lasTables/{}_tmp.txt'.format(db)) res, msg = yield self.thread_func.exec_remote_shell( shell=shell_dump, host=host_160, port=port_160, username=user_160, password=password_160) if res and os.path.isfile(tmp_path): with open(tmp_path, 'r', encoding='utf8') as fp: lines = fp.readlines() tables = list() for line in lines: if line.find('DROP TABLE IF EXISTS') == -1: continue table = re.findall(r'`(\w+)`;', line) table and tables.append('{}\n'.format(table[0])) with open(os.path.join(db_path, '{}.txt'.format(db)), 'w') as fp: fp.writelines(tables) os.remove(tmp_path) else: log.info(msg) old_time = time.time() - 30 * 24 * 3600 old_path = os.path.join(static_path, 'syncDB', time.strftime('%Y%m%d', time.gmtime(old_time))) if os.path.isdir(old_path): shutil.rmtree(old_path)
def __init__(self): self.setting = SettingModule() self.option = OptionModule()
class OptionsFunction(object): def __init__(self): self.setting = SettingModule() self.option = OptionModule() # 通过名称获取系统配置值 @gen.coroutine def get_option_by_name(self, name): option = yield self.option.get_option(name=name) if option: return option.value else: return '' # 获取加解密参数 @gen.coroutine def get_crypt_info(self, pid, do='encrypt'): setting, total = yield self.setting.get_settings_list(pid=pid, s_type='crypt', limit=None) setting = setting[0] if len(setting) > 0 else None if setting: flag = False crypt = json.loads(setting.value) if do == 'encrypt': crypt = crypt['encrypt'] for info in encrypt: if info['name'] == crypt['name']: crypt['mode'] = info['mode'] crypt['function'] = info['function'] flag = True break elif do == 'decrypt': crypt = crypt['decrypt'] for info in decrypt: if info['name'] == crypt['name']: crypt['mode'] = info['mode'] crypt['function'] = info['function'] flag = True break if not flag: crypt = None return munchify(crypt) else: return None # 获取自定义参数配置 @gen.coroutine def get_custom_param(self, pid, correlation={}): setting, total = yield self.setting.get_settings_list(pid=pid, s_type='param', limit=None) setting = setting[0] if len(setting) > 0 else None if setting: params_str = [] params_data = [] params_func = [] for row in json.loads(setting.value): row = json.loads(row) if row['type'] == 'Function': for func in customs_func: if row['value'] == func['name']: row['function'] = func['function'] break params_func.append(row) elif row['type'] == 'Data': params_data.append(row) else: params_str.append(row) for key in correlation: if not isinstance(correlation[key], str): correlation[key] = str(correlation[key]) for param in params_str: for key in correlation: if param['value'].find(key) != -1: param['value'] = param['value'].replace( key, correlation[key]) for param in params_data: for row in params_str: if param['value'].find('{%s}' % row['name']) != -1: param['value'] = param['value'].replace( '{%s}' % row['name'], row['value']) for key in correlation: if param['value'].find(key) != -1: param['value'] = param['value'].replace( key, correlation[key]) argv, do = yield self.parse_sql_argv(param['value'], pid=pid) if argv: try: test = TestingModule(**argv) if do == 'SELECT': result = yield test.get_one_result() if result: param['value'] = result[0] else: param['value'] = '' else: raise Exception('目前只支持SELECT方法', do) except Exception as e: log.warning(e) param['value'] = '' else: param['value'] = '' return params_str + params_data + params_func else: return [] # 解析sql参数 @gen.coroutine def parse_sql_argv(self, sql_params, pid=0): sql_params = sql_params.splitlines() do = '' sql_argv = dict() if len(sql_params) == 5: for line in sql_params: line = line.strip().split(sep='=', maxsplit=1) if len(line) != 2: return dict(), do elif line[0].strip() not in [ 'mysql', 'user', 'password', 'database', 'sql' ]: return dict(), do elif line[1].strip() == '': return dict(), do elif line[0] == 'sql' and re.match( r'^SELECT\s+', line[1].strip().upper()) is not None: do = 'SELECT' elif line[0] == 'sql' and re.match( r'^DELETE\s+', line[1].strip().upper()) is not None: do = 'DELETE' elif line[0] == 'sql' and re.match( r'^UPDATE\s+', line[1].strip().upper()) is not None: do = 'UPDATE' elif line[0] == 'sql' and re.match( r'^INSERT\s+', line[1].strip().upper()) is not None: do = 'INSERT' if line[0].strip() in ['mysql']: sql_argv['engine'] = line[0].strip() line[1] = line[1].split(':', maxsplit=1) sql_argv['host'] = line[1][0].strip() ips, total = yield self.setting.get_settings_list( pid=pid, s_type='host', name=sql_argv['host'], pj_status=1, status=1, limit=None) if ips: sql_argv['host'] = ips[0].value sql_argv['port'] = line[1][1].strip() if len( line[1]) == 2 else 3306 elif line[0].strip() == 'user': sql_argv['user'] = line[1].strip() elif line[0].strip() == 'password': sql_argv['password'] = line[1].strip() elif line[0].strip() == 'database': sql_argv['database'] = line[1].strip() elif line[0].strip() == 'sql': sql_argv['sql'] = line[1].strip() if len(sql_argv) == 7 and do in [ 'SELECT', 'DELETE', 'UPDATE', 'INSERT' ]: return sql_argv, do else: return dict(), do # 获取接口完整性检查配置 @gen.coroutine def get_check_key(self, pid, url): setting, total = yield self.setting.get_settings_list(pid=pid, s_type='url', limit=None) check_key = '' if setting: for row in setting: row = json.loads(row.value) if row['url'] == url: check_key = row['check_key'] break return check_key
class TestRunner(object): Resolver.configure('tornado.netutil.ThreadedResolver') def __init__(self, pid=0, jid=0): self.default_headers = { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36', 'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4', 'Cookie': '' } self.headers = dict() self.setting = SettingModule() self.common_func = CommonFunction() self.option_func = OptionsFunction() self.pid = pid self.jid = jid # 获取请求头 def get_headers(self, headers=''): flag, headers = self.common_func.convert_to_list_or_dict( string=headers, s_type='dict') self.headers = dict() for key in self.default_headers: self.headers[key] = self.default_headers[key] if flag: for key in headers.keys(): self.headers[key] = headers[key] else: headers = headers.splitlines() for header in headers: header = header.strip().split(sep=':', maxsplit=1) if len(header) == 2: name = header[0].strip() value = header[1].strip() self.headers[name] = value return self.headers # 加解密操作 @gen.coroutine def __do_crypt(self, do='encrypt', body=None, crypt_key=''): crypt_info = yield self.option_func.get_crypt_info(pid=self.pid, do=do) if not crypt_info: return body func = crypt_info.function try: if func: if crypt_key: flag, body = self.common_func.convert_to_list_or_dict( string=body, s_type='dict') if not flag and isinstance(body, str) and re.match( r'^.*=.*(&.*=.*)*$', body) is not None: body = self.common_func.url_query_decode(body) if isinstance(body, dict): flag = True if flag: source = body[crypt_key] if not isinstance(source, str): try: source = json.dumps(source, ensure_ascii=False) except Exception as e: log.warning(e) if isinstance(source, bytes): source = source.decode('utf8', errors='ignore') else: source = str(source) body[crypt_key] = func(source, crypt_info['key'], crypt_info['iv'], crypt_info['mode']) else: if not isinstance(body, str): try: body = json.dumps(body, ensure_ascii=False) except Exception as e: log.warning(e) if isinstance(body, bytes): body = body.decode('utf8', errors='ignore') else: body = str(body) body = func(body, crypt_info['key'], crypt_info['iv'], crypt_info['mode']) except Exception as e: log.warning(e) return body # 尝试将请求数据转换成字典 def __parse_body_arguments(self, body): flag, body = self.common_func.convert_to_list_or_dict(string=body, s_type='dict') if not flag and isinstance(body, str) and re.match( r'^.*=.*(&.*=.*)*$', body) is not None: body = self.common_func.url_query_decode(body) if isinstance(body, dict): flag = True if not flag: self.headers = self.headers if len( self.headers) != 0 else self.default_headers try: request_body = dict() if re.match(r'^.*=.*(&.*=.*)*$', body) is not None: parse_body_arguments( content_type=self.headers['Content-Type'], body=body, arguments=request_body, files=request_body, headers=self.headers) if len(request_body) > 0: body = request_body for key in body: if isinstance(body[key], list): body[key] = body[key][0] body[key] = body[key].decode('utf8', errors='ignore') flag = True except Exception as e: log.warning(e) flag = False return flag, body # 获取请求响应数据 @gen.coroutine def __get_body(self, body='', do='encrypt', name='', crypt_key=''): try: body = ast.literal_eval(body) except Exception as e: log.warning(e) if do == 'encrypt' and crypt_key != '': flag, body = self.__parse_body_arguments(body) if isinstance(body, dict): if name != 'none': if do == 'encrypt' and crypt_key == '': body = urlencode(body, encoding='utf8', quote_via=quote) body = yield self.__do_crypt(do=do, body=body, crypt_key=crypt_key) if isinstance(body, dict): if do == 'encrypt' and self.headers['Content-Type'].find( 'x-www-form-urlencoded') != -1: body = urlencode(body, encoding='utf8', quote_via=quote) else: body = json.dumps(body, ensure_ascii=False) else: if name != 'none': body = yield self.__do_crypt(do=do, body=body, crypt_key='') if isinstance(body, dict): body = json.dumps(body, ensure_ascii=False) return body # 解析Host配置 @gen.coroutine def __parse_host(self, url='', env='none'): urls = self.common_func.url_split(url=url) host = urls.host ips, total = yield self.setting.get_settings_list(pid=self.pid, s_type='host', name=host, pj_status=1, limit=None) for row in ips: if env != 'none': url = '{}://{}:{}{}'.format(urls.scheme, env, urls.port, urls.path) break elif row.status == 1: url = '{}://{}:{}{}'.format(urls.scheme, row.value, urls.port, urls.path) break self.headers['Host'] = urls.netloc return url # 解析接口返回值全字段检查配置 def __parse_check_key(self, check_key): keys = [] top = [] rex = re.compile( r'^\[\w+=\d\|(int|float|num|str|/.*/|date|time|datetime|list|dict)(' r',\w+=\d\|(int|float|num|str|/.*/|date|time|datetime|list|dict))*\]$' ) for row in check_key.splitlines(): row = row.strip().split(sep='.', maxsplit=1) if len(row) == 2: if re.match(rex, row[1]) is not None: deeps = [row[1]] else: deeps = row[1].split(sep='.', maxsplit=1) if re.match(r'^\[\d+\]$', deeps[0]) is None: top.append('{}=1|dict'.format(row[0])) else: top.append('{}=1|list'.format(row[0])) deep = '{}.'.format(row[0]) tmp_key = [] for i in range(len(row[1].split('.'))): if len(deeps) == 2 and re.match(rex, deeps[1]) is not None: deep += '{}.'.format(deeps[0]) for j in deeps[1][1:-1].split(','): tmp_key.append(j) break elif re.match(rex, deeps[0]) is not None: for j in deeps[0][1:-1].split(','): tmp_key.append(j) break deep += '{}.'.format(deeps[0]) if len(deeps) == 2: if re.match(rex, deeps[1]) is not None: deeps = deeps[1] else: deeps = deeps[1].split(sep='.', maxsplit=1) keys.append(dict(deep=deep[:-1], keys=tmp_key, result=dict())) else: top.append(row[0]) keys.append(dict(deep='top', keys=list(set(top)), result=dict())) return keys # 返回值全字段检查结果判断 def __check_key_result(self, body, check_key, key, k): check_key[k]['result'][key[0]] = True key[1] = key[1].split(sep='|', maxsplit=1) require = key[1][0] key_type = key[1][1] if isinstance(body[key[0]], str): body[key[0]] = body[key[0]].strip() if require == '1' and (body[key[0]] == '' or body[key[0]] is None): check_key[k]['result'][key[0]] = False elif body[key[0]] != '' and body[key[0]] is not None: if re.match(r'^/.*/$', key_type): rex = re.compile(key_type[1:-1]) if re.match(rex, body[key[0]]) is None: check_key[k]['result'][key[0]] = False elif key_type == 'int' and not isinstance(body[key[0]], int): check_key[k]['result'][key[0]] = False elif key_type == 'float' and not isinstance(body[key[0]], float): check_key[k]['result'][key[0]] = False elif key_type == 'num' and not isinstance( body[key[0]], int) and not isinstance(body[key[0]], float): check_key[k]['result'][key[0]] = False elif key_type == 'str' and not isinstance(body[key[0]], str): check_key[k]['result'][key[0]] = False elif key_type == 'list' and not isinstance(body[key[0]], list): check_key[k]['result'][key[0]] = False elif key_type == 'dict' and not isinstance(body[key[0]], dict): check_key[k]['result'][key[0]] = False elif key_type == 'date' and re.match(r'^\d{4}-\d{2}-\d{2}$', body[key[0]]) is None: check_key[k]['result'][key[0]] = False elif key_type == 'time' and re.match(r'^\d{2}:\d{2}:\d{2}$', body[key[0]]) is None: check_key[k]['result'][key[0]] = False elif key_type == 'datetime' and re.match( r'^\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}$', body[key[0]]) is None: check_key[k]['result'][key[0]] = False if not check_key[k]['result'][key[0]]: check_key[k]['result']['key_result'] = False return body, check_key # 解析响应内容 @gen.coroutine def __parse_response(self, response, name='', crypt_key='', checkpoint='', check_key='', correlation='', method='GET', url=''): body = response.body if response else '' request_body = response.request.body if response else '' if isinstance(body, bytes): body = body.decode('utf8', errors='ignore') if isinstance(request_body, bytes): request_body = request_body.decode('utf8', errors='ignore') headers_dict = dict(response.headers if response else '') headers = '' for key in headers_dict: headers += '{}: {}\r\n'.format(key, headers_dict[key]) request_headers_dict = dict( response.request.headers if response else '') request_headers = '' for key in request_headers_dict: request_headers += '{}: {}\r\n'.format(key, request_headers_dict[key]) if response: error = response.error if not response.error else str( response.error) else: error = str(httpclient.HTTPError(599, 'Timeout while connecting')) resp = dict( body=body, code=response.code if response else 599, effective_url=response.effective_url if response else '', error=error, headers=headers, request_headers=request_headers, request_body=request_body, reason=response.reason if response else 'Timeout while connecting', request_time=response.request_time if response else '', time_info=response.time_info if response else '', method=method, url=url) resp['body_decrypt'] = yield self.__get_body(body, do='decrypt', name=name, crypt_key=crypt_key) resp['checkpoint'] = [] if checkpoint != '': if re.match(r'^/.*/$', checkpoint) is not None: rex = re.compile(checkpoint[1:-1]) if re.findall(rex, resp['body_decrypt']): resp['checkpoint'].append( dict(result=True, checkpoint=checkpoint)) else: log.warning('检查点 {} 检查不通过'.format(checkpoint)) resp['checkpoint'].append( dict(result=False, checkpoint=checkpoint)) else: for check in checkpoint.split('|'): check = check.strip() if resp['body_decrypt'].find(check) != -1: resp['checkpoint'].append( dict(result=True, checkpoint=check)) else: log.warning('检查点 {} 检查不通过'.format(check)) resp['checkpoint'].append( dict(result=False, checkpoint=check)) resp['check_key'] = [] if check_key != '': check_key = self.__parse_check_key(check_key) for k in range(len(check_key)): body = resp['body_decrypt'] check_key[k]['result']['key_result'] = True if check_key[k]['deep'] == 'top': for key in check_key[k]['keys']: if re.match(r'^\[\d+\]$', key) is not None: key = int(key[1:-1]) flag, body = self.common_func.convert_to_list_or_dict( body, 'list') if not flag: log.warning('返回值全字段检查 被检查数据格式不是List, 无法继续') if body in ['', '[]', '{}', [], {}]: check_key[k]['result']['ERROR'] = '被检查数据为空' else: check_key[k]['result'][ 'ERROR'] = '被检查数据类型不是List' check_key[k]['result']['key_result'] = False continue try: body[key] except Exception as e: log.warning(e) check_key[k]['result'][ 'ERROR'] = '被检查数据格式不包含List类型' check_key[k]['result']['key_result'] = False continue else: flag, body = self.common_func.convert_to_list_or_dict( body, 'dict') if not flag and isinstance(body, str) and re.match( r'^.*=.*(&.*=.*)*$', body) is not None: body = self.common_func.url_query_decode(body) if isinstance(body, dict): flag = True if not flag: log.warning('返回值全字段检查 被检查数据格式不是Dict, 无法继续') if body in ['', '[]', '{}', [], {}]: check_key[k]['result']['ERROR'] = '被检查数据为空' else: check_key[k]['result'][ 'ERROR'] = '被检查数据类型不是Dict' check_key[k]['result']['key_result'] = False continue key = key.split(sep='=', maxsplit=1) if key[0] not in body.keys(): log.warning('返回值全字段检查 {} 检查不通过'.format( check_key[k]['keys'])) check_key[k]['result'][key[0]] = '字段不存在' check_key[k]['result']['key_result'] = False continue else: body, check_key = self.__check_key_result( body=body, check_key=check_key, key=key, k=k) else: keys = check_key[k]['keys'] flag, keys = self.common_func.convert_to_list_or_dict( keys, 'list') for key in check_key[k]['deep'].split('.'): if re.match(r'^\[\d+\]$', key) is not None: key = int(key[1:-1]) flag, body = self.common_func.convert_to_list_or_dict( body, 'list') if not flag: log.warning('返回值全字段检查 被检查数据格式不是List, 无法继续') if body in ['', '[]', '{}', [], {}]: check_key[k]['result']['ERROR'] = '被检查数据为空' else: check_key[k]['result'][ 'ERROR'] = '被检查数据类型不是List' check_key[k]['result']['key_result'] = False continue elif not isinstance(body, dict): flag, body = self.common_func.convert_to_list_or_dict( body, 'dict') if not flag and isinstance(body, str) and re.match( r'^.*=.*(&.*=.*)*$', body) is not None: body = self.common_func.url_query_decode(body) if isinstance(body, dict): flag = True if not flag: log.warning('返回值全字段检查 被检查数据格式不是Dict, 无法继续') if body in ['', '[]', '{}', [], {}]: check_key[k]['result']['ERROR'] = '被检查数据为空' else: check_key[k]['result'][ 'ERROR'] = '被检查数据类型不是Dict' check_key[k]['result']['key_result'] = False continue try: body = body[key] except Exception as e: log.warning(e) check_key[k]['result'][ 'ERROR'] = '被检查数据格式不包含List类型' check_key[k]['result']['key_result'] = False continue flag, body = self.common_func.convert_to_list_or_dict( body, 'dict') if not flag and isinstance(body, str) and re.match( r'^.*=.*(&.*=.*)*$', body) is not None: body = self.common_func.url_query_decode(body) if isinstance(body, dict): flag = True if not flag: log.warning('返回值全字段检查 被检查数据格式不是Dict, 无法继续') if body in ['', '[]', '{}', [], {}]: check_key[k]['result']['ERROR'] = '被检查数据为空' else: check_key[k]['result']['ERROR'] = '被检查数据类型不是Dict' check_key[k]['result']['key_result'] = False continue for key in keys: key = key.split(sep='=', maxsplit=1) if key[0] not in body.keys(): log.warning('返回值全字段检查 {} 检查不通过'.format( check_key[k]['keys'])) check_key[k]['result'][key[0]] = '字段不存在' check_key[k]['result']['key_result'] = False continue else: body, check_key = self.__check_key_result( body=body, check_key=check_key, key=key, k=k) resp['check_key'] = check_key resp['correlation'] = dict() if response and correlation != '': correlations = correlation.split('|') correlation = dict() for corr in correlations: body = resp['body_decrypt'] cor = corr.split(sep='=', maxsplit=1) key = cor[0].strip() c_type = 'string' words = cor[1].strip() if re.match(r'^int\(.+\)$', words) is not None: c_type = 1 words = words[4:-1] elif re.match(r'^float\(.+\)$', words) is not None: c_type = 1.00 words = words[6:-1] word = words.split('.') correlation[key] = word for k in word: if k == 'response_headers' and len(word) != 1: header = words.split(sep='.', maxsplit=1) if len(header) == 2: header_key = header[1] if re.match(r'^/.*/$', header_key) is not None: rex = re.compile(header_key[1:-1]) if isinstance(resp['headers'], bytes): resp['headers'] = resp['headers'].decode( 'utf8', errors='ignore') result = re.findall(rex, resp['headers']) if result: if isinstance(result[0], tuple): body = result[0][0] else: body = '' for row in result: row = row if row != '' else '\n' body += row else: body = '' else: body = response.headers.get(header_key) break if k == 'response_body' and len(word) == 1: if isinstance(resp['body'], bytes): resp['body'] = resp['body'].decode('utf8', errors='ignore') body = escape.xhtml_escape(resp['body']) break if re.match(r'^/.*/$', words) is not None: rex = re.compile(words[1:-1]) if isinstance(resp['body'], bytes): resp['body'] = resp['body'].decode('utf8', errors='ignore') result = re.findall(rex, resp['body']) if result: if isinstance(result[0], tuple): body = escape.xhtml_escape(result[0][0]) else: body = escape.xhtml_escape(result[0]) else: body = '' break if re.match(r'^\[\d+\]$', k) is not None: k = int(k[1:-1]) flag, body = self.common_func.convert_to_list_or_dict( body, 'list') if not flag: log.warning('响应数据格式不是List, 无法继续') body = '' break elif not isinstance(body, dict): flag, body = self.common_func.convert_to_list_or_dict( body, 'dict') if not flag and isinstance(body, str) and re.match( r'^.*=.*(&.*=.*)*$', body) is not None: body = self.common_func.url_query_decode(body) if isinstance(body, dict): flag = True if not flag: log.warning('响应数据格式不是Dict, 无法继续') body = '' break try: body = body[k] except Exception as e: log.warning(e) body = '' break correlation[key] = body try: if isinstance(c_type, int): correlation[key] = int( float(re.sub(r'[^\d+\.]', '', body))) elif isinstance(c_type, float): correlation[key] = float(re.sub(r'[^\d+\.]', '', body)) except Exception as e: log.warning(e) resp['correlation'] = correlation flag = True if resp['error'] is not None and resp['code'] != 302 and resp[ 'code'] != 301: flag = False if resp['check_key']: for line in resp['check_key']: if not line['result']['key_result']: flag = False break if resp['checkpoint']: for line in resp['checkpoint']: if not line['result']: flag = False break resp['test_result'] = flag return resp # 解析自定义参数配置 @gen.coroutine def __parse_custom_param(self, headers, body, correlation_result={}): correlation_result = dict(self.common_func.default_param(), **correlation_result) params = yield self.option_func.get_custom_param( pid=self.pid, correlation=correlation_result) if isinstance(headers, bytes): headers = headers.decode('utf8', errors='ignore') if isinstance(body, bytes): body = body.decode('utf8', errors='ignore') for key in correlation_result: if not isinstance(correlation_result[key], str): correlation_result[key] = str(correlation_result[key]) if headers.find(key) != -1: headers = headers.replace(key, correlation_result[key]) if body.find(key) != -1: body = body.replace(key, correlation_result[key]) for param in params: if headers.find('{%s}' % param['name']) != -1: if param['type'] == 'Function': func = param['function'] flag, body_dict = self.__parse_body_arguments(body) encrypt = yield self.option_func.get_crypt_info(self.pid) headers = headers.replace('{%s}' % param['name'], func(body_dict, params, encrypt)) else: headers = headers.replace('{%s}' % param['name'], param['value']) if body.find('{%s}' % param['name']) != -1: if param['type'] == 'Function': func = param['function'] flag, body_dict = self.__parse_body_arguments(body) encrypt = yield self.option_func.get_crypt_info(self.pid) body = body.replace('{%s}' % param['name'], func(body_dict, params, encrypt)) else: body = body.replace('{%s}' % param['name'], param['value']) return headers, body # 请求操作 @gen.coroutine def __request_url(self, url='', env='none', method='GET', body='', follow_redirects=True): test_client = httpclient.AsyncHTTPClient(max_clients=100) argv = dict(method=method, headers=self.headers, follow_redirects=False, request_timeout=600, validate_cert=False, raise_error=False) url = yield self.__parse_host(url=url, env=env) if method == 'GET': url = '{}?{}'.format(url, body) elif method == 'POST': argv['body'] = body try: log.info('开始请求接口 {}'.format(url)) response = yield test_client.fetch(url, **argv) except httpclient.HTTPError as e: response = e.response log.warning('请求接口 {} 异常# {}'.format( url, str(response.error if response else e))) # test_client.close() if response and response.code in [301, 302]: for cookie in response.headers.get_list('Set-Cookie'): self.headers['Cookie'] += '{};'.format(cookie) url = response.headers.get('Location') log.info('{} {} {}'.format(response.code, response.reason, url)) if response.reason.find('Moved') >= 0: response = yield self.__request_url( url=url, env=env, method=method, body=body, follow_redirects=follow_redirects) elif follow_redirects: response = yield self.__request_url( url=url, env=env, method='GET', follow_redirects=follow_redirects) log.info('结束请求接口 {}'.format(url)) return response # 生成测试报告 @gen.coroutine def __gen_report(self, job_name, test_suites, start_time, end_time): setting = yield self.setting.get_settings_by_range(pid=self.pid, s_type='log', start=start_time, end=end_time, sort=self.jid) if setting: elapsed_time = end_time - start_time start_time = time.strftime( '%Y-%m-%d %H:%M:%S', time.gmtime(float(start_time) + 3600 * 8)) end_time = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(float(end_time) + 3600 * 8)) url, total = yield self.setting.get_settings_list(pid=self.pid, s_type='url', limit=None) overview = dict(name=job_name, start_time=start_time, end_time=end_time, elapsed_time=elapsed_time, total=total, total_test=len(test_suites), success_test=0, fail_test=0, success_rate='0.0 %', report_time=time.strftime('%Y-%m-%d %H:%M:%S')) for suite in test_suites: suite['total_test'] = len(suite['cases']) suite['success_test'] = 0 suite['fail_test'] = 0 suite['report'] = [] suite['result'] = True for row in setting: res = json.loads(row.value) for suite in test_suites: if suite['suite_id'] == row.status: if res['test_result']: suite['success_test'] += 1 else: suite['fail_test'] += 1 suite['result'] = False suite['report'].append(res) for suite in test_suites: suite['result'] = suite['result'] if suite[ 'success_test'] + suite['fail_test'] == suite[ 'total_test'] else False if suite['result']: overview['success_test'] += 1 else: overview['fail_test'] += 1 overview['success_rate'] = '{:.2f} %'.format( overview['success_test'] / overview['total_test'] * 100) result = dict(overview=overview, report=test_suites) report_id, msg = yield self.setting.add_setting( pid=self.pid, s_type='report', sort=self.jid, name=time.time(), value=json.dumps(result, ensure_ascii=False)) return report_id else: return False # 执行单接口测试 @gen.coroutine def run_test(self, url='', label='', comment='', method='GET', headers='', body='', crypt='none', encrypt_content='', no_test=False, check_key='', decrypt_content='', checkpoint='', env='none', correlation='', correlation_result={}, follow_redirects=True): headers, body = yield self.__parse_custom_param( headers=headers, body=body, correlation_result=correlation_result) self.get_headers(headers) request_body = yield self.__get_body(body=body, do='encrypt', name=crypt, crypt_key=encrypt_content) if no_test: return request_body resp = yield self.__request_url(url=url, env=env, method=method, body=request_body, follow_redirects=follow_redirects) response = yield self.__parse_response(response=resp, name=crypt, crypt_key=decrypt_content, checkpoint=checkpoint, check_key=check_key, correlation=correlation, method=method, url=url) response['label'] = label response['comment'] = comment if not resp: response['request_body'] = request_body for key in self.headers: response['request_headers'] += '{}: {}\r\n'.format( key, self.headers[key]) elif method == 'GET': response['request_body'] = request_body log.info('响应返回 {}'.format(json.dumps(response, ensure_ascii=False))) return response # 执行多接口测试 @gen.coroutine def __run_all_test(self, job_name, test_suites): if not test_suites: return False start_time = time.time() correlation_result = dict() for test in test_suites: cases = yield self.setting.get_settings_by_ids(test['cases']) for url_info in cases: try: url_info = json.loads(url_info.value) resp = yield self.run_test( correlation_result=correlation_result, **url_info) yield self.setting.add_setting(pid=self.pid, s_type='log', name=time.time(), sort=self.jid, value=json.dumps( resp, ensure_ascii=False), status=test['suite_id']) correlation_result = dict(correlation_result, **resp['correlation']) except Exception as e: log.error(e) end_time = time.time() report_id = yield self.__gen_report(job_name, test_suites, start_time, end_time) return report_id # 执行排队任务 @gen.coroutine def run_job(self): job = yield self.setting.get_setting_by_id(sid=self.jid) if job: start_time = time.time() job_value = json.loads(job.value) start_strftime = time.strftime( '%Y-%m-%d %H:%M:%S', time.gmtime(float(start_time) + 3600 * 8)) job_value['overview']['start_time'] = start_strftime yield self.setting.edit_setting(sid=job.id, status=2, value=json.dumps( job_value, ensure_ascii=False)) test_suites = [] suites = yield self.setting.get_settings_by_ids( job_value['testsuite']) for suite in suites: tests = dict(suite_id=suite.id, suite_name=suite.name, cases=json.loads(suite.value)['cases']) test_suites.append(tests) result = yield self.__run_all_test(job_value['name'], test_suites) if result: status = 3 job_value['lastreport'] = result else: status = 5 end_time = time.time() elapsed_time = end_time - start_time end_time = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(float(end_time) + 3600 * 8)) job_value['overview']['end_time'] = end_time job_value['overview']['elapsed_time'] = elapsed_time name = float(job.name) if job_value['overview']['cycle_time'] != 0: while name < time.time(): name += job_value['overview']['cycle_time'] job_value['overview']['plan_time'] = time.strftime( '%Y-%m-%d %H:%M:%S', time.gmtime(name + 3600 * 8)) status = 0 yield self.setting.edit_setting(sid=job.id, status=status, name=name, value=json.dumps( job_value, ensure_ascii=False)) yield Mail().send_html_report(result)
def __init__(self): self.setting = SettingModule()
def send_html_report(self, rid): if rid: func_option = OptionsFunc() settings = SettingModule() lists = [] pid = 0 project = '' name = '' report_time = '' report_url = yield func_option.get_option_by_name('report_url') report_url = report_url if report_url else '' setting = yield settings.get_setting_by_id(rid) if setting and setting.type == 'report': report = json.loads(setting.value) pid = setting.project_id project = setting.project_name name = report['overview']['name'] report_time = report['overview']['report_time'] lists = report['report'] for row in lists: if row['report']: row['success_rate'] = '{:.2f} %'.format(row['success_test'] / row['total_test'] * 100) else: row['success_rate'] = '0.00 %' lists = munchify(lists) for i in range(len(lists)): param = dict(id=i+1, suite_name=lists[i].suite_name, project=project, total_test=lists[i].total_test, success_test=lists[i].success_test, fail_test=lists[i].fail_test, rid=rid, success_rate=lists[i].success_rate, suite_id=lists[i].suite_id, report_url=report_url) if lists[i].result: param['result'] = '<span class ="label label-info">通过</span>' else: param['result'] = '<span class ="label label-danger">失败</span>' rows = """ <tr> <td>{id}</td> <td>{suite_name}</td> <td>{project}</td> <td>{total_test}</td> <td>{success_test}</td> <td>{fail_test}</td> <td>{success_rate}</td> <td>{result}</td> <td><a class ="btn btn-primary" href="{report_url}/admin/interface-test/reports/list/{rid}/{suite_id}">查看详情</a></td> </tr>""".format(**param) style = """ <style type="text/css"> .report-body * { -webkit-box-sizing: border-box; -moz-box-sizing: border-box; box-sizing: border-box; } .report-body { padding-top: 50px; padding-bottom: 40px; background-color: #eee; font-family: "Microsoft Yahei"; font-size: 14px; line-height: 1.42857143; color: #333; } .report-body .container-fluid { margin-right: auto; margin-left: auto; } .report-body .well { min-height: 20px; padding: 19px; margin-bottom: 20px; background-color: #f5f5f5; border: 1px solid #e3e3e3; border-radius: 4px; box-shadow: inset 0 1px 1px rgba(0,0,0,.05); } .report-body .table-responsive { min-height: .01%; overflow-x: auto; } .report-body table { background-color: transparent; border-spacing: 0; border-collapse: collapse; border-color: grey; } .report-body thead { display: table-header-group; vertical-align: middle; vertical-align: middle; } .report-body tr { display: table-row; vertical-align: inherit; border-color: inherit; } .report-body th { text-align: left; font-weight: bold; display: table-cell; } .report-body tbody { display: table-row-group; vertical-align: middle; border-color: inherit; } .report-body .table { width: 100%; max-width: 100%; margin-bottom: 20px; } .report-body .table>thead:first-child>tr:first-child>th { border-top: 0; } .report-body .table>thead>tr>th { vertical-align: bottom; border-bottom: 2px solid #ddd; padding: 8px; line-height: 1.42857143; } .report-body .table>tbody>tr>td { padding: 8px; line-height: 1.42857143; vertical-align: top; border-top: 1px solid #ddd; display: table-cell; } .report-body .table-striped>tbody>tr:nth-of-type(odd) { background-color: #f9f9f9; } .report-body .label-info { background-color: #5bc0de; } .report-body .label-danger { background-color: #d9534f; } .report-body .label { display: inline; padding: .2em .6em .3em; font-size: 75%; font-weight: 700; line-height: 1; color: #fff; text-align: center; white-space: nowrap; vertical-align: baseline; border-radius: .25em; } .report-body .btn-primary { color: #fff; background-color: #337ab7; border-color: #2e6da4; } .report-body .btn { display: inline-block; padding: 6px 12px; margin-bottom: 0; font-size: 14px; font-weight: 400; line-height: 1.42857143; text-align: center; white-space: nowrap; vertical-align: middle; -ms-touch-action: manipulation; touch-action: manipulation; cursor: pointer; -webkit-user-select: none; -moz-user-select: none; -ms-user-select: none; user-select: none; background-image: none; border: 1px solid transparent; border-radius: 4px; } .report-body a { color: #337ab7; text-decoration: none; } </style> """ content = """ <div class="report-body"> <div class="container-fluid well"> <h2 style="text-align: center;">任务 {name} 测试报告 / 报告时间 {report_time}</h2> <div class="table-responsive"> <table class="table table-striped table-hover"> <thead> <tr> <th>#</th> <th>用例名称</th> <th>所属项目</th> <th>测试接口数</th> <th>通过数</th> <th>失败数</th> <th>通过率</th> <th>测试结果</th> <th>操作</th> </tr> </thead> <tbody> {rows} </tbody> </table> </div> </div> {style} </div>""".format(**dict(style=style, name=name, report_time=report_time, rows=rows)) smtp_server = yield func_option.get_option_by_name('smtp_host') smtp_port = yield func_option.get_option_by_name('smtp_port') use_ssl = yield func_option.get_option_by_name('use_ssl') smtp_user = yield func_option.get_option_by_name('smtp_user') smtp_password = yield func_option.get_option_by_name('smtp_password') mail_from = yield func_option.get_option_by_name('mail_from') mail_report = yield func_option.get_option_by_name('mail_report') company = yield func_option.get_option_by_name('company') self.smtp_server = smtp_server if smtp_server else self.smtp_server self.smtp_port = smtp_port if smtp_port else self.smtp_port self.use_ssl = True if use_ssl == 'on' else self.use_ssl self.smtp_user = smtp_user if smtp_user else self.smtp_user self.smtp_password = smtp_password if smtp_password else self.smtp_password self.mail_from = mail_from if mail_from else self.mail_from mail_report = True if mail_report == 'on' else False users = yield ProjectModule().get_project(pid=pid, status=1) if users: mail_to = [] users = json.loads(users.user) if users.user else [] for user in users: user = json.loads(user) if user['mail']: mail_to.append(user['email']) if mail_to and mail_report: result, msg = yield self.send_mail(subject='[{}][测试报告]'.format(company), message=content, to=mail_to) if result: log.info('发送任务 {} 测试报告邮件成功'.format(name)) else: log.warning('发送任务 {} 测试报告邮件失败# {}'.format(name, msg))