def prepare(self):
self.user = UserModule()
self.project = ProjectModule()
self.setting = SettingModule()
self.option = OptionModule()
self.msg = MessagesModule()
self.statistics = StatisticsModule()
self.common_func = CommonFunction()
self.option_func = OptionsFunction()
self.thread_func = ThreadFunction()
def __init__(self):
self.project = ProjectModule()
self.setting = SettingModule()
self.option_func = OptionsFunction()
self.common_func = CommonFunction()
self.thread_func = ThreadFunction()
self.jenkins_server = jenkins.Jenkins(url=jenkins_url,
username=jenkins_user,
password=jenkins_password,
timeout=5)
def __init__(self, pid=0, jid=0):
self.default_headers = {
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36',
'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4',
'Cookie': ''
}
self.headers = dict()
self.setting = SettingModule()
self.common_func = CommonFunction()
self.option_func = OptionsFunction()
self.pid = pid
self.jid = jid
def prepare(self):
self.common_func = CommonFunction()
self.user = UserModule()
self.project = ProjectModule()
self.setting = SettingModule()
self.option = OptionModule()
self.option_func = OptionsFunction()
self.company = yield self.option_func.get_option_by_name(name='company')
self.limit = yield self.option_func.get_option_by_name(name='page_limit')
self.company = self.company or '开源接口测试平台'
self.argv = dict(company=self.company)
def __init__(self):
self.common_func = CommonFunction()
class UserModule(object):
"""
用户表相关操作
"""
def __init__(self):
self.common_func = CommonFunction()
# 获取用户信息
@gen.coroutine
def get_user_info(self, email_or_username=None, uid=None, status=None):
where = []
params = dict()
if uid is not None:
where.append("u.id=%(uid)s")
params['uid'] = uid
if email_or_username is not None:
where.append("(u.email=%(user)s OR u.username=%(user)s)")
params['user'] = email_or_username
if status is not None:
where.append("u.status=%(status)s")
params['status'] = status
sql = "SELECT u.*, (SELECT COUNT(IF(m.`type` IN ('notice', 'message') AND m.`status` NOT IN (0, 2), m.`type`, NULL)) + COUNT(IF(m.`type` IN ('todo') AND m.`status` NOT IN (0, 5), m.`type`, NULL)) FROM t_messages m WHERE m.type <> 'active' AND m.userId=u.id AND m.`status` <> 0) as unreadCount FROM t_users u"
if where:
sql += ' WHERE {}'.format(' AND '.join(where))
try:
cursor = yield pool.execute(sql, params)
result = cursor.fetchone()
cursor.close()
return munchify(result)
except pymysql.Error as e:
log.error(e)
return None
# 通过user_id获取用户信息
@gen.coroutine
def get_users_info_by_id(self, uid, status=None):
if isinstance(uid, list):
uid = ','.join([str(u) for u in uid])
sql = 'SELECT * FROM t_users u WHERE u.id in ({})'.format(uid)
param = dict()
if status is not None:
sql += ' AND status=%(status)s'
param['status'] = status
sql += ' ORDER BY u.role'
try:
cursor = yield pool.execute(sql, param)
result = cursor.fetchall()
cursor.close()
return munchify(result)
except pymysql.Error as e:
log.error(e)
return []
# 获取用户列表
@gen.coroutine
def get_users_list(self, page=1, limit=None, status=None, name=None):
sql = 'SELECT * FROM t_users u'
sql_count = 'SELECT COUNT(*) count FROM t_users u'
where = []
params = dict()
if name is not None:
where.append(
"(u.realname like %(name)s OR u.username like %(name)s OR u.email like %(name)s OR u.profile like %(name)s)"
)
params['name'] = '%{}%'.format(name)
if status is not None:
where.append("u.status=%(status)s")
params['status'] = status
if where:
where = ' WHERE {}'.format(' AND '.join(where))
sql += where
sql_count += where
sql += ' ORDER BY u.role'
if limit is not None:
offset = (page - 1) * limit
sql += ' LIMIT {},{}'.format(offset, limit)
try:
cursor = yield pool.execute(sql, params)
result = cursor.fetchall()
cursor = yield pool.execute(sql_count, params)
total = cursor.fetchone()
cursor.close()
return munchify(result), munchify(total).count
except pymysql.Error as e:
log.error(e)
return [], 0
# 注册用户
@gen.coroutine
def register_user(self,
email,
password,
username=None,
real_name=None,
profile=None,
role=1,
status=1):
register_time = time.strftime('%Y-%m-%d %H:%M:%S')
password = self.common_func.encode_password(password)
try:
cursor = yield pool.execute('SELECT COUNT(*) count FROM t_users')
total = munchify(cursor.fetchone())
if total.count == 0:
role = 0
status = 2
except pymysql.Error as e:
log.error(e)
username = username or '{}_{}'.format(
email.split('@')[0], str(int(time.time() * 1000)))
sql = """
INSERT INTO t_users (username, email, password, realname, profile, registerTime, lastLoginTime, role, status)
VALUE(%(username)s, %(email)s, %(password)s, %(realname)s, %(profile)s, %(registerTime)s, %(lastLoginTime)s, %(role)s, %(status)s)
"""
user = yield self.get_user_info(email_or_username=email)
if not user:
with (yield pool.Connection()) as conn:
with conn.cursor() as cursor:
try:
yield cursor.execute(
sql,
dict(
username=username,
email=email,
password=password,
realname=real_name or '',
registerTime=register_time,
lastLoginTime=register_time,
profile=json.dumps(profile, ensure_ascii=False)
or '',
role=role,
status=status))
except pymysql.Error as e:
yield conn.rollback()
log.error('注册用户 {} 失败#{}'.format(email, e))
flag, msg = False, '注册用户 {} 失败#{}'.format(email, e)
else:
yield conn.commit()
log.info('注册用户 {} 成功'.format(email))
flag, msg = munchify(
dict(id=cursor.lastrowid, status=status,
role=role)), '注册用户成功!'
else:
log.error('该邮箱已注册!')
flag, msg = False, '该邮箱已注册!'
return flag, msg
# 编辑用户
@gen.coroutine
def edit_user(self,
email=None,
uid=None,
password=None,
username=None,
real_name=None,
last_login_time=None,
role=None,
status=None,
profile=None):
user = yield self.get_user_info(
email_or_username=None if uid else email, uid=uid)
if user:
update = []
param = dict(email=email if email else user.email)
if password is not None:
update.append("password=%(password)s")
param['password'] = self.common_func.encode_password(password)
if username is not None:
sql = "SELECT id, username FROM t_users u WHERE u.email != %(email)s AND u.username = %(username)s"
param['username'] = username
try:
cursor = yield pool.execute(sql, param)
user_info = cursor.fetchone()
if (user_info and uid
and uid != user_info['id']) or (user_info
and uid is None):
log.error('用户名 {} 已存在'.format(username))
return False, '用户名 {} 已存在'.format(username)
else:
update.append("username=%(username)s")
except pymysql.Error as e:
log.error(e)
return False, '编辑用户失败#{}'.format(e)
if email is not None and uid is not None:
is_exist_user = yield self.get_user_info(email)
if is_exist_user and is_exist_user.id != uid:
log.error('该邮箱 {} 已注册'.format(email))
return False, '该邮箱 {} 已注册'.format(email)
else:
update.append("email=%(email)s")
if real_name is not None:
update.append("realname=%(realname)s")
param['realname'] = real_name
if last_login_time is not None:
update.append("lastLoginTime=%(lastLoginTime)s")
param['lastLoginTime'] = last_login_time
if role is not None:
update.append('role=%(role)s')
param['role'] = role
if profile is not None:
update.append('profile=%(profile)s')
param['profile'] = json.dumps(profile, ensure_ascii=False)
if status is not None:
update.append('status=%(status)s')
param['status'] = status
if update:
if uid is not None:
param['uid'] = uid
sql = "UPDATE t_users SET {} WHERE id=%(uid)s".format(
', '.join(update))
else:
sql = "UPDATE t_users SET {} WHERE email=%(email)s".format(
', '.join(update))
tx = yield pool.begin()
try:
yield tx.execute(sql, param)
except pymysql.Error as e:
yield tx.rollback()
log.error('编辑用户失败#{}'.format(e))
flag, msg = False, '用户 {} 资料修改失败'.format(user.email)
else:
yield tx.commit()
log.info('用户 {} 资料修改成功'.format(user.email))
flag, msg = True, '用户 {} 资料修改成功'.format(user.email)
return flag, msg
else:
log.error('没有可更新的项')
return False, '没有可更新的项'
else:
log.error('用户 {} 不存在!'.format(email))
return False, '用户 {} 不存在!'.format(email)
# 删除用户信息
@gen.coroutine
def delete_user(self, uid=None, username_or_email=None, status=None):
user = yield self.get_user_info(uid=uid,
email_or_username=username_or_email,
status=status)
if user:
where = []
params = dict()
if uid is not None:
where.append("id=%(uid)s")
params['uid'] = uid
if username_or_email is not None:
where.append("(email=%(user)s OR username=%(user)s)")
params['user'] = username_or_email
if status is not None:
where.append("status=%(status)s")
params['status'] = status
sql = 'DELETE FROM t_users'
if where:
sql += ' WHERE {}'.format(' AND '.join(where))
tx = yield pool.begin()
try:
yield tx.execute(sql, params)
except pymysql.Error as e:
yield tx.rollback()
log.error('删除用户失败#{}'.format(e))
flag, msg = False, '删除用户失败'
else:
yield tx.commit()
log.info('删除用户成功')
flag, msg = True, '删除用户成功'
return flag, msg
else:
log.error('用户不存在!')
return False, '用户不存在!'
class JobsMonitor(object):
def __init__(self):
self.project = ProjectModule()
self.setting = SettingModule()
self.option_func = OptionsFunction()
self.common_func = CommonFunction()
self.thread_func = ThreadFunction()
self.jenkins_server = jenkins.Jenkins(url=jenkins_url,
username=jenkins_user,
password=jenkins_password,
timeout=5)
# 监控任务状态
@gen.coroutine
def jobs_status(self):
try:
job_list, total = yield self.setting.get_settings_list(
s_type=['job', 'jobG', 'jobA'], status=[0, 2], pj_status=1)
for job in job_list:
desc = json.loads(job.value)
if (desc.get('buildEnv')
or desc.get('dayBuild')) and job.status == 0 and int(
time.time()) > int(job.createTime.timestamp()):
yield self.setting.edit_setting(sid=job.id, status=1)
continue
job_name = desc.get(
'jobName') if job.type != 'job' else jenkins_jacoco
if job_name and job.status == 2:
if job.type != 'job':
last_build_number = self.jenkins_server.get_job_info(
job_name).get('lastBuild').get('number')
last_build = self.jenkins_server.get_build_info(
job_name, last_build_number)
status = None if last_build.get('building') else 0
else:
status = None
if (job.type != 'job' and last_build.get('queueId')
== desc.get('queueId')) or (job_name
== jenkins_jacoco):
if (job.type != 'job' and status is None
and desc['url'] == last_build.get('url')) or (
job_name == jenkins_jacoco):
if desc.get('jacocoId'):
jacoco_job = self.jenkins_server.get_job_info(
jenkins_jacoco)
first_build_number = jacoco_job.get(
'firstBuild').get('number')
last_build_number = jacoco_job.get(
'lastBuild').get('number')
for num in range(first_build_number,
last_build_number + 1):
try:
jacoco = self.jenkins_server.get_build_info(
jenkins_jacoco, num)
if jacoco.get('queueId') != desc.get(
'jacocoId'):
continue
if not jacoco.get(
'building') and jacoco.get(
'result') == 'FAILURE':
day_build_env = desc.get(
'buildEnv') or desc.get(
'dayBuild')
apps = yield self.option_func.get_env_info(
eid=day_build_env)
stop_time = time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() +
3600 * 24))
parameters = [
('ACTION', 'Monitoring'),
('STOPTIME', stop_time),
('CLEAN', 'No'),
('JOB_ID', job.name)
]
for app in [
a.get('details')
for a in apps
]:
for ap in app:
if ap.get('title').lower(
).find('linux_app') != -1:
parameters.append(
('ENV',
ap.get('ip')))
if len(parameters) > 4:
version_file = os.path.join(
static_path, 'diffAPP',
job.name)
if not os.path.isdir(
version_file):
os.makedirs(version_file)
version_file = os.path.join(
version_file,
'jacoco_app_version.txt')
if os.path.isfile(
version_file):
os.remove(version_file)
for app in (desc.get('runApps')
or []):
parameters.append(
('APP', app))
jacoco_id = self.jenkins_server.build_job(
jenkins_jacoco,
parameters=parameters)
desc['jacocoId'] = jacoco_id
yield self.setting.edit_setting(
sid=job.id, value=desc)
except Exception as e:
log.error(e)
continue
continue
desc['url'] = last_build.get('url')
cycle = desc.get('cycle')
if cycle == 'hour':
next_time = job.createTime + timedelta(hours=1)
elif cycle == 'day':
next_time = job.createTime + timedelta(days=1)
elif cycle == 'week':
next_time = job.createTime + timedelta(weeks=1)
elif cycle == 'mouth':
next_time = job.createTime + timedelta(days=30)
elif cycle == 'year':
next_time = job.createTime + timedelta(days=365)
else:
next_time = job.createTime
if status == 0: status = 3
if status in [0, 3] and desc.get('jacocoId'):
jacoco_job = self.jenkins_server.get_job_info(
jenkins_jacoco)
first_build_number = jacoco_job.get(
'firstBuild').get('number')
last_build_number = jacoco_job.get(
'lastBuild').get('number')
for num in range(first_build_number,
last_build_number + 1):
try:
jacoco = self.jenkins_server.get_build_info(
jenkins_jacoco, num)
if jacoco.get('queueId') != desc.get(
'jacocoId'):
continue
self.jenkins_server.stop_build(
jenkins_jacoco, num)
except Exception as e:
log.error(e)
continue
apps = yield self.option_func.get_env_info(
eid=desc.get('buildEnv')
or desc.get('dayBuild'))
desc['buildEnv'] = ''
parameters = [('ACTION', 'Reporting'),
('HIS_BUILD_ID', num),
('CLEAN', 'Yes'),
('JOB_ID', job.name)]
for app in [a.get('details') for a in apps]:
for ap in app:
if ap.get('title').lower().find(
'linux_app') != -1:
parameters.append(
('ENV', ap.get('ip')))
if len(parameters) > 4:
for app in (desc.get('runApps') or []):
parameters.append(('APP', app))
self.jenkins_server.build_job(
jenkins_jacoco, parameters=parameters)
yield self.setting.edit_setting(
sid=job.id,
value=desc,
status=status,
create_time=next_time.strftime('%Y-%m-%d %H:%M:%S')
if status in [0, 3] else None)
except Exception as e:
log.warning(e)
# 执行定时任务
@gen.coroutine
def run_jobs(self):
try:
job_list, total = yield self.setting.get_settings_list(
s_type=['jobG', 'jobA'], status=1, pj_status=1)
for job in job_list:
desc = json.loads(job.value)
job_name = desc.get('jobName')
day_build_env = desc.get('buildEnv') or desc.get('dayBuild')
if not job_name or not day_build_env:
continue
data = munchify(
dict(env=day_build_env,
type='api' if job.type == 'jobA' else 'gui',
exec=job_name))
yield self.option_func.get_hosts_info(data)
yield self.option_func.get_mysql_jdbc(eid=day_build_env,
data=data)
yield self.option_func.get_cases_info(job, data)
queue_id = self.jenkins_server.build_job(
job_name, parameters=dict(JOB_ID=job.name))
if queue_id:
apps = yield self.option_func.get_env_info(
eid=day_build_env)
stop_time = time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() + 3600 * 24))
parameters = [('ACTION', 'Monitoring'),
('STOPTIME', stop_time), ('CLEAN', 'Yes'),
('JOB_ID', job.name)]
for app in [a.get('details') for a in apps]:
for ap in app:
if ap.get('title').lower().find('linux_app') != -1:
parameters.append(('ENV', ap.get('ip')))
if len(parameters) > 4:
version_file = os.path.join(static_path, 'diffAPP',
job.name)
if not os.path.isdir(version_file):
os.makedirs(version_file)
version_file = os.path.join(version_file,
'jacoco_app_version.txt')
if os.path.isfile(version_file):
os.remove(version_file)
for app in (desc.get('runApps') or []):
parameters.append(('APP', app))
jacoco_id = self.jenkins_server.build_job(
jenkins_jacoco, parameters=parameters)
desc['jacocoId'] = jacoco_id
desc['queueId'] = queue_id
desc['url'] = self.jenkins_server.get_job_info(
job_name).get('url')
yield self.setting.edit_setting(sid=job.id,
value=desc,
status=2)
except Exception as e:
log.warning(e)
# 申请关闭外网权限
@gen.coroutine
def close_network(self):
try:
now_date = time.strftime('%Y-%m-%d')
now_time = time.strftime('%H:%M:%S')
if now_time > '08:30:00':
server, total = yield self.setting.get_settings_list(
s_type='env', status=1)
ips = list()
for svr in server:
desc = json.loads(svr.value)
if (svr.createTime.strftime('%Y-%m-%d') < now_date or
(svr.createTime.strftime('%Y-%m-%d') == now_date
and now_time > svr.createTime.strftime('%H:%M:%S'))
) and desc.get('network') == 'yes' and desc.get('ip'):
ips.append(desc.get('ip').strip())
if set(ips):
title = '测试环境申请关闭外网权限'
mail_content = '''
<p>Hi 你好！</p>
<p style="padding-left:30px;">测试人员已完成测试任务，申请关闭以下测试环境服务器外网权限。请帮忙处理一下，3ks~</p>
<p style="padding-left:30px;">1、服务器</p>
<p style="padding-left:60px;">{}</p>
'''.format('</br>'.join(set(ips)))
res, msg = yield self.common_func.send_email(
subject=title,
content=mail_content,
to=net_mail_to,
cc=net_mail_cc)
if res:
log.info(msg)
for svr in server:
desc = json.loads(svr.value)
if (svr.createTime.strftime('%Y-%m-%d') < now_date
or (svr.createTime.strftime('%Y-%m-%d')
== now_date and now_time >
svr.createTime.strftime('%H:%M:%S'))
) and desc.get(
'network') == 'yes' and desc.get('ip'):
desc['network'] = 'no'
yield self.setting.edit_setting(sid=svr.id,
value=desc)
else:
log.warn(msg)
except Exception as e:
log.error(e)
# 同步准生产数据库表
@gen.coroutine
def sync_db_tables(self):
dumps = dict()
db_path = os.path.join(static_path, 'syncDB', 'lasTables')
if not os.path.isdir(db_path):
os.makedirs(db_path)
flag_file = os.path.join(db_path, 'TAG')
old_date = ''
if os.path.isfile(flag_file):
with open(flag_file, 'r') as fp:
old_date = fp.read()
now_date = time.strftime('%Y%m%d')
if old_date != now_date:
env_list, total = yield self.project.get_projects_list(
p_type='env', status=1, search='准生产')
for env in env_list:
details, total = yield self.setting.get_settings_list(
s_type='env', name=env.name)
for detail in details:
desc = json.loads(detail.value)
if desc.get('type') != 'APPLICATION' and desc.get(
'title').upper().find('MYSQL') == -1:
continue
dumps[detail.id] = dict(
ip=desc.get('ip').strip(),
port=desc.get('port').strip(),
user=desc.get('user').strip(),
password=desc.get('password').strip(),
dbs=desc.get('description').split(','))
break
with open(flag_file, 'w') as fp:
fp.write(time.strftime('%Y%m%d'))
for key in dumps:
for db in dumps[key]['dbs']:
tmp_path = os.path.join(db_path, '{}_tmp.txt'.format(db))
shell_dump = '''cd {}
/opt/lampp/bin/mysqldump -h{} -P{} -u{} -p{} -d {} > {}'''.format(
root_160, dumps[key]['ip'], dumps[key]['port'],
dumps[key]['user'], dumps[key]['password'], db,
'lasTables/{}_tmp.txt'.format(db))
res, msg = yield self.thread_func.exec_remote_shell(
shell=shell_dump,
host=host_160,
port=port_160,
username=user_160,
password=password_160)
if res and os.path.isfile(tmp_path):
with open(tmp_path, 'r', encoding='utf8') as fp:
lines = fp.readlines()
tables = list()
for line in lines:
if line.find('DROP TABLE IF EXISTS') == -1:
continue
table = re.findall(r'`(\w+)`;', line)
table and tables.append('{}\n'.format(table[0]))
with open(os.path.join(db_path, '{}.txt'.format(db)),
'w') as fp:
fp.writelines(tables)
os.remove(tmp_path)
else:
log.info(msg)
old_time = time.time() - 30 * 24 * 3600
old_path = os.path.join(static_path, 'syncDB',
time.strftime('%Y%m%d', time.gmtime(old_time)))
if os.path.isdir(old_path):
shutil.rmtree(old_path)
class TestRunner(object):
Resolver.configure('tornado.netutil.ThreadedResolver')
def __init__(self, pid=0, jid=0):
self.default_headers = {
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36',
'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4',
'Cookie': ''
}
self.headers = dict()
self.setting = SettingModule()
self.common_func = CommonFunction()
self.option_func = OptionsFunction()
self.pid = pid
self.jid = jid
# 获取请求头
def get_headers(self, headers=''):
flag, headers = self.common_func.convert_to_list_or_dict(
string=headers, s_type='dict')
self.headers = dict()
for key in self.default_headers:
self.headers[key] = self.default_headers[key]
if flag:
for key in headers.keys():
self.headers[key] = headers[key]
else:
headers = headers.splitlines()
for header in headers:
header = header.strip().split(sep=':', maxsplit=1)
if len(header) == 2:
name = header[0].strip()
value = header[1].strip()
self.headers[name] = value
return self.headers
# 加解密操作
@gen.coroutine
def __do_crypt(self, do='encrypt', body=None, crypt_key=''):
crypt_info = yield self.option_func.get_crypt_info(pid=self.pid, do=do)
if not crypt_info:
return body
func = crypt_info.function
try:
if func:
if crypt_key:
flag, body = self.common_func.convert_to_list_or_dict(
string=body, s_type='dict')
if not flag and isinstance(body, str) and re.match(
r'^.*=.*(&.*=.*)*$', body) is not None:
body = self.common_func.url_query_decode(body)
if isinstance(body, dict):
flag = True
if flag:
source = body[crypt_key]
if not isinstance(source, str):
try:
source = json.dumps(source, ensure_ascii=False)
except Exception as e:
log.warning(e)
if isinstance(source, bytes):
source = source.decode('utf8',
errors='ignore')
else:
source = str(source)
body[crypt_key] = func(source, crypt_info['key'],
crypt_info['iv'],
crypt_info['mode'])
else:
if not isinstance(body, str):
try:
body = json.dumps(body, ensure_ascii=False)
except Exception as e:
log.warning(e)
if isinstance(body, bytes):
body = body.decode('utf8', errors='ignore')
else:
body = str(body)
body = func(body, crypt_info['key'], crypt_info['iv'],
crypt_info['mode'])
except Exception as e:
log.warning(e)
return body
# 尝试将请求数据转换成字典
def __parse_body_arguments(self, body):
flag, body = self.common_func.convert_to_list_or_dict(string=body,
s_type='dict')
if not flag and isinstance(body, str) and re.match(
r'^.*=.*(&.*=.*)*$', body) is not None:
body = self.common_func.url_query_decode(body)
if isinstance(body, dict):
flag = True
if not flag:
self.headers = self.headers if len(
self.headers) != 0 else self.default_headers
try:
request_body = dict()
if re.match(r'^.*=.*(&.*=.*)*$', body) is not None:
parse_body_arguments(
content_type=self.headers['Content-Type'],
body=body,
arguments=request_body,
files=request_body,
headers=self.headers)
if len(request_body) > 0:
body = request_body
for key in body:
if isinstance(body[key], list):
body[key] = body[key][0]
body[key] = body[key].decode('utf8', errors='ignore')
flag = True
except Exception as e:
log.warning(e)
flag = False
return flag, body
# 获取请求响应数据
@gen.coroutine
def __get_body(self, body='', do='encrypt', name='', crypt_key=''):
try:
body = ast.literal_eval(body)
except Exception as e:
log.warning(e)
if do == 'encrypt' and crypt_key != '':
flag, body = self.__parse_body_arguments(body)
if isinstance(body, dict):
if name != 'none':
if do == 'encrypt' and crypt_key == '':
body = urlencode(body, encoding='utf8', quote_via=quote)
body = yield self.__do_crypt(do=do,
body=body,
crypt_key=crypt_key)
if isinstance(body, dict):
if do == 'encrypt' and self.headers['Content-Type'].find(
'x-www-form-urlencoded') != -1:
body = urlencode(body, encoding='utf8', quote_via=quote)
else:
body = json.dumps(body, ensure_ascii=False)
else:
if name != 'none':
body = yield self.__do_crypt(do=do, body=body, crypt_key='')
if isinstance(body, dict):
body = json.dumps(body, ensure_ascii=False)
return body
# 解析Host配置
@gen.coroutine
def __parse_host(self, url='', env='none'):
urls = self.common_func.url_split(url=url)
host = urls.host
ips, total = yield self.setting.get_settings_list(pid=self.pid,
s_type='host',
name=host,
pj_status=1,
limit=None)
for row in ips:
if env != 'none':
url = '{}://{}:{}{}'.format(urls.scheme, env, urls.port,
urls.path)
break
elif row.status == 1:
url = '{}://{}:{}{}'.format(urls.scheme, row.value, urls.port,
urls.path)
break
self.headers['Host'] = urls.netloc
return url
# 解析接口返回值全字段检查配置
def __parse_check_key(self, check_key):
keys = []
top = []
rex = re.compile(
r'^\[\w+=\d\|(int|float|num|str|/.*/|date|time|datetime|list|dict)('
r',\w+=\d\|(int|float|num|str|/.*/|date|time|datetime|list|dict))*\]$'
)
for row in check_key.splitlines():
row = row.strip().split(sep='.', maxsplit=1)
if len(row) == 2:
if re.match(rex, row[1]) is not None:
deeps = [row[1]]
else:
deeps = row[1].split(sep='.', maxsplit=1)
if re.match(r'^\[\d+\]$', deeps[0]) is None:
top.append('{}=1|dict'.format(row[0]))
else:
top.append('{}=1|list'.format(row[0]))
deep = '{}.'.format(row[0])
tmp_key = []
for i in range(len(row[1].split('.'))):
if len(deeps) == 2 and re.match(rex, deeps[1]) is not None:
deep += '{}.'.format(deeps[0])
for j in deeps[1][1:-1].split(','):
tmp_key.append(j)
break
elif re.match(rex, deeps[0]) is not None:
for j in deeps[0][1:-1].split(','):
tmp_key.append(j)
break
deep += '{}.'.format(deeps[0])
if len(deeps) == 2:
if re.match(rex, deeps[1]) is not None:
deeps = deeps[1]
else:
deeps = deeps[1].split(sep='.', maxsplit=1)
keys.append(dict(deep=deep[:-1], keys=tmp_key, result=dict()))
else:
top.append(row[0])
keys.append(dict(deep='top', keys=list(set(top)), result=dict()))
return keys
# 返回值全字段检查结果判断
def __check_key_result(self, body, check_key, key, k):
check_key[k]['result'][key[0]] = True
key[1] = key[1].split(sep='|', maxsplit=1)
require = key[1][0]
key_type = key[1][1]
if isinstance(body[key[0]], str):
body[key[0]] = body[key[0]].strip()
if require == '1' and (body[key[0]] == '' or body[key[0]] is None):
check_key[k]['result'][key[0]] = False
elif body[key[0]] != '' and body[key[0]] is not None:
if re.match(r'^/.*/$', key_type):
rex = re.compile(key_type[1:-1])
if re.match(rex, body[key[0]]) is None:
check_key[k]['result'][key[0]] = False
elif key_type == 'int' and not isinstance(body[key[0]], int):
check_key[k]['result'][key[0]] = False
elif key_type == 'float' and not isinstance(body[key[0]], float):
check_key[k]['result'][key[0]] = False
elif key_type == 'num' and not isinstance(
body[key[0]], int) and not isinstance(body[key[0]], float):
check_key[k]['result'][key[0]] = False
elif key_type == 'str' and not isinstance(body[key[0]], str):
check_key[k]['result'][key[0]] = False
elif key_type == 'list' and not isinstance(body[key[0]], list):
check_key[k]['result'][key[0]] = False
elif key_type == 'dict' and not isinstance(body[key[0]], dict):
check_key[k]['result'][key[0]] = False
elif key_type == 'date' and re.match(r'^\d{4}-\d{2}-\d{2}$',
body[key[0]]) is None:
check_key[k]['result'][key[0]] = False
elif key_type == 'time' and re.match(r'^\d{2}:\d{2}:\d{2}$',
body[key[0]]) is None:
check_key[k]['result'][key[0]] = False
elif key_type == 'datetime' and re.match(
r'^\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}$',
body[key[0]]) is None:
check_key[k]['result'][key[0]] = False
if not check_key[k]['result'][key[0]]:
check_key[k]['result']['key_result'] = False
return body, check_key
# 解析响应内容
@gen.coroutine
def __parse_response(self,
response,
name='',
crypt_key='',
checkpoint='',
check_key='',
correlation='',
method='GET',
url=''):
body = response.body if response else ''
request_body = response.request.body if response else ''
if isinstance(body, bytes):
body = body.decode('utf8', errors='ignore')
if isinstance(request_body, bytes):
request_body = request_body.decode('utf8', errors='ignore')
headers_dict = dict(response.headers if response else '')
headers = ''
for key in headers_dict:
headers += '{}: {}\r\n'.format(key, headers_dict[key])
request_headers_dict = dict(
response.request.headers if response else '')
request_headers = ''
for key in request_headers_dict:
request_headers += '{}: {}\r\n'.format(key,
request_headers_dict[key])
if response:
error = response.error if not response.error else str(
response.error)
else:
error = str(httpclient.HTTPError(599, 'Timeout while connecting'))
resp = dict(
body=body,
code=response.code if response else 599,
effective_url=response.effective_url if response else '',
error=error,
headers=headers,
request_headers=request_headers,
request_body=request_body,
reason=response.reason if response else 'Timeout while connecting',
request_time=response.request_time if response else '',
time_info=response.time_info if response else '',
method=method,
url=url)
resp['body_decrypt'] = yield self.__get_body(body,
do='decrypt',
name=name,
crypt_key=crypt_key)
resp['checkpoint'] = []
if checkpoint != '':
if re.match(r'^/.*/$', checkpoint) is not None:
rex = re.compile(checkpoint[1:-1])
if re.findall(rex, resp['body_decrypt']):
resp['checkpoint'].append(
dict(result=True, checkpoint=checkpoint))
else:
log.warning('检查点 {} 检查不通过'.format(checkpoint))
resp['checkpoint'].append(
dict(result=False, checkpoint=checkpoint))
else:
for check in checkpoint.split('|'):
check = check.strip()
if resp['body_decrypt'].find(check) != -1:
resp['checkpoint'].append(
dict(result=True, checkpoint=check))
else:
log.warning('检查点 {} 检查不通过'.format(check))
resp['checkpoint'].append(
dict(result=False, checkpoint=check))
resp['check_key'] = []
if check_key != '':
check_key = self.__parse_check_key(check_key)
for k in range(len(check_key)):
body = resp['body_decrypt']
check_key[k]['result']['key_result'] = True
if check_key[k]['deep'] == 'top':
for key in check_key[k]['keys']:
if re.match(r'^\[\d+\]$', key) is not None:
key = int(key[1:-1])
flag, body = self.common_func.convert_to_list_or_dict(
body, 'list')
if not flag:
log.warning('返回值全字段检查 被检查数据格式不是List, 无法继续')
if body in ['', '[]', '{}', [], {}]:
check_key[k]['result']['ERROR'] = '被检查数据为空'
else:
check_key[k]['result'][
'ERROR'] = '被检查数据类型不是List'
check_key[k]['result']['key_result'] = False
continue
try:
body[key]
except Exception as e:
log.warning(e)
check_key[k]['result'][
'ERROR'] = '被检查数据格式不包含List类型'
check_key[k]['result']['key_result'] = False
continue
else:
flag, body = self.common_func.convert_to_list_or_dict(
body, 'dict')
if not flag and isinstance(body, str) and re.match(
r'^.*=.*(&.*=.*)*$', body) is not None:
body = self.common_func.url_query_decode(body)
if isinstance(body, dict):
flag = True
if not flag:
log.warning('返回值全字段检查 被检查数据格式不是Dict, 无法继续')
if body in ['', '[]', '{}', [], {}]:
check_key[k]['result']['ERROR'] = '被检查数据为空'
else:
check_key[k]['result'][
'ERROR'] = '被检查数据类型不是Dict'
check_key[k]['result']['key_result'] = False
continue
key = key.split(sep='=', maxsplit=1)
if key[0] not in body.keys():
log.warning('返回值全字段检查 {} 检查不通过'.format(
check_key[k]['keys']))
check_key[k]['result'][key[0]] = '字段不存在'
check_key[k]['result']['key_result'] = False
continue
else:
body, check_key = self.__check_key_result(
body=body,
check_key=check_key,
key=key,
k=k)
else:
keys = check_key[k]['keys']
flag, keys = self.common_func.convert_to_list_or_dict(
keys, 'list')
for key in check_key[k]['deep'].split('.'):
if re.match(r'^\[\d+\]$', key) is not None:
key = int(key[1:-1])
flag, body = self.common_func.convert_to_list_or_dict(
body, 'list')
if not flag:
log.warning('返回值全字段检查 被检查数据格式不是List, 无法继续')
if body in ['', '[]', '{}', [], {}]:
check_key[k]['result']['ERROR'] = '被检查数据为空'
else:
check_key[k]['result'][
'ERROR'] = '被检查数据类型不是List'
check_key[k]['result']['key_result'] = False
continue
elif not isinstance(body, dict):
flag, body = self.common_func.convert_to_list_or_dict(
body, 'dict')
if not flag and isinstance(body, str) and re.match(
r'^.*=.*(&.*=.*)*$', body) is not None:
body = self.common_func.url_query_decode(body)
if isinstance(body, dict):
flag = True
if not flag:
log.warning('返回值全字段检查 被检查数据格式不是Dict, 无法继续')
if body in ['', '[]', '{}', [], {}]:
check_key[k]['result']['ERROR'] = '被检查数据为空'
else:
check_key[k]['result'][
'ERROR'] = '被检查数据类型不是Dict'
check_key[k]['result']['key_result'] = False
continue
try:
body = body[key]
except Exception as e:
log.warning(e)
check_key[k]['result'][
'ERROR'] = '被检查数据格式不包含List类型'
check_key[k]['result']['key_result'] = False
continue
flag, body = self.common_func.convert_to_list_or_dict(
body, 'dict')
if not flag and isinstance(body, str) and re.match(
r'^.*=.*(&.*=.*)*$', body) is not None:
body = self.common_func.url_query_decode(body)
if isinstance(body, dict):
flag = True
if not flag:
log.warning('返回值全字段检查 被检查数据格式不是Dict, 无法继续')
if body in ['', '[]', '{}', [], {}]:
check_key[k]['result']['ERROR'] = '被检查数据为空'
else:
check_key[k]['result']['ERROR'] = '被检查数据类型不是Dict'
check_key[k]['result']['key_result'] = False
continue
for key in keys:
key = key.split(sep='=', maxsplit=1)
if key[0] not in body.keys():
log.warning('返回值全字段检查 {} 检查不通过'.format(
check_key[k]['keys']))
check_key[k]['result'][key[0]] = '字段不存在'
check_key[k]['result']['key_result'] = False
continue
else:
body, check_key = self.__check_key_result(
body=body, check_key=check_key, key=key, k=k)
resp['check_key'] = check_key
resp['correlation'] = dict()
if response and correlation != '':
correlations = correlation.split('|')
correlation = dict()
for corr in correlations:
body = resp['body_decrypt']
cor = corr.split(sep='=', maxsplit=1)
key = cor[0].strip()
c_type = 'string'
words = cor[1].strip()
if re.match(r'^int\(.+\)$', words) is not None:
c_type = 1
words = words[4:-1]
elif re.match(r'^float\(.+\)$', words) is not None:
c_type = 1.00
words = words[6:-1]
word = words.split('.')
correlation[key] = word
for k in word:
if k == 'response_headers' and len(word) != 1:
header = words.split(sep='.', maxsplit=1)
if len(header) == 2:
header_key = header[1]
if re.match(r'^/.*/$', header_key) is not None:
rex = re.compile(header_key[1:-1])
if isinstance(resp['headers'], bytes):
resp['headers'] = resp['headers'].decode(
'utf8', errors='ignore')
result = re.findall(rex, resp['headers'])
if result:
if isinstance(result[0], tuple):
body = result[0][0]
else:
body = ''
for row in result:
row = row if row != '' else '\n'
body += row
else:
body = ''
else:
body = response.headers.get(header_key)
break
if k == 'response_body' and len(word) == 1:
if isinstance(resp['body'], bytes):
resp['body'] = resp['body'].decode('utf8',
errors='ignore')
body = escape.xhtml_escape(resp['body'])
break
if re.match(r'^/.*/$', words) is not None:
rex = re.compile(words[1:-1])
if isinstance(resp['body'], bytes):
resp['body'] = resp['body'].decode('utf8',
errors='ignore')
result = re.findall(rex, resp['body'])
if result:
if isinstance(result[0], tuple):
body = escape.xhtml_escape(result[0][0])
else:
body = escape.xhtml_escape(result[0])
else:
body = ''
break
if re.match(r'^\[\d+\]$', k) is not None:
k = int(k[1:-1])
flag, body = self.common_func.convert_to_list_or_dict(
body, 'list')
if not flag:
log.warning('响应数据格式不是List, 无法继续')
body = ''
break
elif not isinstance(body, dict):
flag, body = self.common_func.convert_to_list_or_dict(
body, 'dict')
if not flag and isinstance(body, str) and re.match(
r'^.*=.*(&.*=.*)*$', body) is not None:
body = self.common_func.url_query_decode(body)
if isinstance(body, dict):
flag = True
if not flag:
log.warning('响应数据格式不是Dict, 无法继续')
body = ''
break
try:
body = body[k]
except Exception as e:
log.warning(e)
body = ''
break
correlation[key] = body
try:
if isinstance(c_type, int):
correlation[key] = int(
float(re.sub(r'[^\d+\.]', '', body)))
elif isinstance(c_type, float):
correlation[key] = float(re.sub(r'[^\d+\.]', '', body))
except Exception as e:
log.warning(e)
resp['correlation'] = correlation
flag = True
if resp['error'] is not None and resp['code'] != 302 and resp[
'code'] != 301:
flag = False
if resp['check_key']:
for line in resp['check_key']:
if not line['result']['key_result']:
flag = False
break
if resp['checkpoint']:
for line in resp['checkpoint']:
if not line['result']:
flag = False
break
resp['test_result'] = flag
return resp
# 解析自定义参数配置
@gen.coroutine
def __parse_custom_param(self, headers, body, correlation_result={}):
correlation_result = dict(self.common_func.default_param(),
**correlation_result)
params = yield self.option_func.get_custom_param(
pid=self.pid, correlation=correlation_result)
if isinstance(headers, bytes):
headers = headers.decode('utf8', errors='ignore')
if isinstance(body, bytes):
body = body.decode('utf8', errors='ignore')
for key in correlation_result:
if not isinstance(correlation_result[key], str):
correlation_result[key] = str(correlation_result[key])
if headers.find(key) != -1:
headers = headers.replace(key, correlation_result[key])
if body.find(key) != -1:
body = body.replace(key, correlation_result[key])
for param in params:
if headers.find('{%s}' % param['name']) != -1:
if param['type'] == 'Function':
func = param['function']
flag, body_dict = self.__parse_body_arguments(body)
encrypt = yield self.option_func.get_crypt_info(self.pid)
headers = headers.replace('{%s}' % param['name'],
func(body_dict, params, encrypt))
else:
headers = headers.replace('{%s}' % param['name'],
param['value'])
if body.find('{%s}' % param['name']) != -1:
if param['type'] == 'Function':
func = param['function']
flag, body_dict = self.__parse_body_arguments(body)
encrypt = yield self.option_func.get_crypt_info(self.pid)
body = body.replace('{%s}' % param['name'],
func(body_dict, params, encrypt))
else:
body = body.replace('{%s}' % param['name'], param['value'])
return headers, body
# 请求操作
@gen.coroutine
def __request_url(self,
url='',
env='none',
method='GET',
body='',
follow_redirects=True):
test_client = httpclient.AsyncHTTPClient(max_clients=100)
argv = dict(method=method,
headers=self.headers,
follow_redirects=False,
request_timeout=600,
validate_cert=False,
raise_error=False)
url = yield self.__parse_host(url=url, env=env)
if method == 'GET':
url = '{}?{}'.format(url, body)
elif method == 'POST':
argv['body'] = body
try:
log.info('开始请求接口 {}'.format(url))
response = yield test_client.fetch(url, **argv)
except httpclient.HTTPError as e:
response = e.response
log.warning('请求接口 {} 异常# {}'.format(
url, str(response.error if response else e)))
# test_client.close()
if response and response.code in [301, 302]:
for cookie in response.headers.get_list('Set-Cookie'):
self.headers['Cookie'] += '{};'.format(cookie)
url = response.headers.get('Location')
log.info('{} {} {}'.format(response.code, response.reason, url))
if response.reason.find('Moved') >= 0:
response = yield self.__request_url(
url=url,
env=env,
method=method,
body=body,
follow_redirects=follow_redirects)
elif follow_redirects:
response = yield self.__request_url(
url=url,
env=env,
method='GET',
follow_redirects=follow_redirects)
log.info('结束请求接口 {}'.format(url))
return response
# 生成测试报告
@gen.coroutine
def __gen_report(self, job_name, test_suites, start_time, end_time):
setting = yield self.setting.get_settings_by_range(pid=self.pid,
s_type='log',
start=start_time,
end=end_time,
sort=self.jid)
if setting:
elapsed_time = end_time - start_time
start_time = time.strftime(
'%Y-%m-%d %H:%M:%S', time.gmtime(float(start_time) + 3600 * 8))
end_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.gmtime(float(end_time) + 3600 * 8))
url, total = yield self.setting.get_settings_list(pid=self.pid,
s_type='url',
limit=None)
overview = dict(name=job_name,
start_time=start_time,
end_time=end_time,
elapsed_time=elapsed_time,
total=total,
total_test=len(test_suites),
success_test=0,
fail_test=0,
success_rate='0.0 %',
report_time=time.strftime('%Y-%m-%d %H:%M:%S'))
for suite in test_suites:
suite['total_test'] = len(suite['cases'])
suite['success_test'] = 0
suite['fail_test'] = 0
suite['report'] = []
suite['result'] = True
for row in setting:
res = json.loads(row.value)
for suite in test_suites:
if suite['suite_id'] == row.status:
if res['test_result']:
suite['success_test'] += 1
else:
suite['fail_test'] += 1
suite['result'] = False
suite['report'].append(res)
for suite in test_suites:
suite['result'] = suite['result'] if suite[
'success_test'] + suite['fail_test'] == suite[
'total_test'] else False
if suite['result']:
overview['success_test'] += 1
else:
overview['fail_test'] += 1
overview['success_rate'] = '{:.2f} %'.format(
overview['success_test'] / overview['total_test'] * 100)
result = dict(overview=overview, report=test_suites)
report_id, msg = yield self.setting.add_setting(
pid=self.pid,
s_type='report',
sort=self.jid,
name=time.time(),
value=json.dumps(result, ensure_ascii=False))
return report_id
else:
return False
# 执行单接口测试
@gen.coroutine
def run_test(self,
url='',
label='',
comment='',
method='GET',
headers='',
body='',
crypt='none',
encrypt_content='',
no_test=False,
check_key='',
decrypt_content='',
checkpoint='',
env='none',
correlation='',
correlation_result={},
follow_redirects=True):
headers, body = yield self.__parse_custom_param(
headers=headers, body=body, correlation_result=correlation_result)
self.get_headers(headers)
request_body = yield self.__get_body(body=body,
do='encrypt',
name=crypt,
crypt_key=encrypt_content)
if no_test:
return request_body
resp = yield self.__request_url(url=url,
env=env,
method=method,
body=request_body,
follow_redirects=follow_redirects)
response = yield self.__parse_response(response=resp,
name=crypt,
crypt_key=decrypt_content,
checkpoint=checkpoint,
check_key=check_key,
correlation=correlation,
method=method,
url=url)
response['label'] = label
response['comment'] = comment
if not resp:
response['request_body'] = request_body
for key in self.headers:
response['request_headers'] += '{}: {}\r\n'.format(
key, self.headers[key])
elif method == 'GET':
response['request_body'] = request_body
log.info('响应返回 {}'.format(json.dumps(response, ensure_ascii=False)))
return response
# 执行多接口测试
@gen.coroutine
def __run_all_test(self, job_name, test_suites):
if not test_suites:
return False
start_time = time.time()
correlation_result = dict()
for test in test_suites:
cases = yield self.setting.get_settings_by_ids(test['cases'])
for url_info in cases:
try:
url_info = json.loads(url_info.value)
resp = yield self.run_test(
correlation_result=correlation_result, **url_info)
yield self.setting.add_setting(pid=self.pid,
s_type='log',
name=time.time(),
sort=self.jid,
value=json.dumps(
resp,
ensure_ascii=False),
status=test['suite_id'])
correlation_result = dict(correlation_result,
**resp['correlation'])
except Exception as e:
log.error(e)
end_time = time.time()
report_id = yield self.__gen_report(job_name, test_suites, start_time,
end_time)
return report_id
# 执行排队任务
@gen.coroutine
def run_job(self):
job = yield self.setting.get_setting_by_id(sid=self.jid)
if job:
start_time = time.time()
job_value = json.loads(job.value)
start_strftime = time.strftime(
'%Y-%m-%d %H:%M:%S', time.gmtime(float(start_time) + 3600 * 8))
job_value['overview']['start_time'] = start_strftime
yield self.setting.edit_setting(sid=job.id,
status=2,
value=json.dumps(
job_value, ensure_ascii=False))
test_suites = []
suites = yield self.setting.get_settings_by_ids(
job_value['testsuite'])
for suite in suites:
tests = dict(suite_id=suite.id,
suite_name=suite.name,
cases=json.loads(suite.value)['cases'])
test_suites.append(tests)
result = yield self.__run_all_test(job_value['name'], test_suites)
if result:
status = 3
job_value['lastreport'] = result
else:
status = 5
end_time = time.time()
elapsed_time = end_time - start_time
end_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.gmtime(float(end_time) + 3600 * 8))
job_value['overview']['end_time'] = end_time
job_value['overview']['elapsed_time'] = elapsed_time
name = float(job.name)
if job_value['overview']['cycle_time'] != 0:
while name < time.time():
name += job_value['overview']['cycle_time']
job_value['overview']['plan_time'] = time.strftime(
'%Y-%m-%d %H:%M:%S', time.gmtime(name + 3600 * 8))
status = 0
yield self.setting.edit_setting(sid=job.id,
status=status,
name=name,
value=json.dumps(
job_value, ensure_ascii=False))
yield Mail().send_html_report(result)
def __get_links(self, url, user_agent='', cookie='', check_all=False):
response = yield self.__request_url(url, user_agent, cookie)
common_func = CommonFunction()
url_split = common_func.url_split(url)
host = url_split.host
scheme = url_split.scheme
netloc = '{}://{}'.format(scheme, url_split.netloc)
links = []
if response and response.body:
body = response.body if not isinstance(
response.body, bytes) else response.body.decode(
'utf8', errors='ignore')
body = body if isinstance(body, str) else str(body)
link_a = re.findall(
r'<a.*?href=[\'"](.*?)[\'"].*?title=[\'"](.*?)[\'"].*?>(.*?)</a>',
body)
link_b = re.findall(
r'<a.*?title=[\'"](.*?)[\'"].*?href=[\'"](.*?)[\'"].*?>(.*?)</a>',
body)
link_c = re.findall(r'<a.*?href=[\'"](.*?)[\'"].*?>(.*?)</a>',
body)
link_d = re.findall(r'data-href=[\'"](.*?)[\'"]', body)
link_e = re.findall(r'data-url=[\'"](.*?)[\'"]', body)
for link in link_a:
href = link[0].strip()
if re.match(r'^(http|https):', href) is not None:
pass
elif re.match(r'^//', href) is not None:
href = scheme + href
elif re.match(r'^/', href) is not None:
href = netloc + href
elif re.match(r'^(javascript|#)', href.lower()) is not None:
continue
else:
href = '{}/{}'.format(url, href)
title = link[1]
text = link[2] if link[2] != '' else link[1]
log.info('获取到链接: {} {}'.format(text, href))
link = dict(href=href, title=title, text=text)
if link not in links:
links.append(link)
for link in link_b:
href = link[1].strip()
if re.match(r'^(http|https):', href) is not None:
pass
elif re.match(r'^//', href) is not None:
href = scheme + href
elif re.match(r'^/', href) is not None:
href = netloc + href
elif re.match(r'^(javascript|#)', href.lower()) is not None:
continue
else:
href = '{}/{}'.format(url, href)
title = link[0]
text = link[2] if link[2] != '' else link[0]
log.info('获取到链接: {} {}'.format(text, href))
link = dict(href=href, title=title, text=text)
if link not in links:
links.append(link)
for link in link_c:
href = link[0].strip()
if re.match(r'^(http|https):', href) is not None:
pass
elif re.match(r'^//', href) is not None:
href = scheme + href
elif re.match(r'^/', href) is not None:
href = netloc + href
elif re.match(r'^(javascript|#)', href.lower()) is not None:
continue
else:
href = '{}/{}'.format(url, href)
title = ''
text = link[1]
log.info('获取到链接: {} {}'.format(text, href))
link = dict(href=href, title=title, text=text)
if link not in links:
links.append(link)
for link in link_d + link_e:
href = link[0].strip()
if re.match(r'^(http|https):', href) is not None:
pass
elif re.match(r'^//', href) is not None:
href = scheme + href
elif re.match(r'^/', href) is not None:
href = netloc + href
elif re.match(r'^(javascript|#)', href.lower()) is not None:
continue
else:
href = '{}/{}'.format(url, href)
title = ''
text = ''
log.info('获取到链接: {} {}'.format(text, href))
link = dict(href=href, title=title, text=text)
if link not in links:
links.append(link)
next_links = []
if check_all:
for link in links:
cur_host = common_func.url_split(link['href']).host
if cur_host == host and link['href'] not in next_links:
next_links.append(link['href'])
return links, next_links, response
class BaseHandler(RequestHandler):
"""
后台管理父类，后台相关handlers继承本类
"""
current_user = None
# 初始化方法
@gen.coroutine
def prepare(self):
self.user = UserModule()
self.project = ProjectModule()
self.setting = SettingModule()
self.option = OptionModule()
self.msg = MessagesModule()
self.statistics = StatisticsModule()
self.common_func = CommonFunction()
self.option_func = OptionsFunction()
self.thread_func = ThreadFunction()
# 获取当前用户信息
@gen.coroutine
def get_current_user_async(self):
user = self.get_secure_cookie('OSTSESSION', None)
try:
token = self.request.headers.get('Token')
if token and not user:
user = base64.b64decode(
base64.b16decode(
token.strip().encode('utf8'))).decode('utf8')
except Exception as e:
log.error(e)
if user is not None:
if isinstance(user, bytes):
user = user.decode('utf8', errors='ignore')
user = yield self.user.get_user_info(email_or_username=user)
if not user:
self.clear_cookie('OSTSESSION')
else:
self.set_secure_cookie('OSTSESSION',
user.email,
expires=time.time() + 1800)
return user
# 返回json格式字符串
@gen.coroutine
def write_json(self, msg):
if isinstance(msg, dict):
msg = json.dumps(msg, ensure_ascii=False)
self.set_header("Content-Type", "application/json; charset=UTF-8")
self.finish(msg)
# 获取json格式请求参数
@gen.coroutine
def get_request_body_to_json(self):
params = self.request.body
if isinstance(params, bytes):
params = params.decode('utf8')
try:
params = munchify(json.loads(params))
except Exception as e:
log.error(e)
params = None
return params
# 登录或注册
@gen.coroutine
def _login_or_register(self,
username='',
email='',
name='',
department='',
password='******'):
self.clear_cookie('OSTSESSION')
if (username == '' and email
== '') or password == '' or name == '' or department == '':
return False, '[自动登录]请求参数不对!'
username = email if email != '' else username
user = yield self.user.get_user_info(username)
if not user:
if not self.common_func.check_string(
username, 'email') and not self.common_func.check_string(
password, 'password'):
return False, '[自动注册]邮箱或密码格式不对!'
else:
user, msg = yield self.user.register_user(
email=email,
password=password,
real_name=name,
status=2,
profile=dict(
department=department,
avatar=
'https://gw.alipayobjects.com/zos/rmsportal/BiazfanxmamNRoxxVxka.png',
position=''))
if user:
self.set_secure_cookie('OSTSESSION',
email,
expires=time.time() + 1800)
content = dict(group=dict(name='', link=''),
project=dict(name='', link=''),
template='同学, 欢迎加入组织!')
self.msg.add_message(user_id=user.id,
m_type='active',
content=content)
return user, '[自动注册]{}'.format(msg)
elif user.password == self.common_func.encode_password(
password) and user.status != 0:
self.user.edit_user(
username, last_login_time=time.strftime('%Y-%m-%d %H:%M:%S'))
self.set_secure_cookie('OSTSESSION',
email,
expires=time.time() + 1800)
content = dict(group=dict(name='', link=''),
project=dict(name='', link=''),
template='同学, 欢迎回来!')
self.msg.add_message(user_id=user.id,
m_type='active',
content=content)
return user, '[自动登录]操作成功!'
else:
return False, '[自动登录]登录密码不对或账户被禁用!'
class UserModule(object):
"""
用户表相关操作
"""
def __init__(self):
self.common_func = CommonFunction()
# 获取用户信息
@gen.coroutine
def get_user_info(self, email_or_username, status=None):
sql = "SELECT * FROM t_users u WHERE (u.email=%(user)s OR u.username=%(user)s)"
param = dict(user=email_or_username)
if status is not None:
sql += ' AND u.status=%(status)s'
param['status'] = status
cursor = yield pool.execute(sql, param)
result = cursor.fetchone()
cursor.close()
return munchify(result)
# 通过user_id获取用户信息
@gen.coroutine
def get_users_info_by_ids(self, ids, status=None):
if ids and isinstance(ids, list):
param = dict()
ins = ''
for i in range(len(ids)):
param['id{}'.format(i)] = ids[i]
ins += '%(id{})s,'.format(i)
sql = 'SELECT * FROM t_users u WHERE u.id in ({})'.format(ins[:-1])
if status is not None:
sql += ' AND status=%(status)s'
param['status'] = status
sql += ' ORDER BY u.role'
cursor = yield pool.execute(sql, param)
result = cursor.fetchall()
cursor.close()
return munchify(result)
else:
return []
# 获取用户列表
@gen.coroutine
def get_users_list(self, page=1, limit=10, status=None):
sql = 'SELECT * FROM t_users u'
sql_count = 'SELECT COUNT(*) count FROM t_users u'
if status is not None:
sql += ' WHERE u.status=%(status)s'
sql_count += ' WHERE u.status=%(status)s'
sql += ' ORDER BY u.role'
if limit is not None:
offset = (page - 1) * limit
sql += ' LIMIT {},{}'.format(offset, limit)
cursor = yield pool.execute(sql, dict(status=status))
result = cursor.fetchall()
cursor = yield pool.execute(sql_count, dict(status=status))
total = cursor.fetchone()
cursor.close()
return munchify(result), munchify(total).count
# 注册用户
@gen.coroutine
def register_user(self, email, password):
register_time = time.strftime('%Y-%m-%d %H:%M:%S')
password = self.common_func.encode_password(password)
cursor = yield pool.execute('SELECT COUNT(*) count FROM t_users')
total = munchify(cursor.fetchone())
if total.count == 0:
role = 0
else:
role = 2
username = '******'.format(
email.split('@')[0], str(int(time.time() * 1000)))
sql = """
INSERT INTO t_users (username, email, password, registerTime, lastLoginTime, role)
VALUE(%(username)s, %(email)s, %(password)s, %(registerTime)s, %(lastLoginTime)s, %(role)s)
"""
user = yield self.get_user_info(email_or_username=email)
if not user:
with (yield pool.Connection()) as conn:
with conn.cursor() as cursor:
try:
yield cursor.execute(
sql,
dict(username=username,
email=email,
password=password,
registerTime=register_time,
lastLoginTime=register_time,
role=role))
except pymysql.Error as e:
yield conn.rollback()
log.error('注册用户失败#{}'.format(e))
flag, msg = False, '注册用户失败#{}'.format(e)
else:
yield conn.commit()
log.info('注册用户成功')
flag, msg = cursor.lastrowid, '注册用户成功'
else:
log.error('该用户已存在, 请更换注册邮箱')
flag, msg = False, '该用户已存在, 请更换注册邮箱'
return flag, msg
# 编辑用户
@gen.coroutine
def edit_user(self,
email,
password=None,
username=None,
realname=None,
last_login_time=None,
role=None,
status=None):
user = yield self.get_user_info(email)
if user:
update = []
param = dict(email=user.email)
if password is not None:
update.append("password=%(password)s")
param['password'] = self.common_func.encode_password(password)
if username is not None:
sql = "SELECT username FROM t_users u WHERE u.email != %(email)s AND u.username = %(username)s"
param['username'] = username
cursor = yield pool.execute(sql, param)
user_info = cursor.fetchone()
if user_info:
log.error('用户名 {} 已存在'.format(username))
return False, '用户名 {} 已存在'.format(username)
else:
update.append("username=%(username)s")
if realname is not None:
update.append("realname=%(realname)s")
param['realname'] = realname
if last_login_time is not None:
update.append("lastLoginTime=%(lastLoginTime)s")
param['lastLoginTime'] = last_login_time
if role is not None:
update.append('role=%(role)s')
param['role'] = role
if status is not None:
update.append('status=%(status)s')
param['status'] = status
if update:
sql = "UPDATE t_users SET {} WHERE email=%(email)s".format(
', '.join(update))
tx = yield pool.begin()
try:
yield tx.execute(sql, param)
except pymysql.Error as e:
yield tx.rollback()
log.error('编辑用户失败#{}'.format(e))
flag, msg = False, '用户 {} 资料修改失败'.format(email)
else:
yield tx.commit()
log.info('用户 {} 资料修改成功'.format(email))
flag, msg = True, '用户 {} 资料修改成功'.format(email)
return flag, msg
else:
log.error('没有可更新的项')
return False, '没有可更新的项'
else:
log.error('没有可编辑的用户#{}'.format(email))
return False, '没有可编辑的用户#{}'.format(email)
def post(self, op='option', do='save'):
if self.current_user.role == 2:
msg = dict(result=False, msg='请使用管理员登录后再进行操作!')
yield self.return_json(msg)
return
if op == 'option':
company = self.get_argument('company', default='')
page_limit = self.get_argument('page_limit', default='')
email_ext = self.get_argument('email_ext', default='')
smtp_host = self.get_argument('smtp_host', default='')
smtp_port = self.get_argument('smtp_port', default='')
use_ssl = self.get_argument('use_ssl', default='off')
smtp_user = self.get_argument('smtp_user', default='')
smtp_password = self.get_argument('smtp_password', default='')
mail_from = self.get_argument('mail_from', default='')
report_url = self.get_argument('report_url', default='')
mail_report = self.get_argument('mail_report', default='off')
common_func = CommonFunction()
if smtp_user != '' and not common_func.check_string(
smtp_user, 'email'):
msg = dict(result=False, msg='【SMTP 登录用户】必须是Email格式')
yield self.return_json(msg)
return
if mail_from != '' and not common_func.check_string(
mail_from, 'email'):
msg = dict(result=False, msg='【发件人显示为】必须是Email格式')
yield self.return_json(msg)
return
if do == 'test':
send_to = self.get_argument('send_to', default='')
if '' in (send_to, smtp_port, smtp_host, smtp_user,
smtp_password):
msg = dict(result=False, msg='所有配置不能为空, 请检查配置是否正确')
yield self.return_json(msg)
return
if not common_func.check_string(send_to, 'email'):
msg = dict(result=False, msg='【配置测试】地址必须是Email格式')
yield self.return_json(msg)
return
mail = Mail(smtp_server=smtp_host,
smtp_port=smtp_port,
use_ssl=use_ssl,
smtp_user=smtp_user,
smtp_password=smtp_password,
mail_from=mail_from)
message = '<p>[{}]邮件配置测试, 收到此邮件说明配置正确。</p>'.format(
self.company)
result, msg = yield mail.send_mail(
subject='[{}][系统测试邮件]'.format(self.company),
message=message,
to=[send_to])
if result:
msg = dict(result=True, msg='邮件发送成功')
else:
msg = dict(result=False, msg='邮件发送失败, {}'.format(msg))
else:
if page_limit == '':
page_limit = 0
else:
page_limit = int(page_limit)
if len(company) < 2 or len(company) > 60:
msg = dict(result=False, msg='【公司名称】格式不正确')
yield self.return_json(msg)
return
if page_limit < 10 or page_limit > 100 or page_limit % 10 != 0:
msg = dict(result=False, msg='【分页数目】必须为10的倍数且是10与100之间的整数')
yield self.return_json(msg)
return
if report_url != '' and not common_func.check_string(
report_url, 'url'):
msg = dict(result=False, msg='【邮件报告链接域名】格式不对, 请检查')
yield self.return_json(msg)
return
edit_options = dict(company=company,
page_limit=page_limit,
email_ext=email_ext,
report_url=report_url,
smtp_host=smtp_host,
smtp_port=smtp_port,
smtp_user=smtp_user,
mail_report=mail_report,
smtp_password=smtp_password,
use_ssl=use_ssl,
mail_from=mail_from)
options = yield self.option.get_options_list()
flag = True
msg = ''
options_name = []
for option in options:
options_name.append(option.name)
if options_name:
for option in edit_options:
if option in options_name:
result, msg = yield self.option.edit_option(
name=option, value=edit_options[option])
else:
result, msg = yield self.option.add_option(
name=option, value=edit_options[option])
if not result:
flag = False
break
else:
for option in edit_options:
result, msg = yield self.option.add_option(
name=option, value=edit_options[option])
if not result:
flag = False
break
if flag:
msg = dict(result=True, msg='更新系统配置成功')
else:
msg = dict(result=False, msg=msg)
yield self.return_json(msg)
elif op == 'users':
if do == 'add':
email = self.get_argument('email', default='')
if self.common_func.check_string(email, 'email'):
user = yield self.user.get_user_info(email)
if user is None:
self.user.register_user(email, '123456')
self.redirect('/admin/manage/users')
return
uid = self.get_argument('id', default='')
user = yield self.user.get_users_info_by_ids(ids=[uid])
if not user:
msg = dict(result=False, msg='所编辑的用户不存在')
yield self.return_json(msg)
return
if do == 'status':
status = int(self.get_argument('status', default=0))
result, msg = yield self.user.edit_user(email=user[0].email,
status=status)
if result:
msg = dict(result=True, msg=msg)
else:
msg = dict(result=False, msg=msg)
yield self.return_json(msg)
return
elif do == 'role':
role = int(self.get_argument('role', default=0))
result, msg = yield self.user.edit_user(email=user[0].email,
role=role)
if result:
msg = dict(result=True, msg=msg)
else:
msg = dict(result=False, msg=msg)
yield self.return_json(msg)
return
elif do == 'reset':
result, msg = yield self.user.edit_user(email=user[0].email,
password='******')
if result:
msg = dict(result=True,
msg='用户 {} 的密码已重置为 123456'.format(
user[0].email))
else:
msg = dict(result=False, msg=msg)
yield self.return_json(msg)
return
elif op == 'logs':
yield self.setting.delete_settings_list(s_type='log')
self.redirect('/admin/manage/logs')