def generate_report(taskid, flowid, jtl_url):
"""
执行一个任务，并汇总生成的jtl文件，执行任务前会先插入一条任务的flow流水记录
:param cmds: 一个字段，key是jtl文件路径，value是jmx执行命令
:return:
"""
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jmeter_platform.settings")
django.setup()
from reports.models import Reports
report_output = settings.OUTPUT_URL + Tools.filename(jtl_url)
# 如果存在，则删除之前的，目录重新生成
if os.path.exists(report_output):
shutil.rmtree(report_output)
cmd = f"{settings.JMETER} -g {jtl_url} -e -o {report_output}"
logger.info(f'开始生成报告:{cmd}')
os.system(cmd)
if os.path.exists(report_output):
report = Reports(task_id=taskid,
flow_id=flowid,
report_url=report_output)
report.save()
logger.info(f'[{taskid}:{flowid}]报告数据插入数据成功')
else:
logger.error(f'[{taskid}:{flowid}]生成报告失败')
def post(self, request, taskid):
# 判断任务是否存在或者是否绑定了jmx
task = TasksDetails.objects.filter(task_id=taskid)
if not task:
return APIRsp(code=400,
msg='该任务不存在或未绑定jmx！',
status=status.HTTP_200_OK)
# 判断是否存在运行中的流水任务
running = TaskFlow.objects.filter(task_id=taskid, task_status=0)
if running:
return APIRsp(code=400,
msg='存在运行中的流水任务，请稍后重试！',
status=status.HTTP_200_OK)
try:
jmxs_id = TasksDetails.objects.values('jmx').filter(task_id=taskid)
rsp = Jmxs.objects.values('id', 'jmx').filter(id__in=jmxs_id)
jmxs = []
for jmx in rsp:
jmxs.append(jmx)
except Exception as e:
logger.exception(f'获取任务信息失败\n{e}')
return APIRsp(code=500,
msg='获取任务信息失败',
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
task_flow_str = Tools.random_str()
celery_task_id = run_task.delay(taskid, task_flow_str, jmxs)
url = settings.GRAFANA_ENNDPOINT + f'&var-application={task_flow_str}'
flow = TaskFlow(task_id=taskid,
celery_task_id=celery_task_id,
randomstr=task_flow_str,
task_flow_url=url)
flow.save()
return APIRsp()
def post(self, request, userid, jmxid):
# 判断任务是否存在或者是否绑定了jmx
task = TasksDetails.objects.filter(jmx_id=jmxid, task__task_type=1)
if not task:
try:
task_name = Jmxs.objects.get(id=jmxid).jmx_alias
except:
return APIRsp(code=400, msg='无效的jmxid')
# 创建任务
t = Tasks(task_name=task_name, task_type=1, add_user_id=userid)
t.save()
taskid = t.id
# 将jmx加入任务详情
td = TasksDetails(task_id=taskid, jmx_id=jmxid)
td.save()
else:
taskid = task[0].task_id
running = TaskFlow.objects.filter(task_id=taskid, task_status=0)
if running:
return APIRsp(code=400,
msg='存在运行中的流水任务，请稍后重试！',
status=status.HTTP_200_OK)
try:
jmxs_id = TasksDetails.objects.values('jmx').filter(task_id=taskid)
rsp = Jmxs.objects.values('id', 'jmx').filter(id__in=jmxs_id)
jmxs = []
for jmx in rsp:
jmxs.append(jmx)
except Exception as e:
logger.exception(f'获取任务信息失败\n{e}')
return APIRsp(code=500,
msg='获取任务信息失败',
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
task_flow_str = Tools.random_str()
celery_task_id = run_task.delay(taskid, task_flow_str, jmxs)
url = settings.GRAFANA_ENNDPOINT + f'&var-application={task_flow_str}'
flow = TaskFlow(task_id=taskid,
celery_task_id=celery_task_id,
randomstr=task_flow_str,
task_flow_url=url)
flow.save()
return APIRsp()
def run_task(taskid, task_flow_str, jmxs):
"""
执行一个任务，并汇总生成的jtl文件，执行任务前会先插入一条任务的flow流水记录
:param cmds: 一个字段，key是jtl文件路径，value是jmx执行命令
:return:
"""
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jmeter_platform.settings")
django.setup()
from jtls.models import JtlsSummary
from tasks.models import TaskFlow, FlowTaskAggregateReport, RspResult, PngResult
from jmxs.models import JmxThreadGroup, Jmxs
from params.models import UsersParams
try:
cmds = []
save_path_dict = {}
sampler_id_name_dict = {}
temp_dir = settings.TEMP_URL + task_flow_str
jtl = settings.JTL_URL + task_flow_str + '.jtl'
pt = re.compile('{{(.*?)}}')
logger.debug(f'创建临时目录{temp_dir}')
os.makedirs(temp_dir)
logger.debug('将jmx复制到临时目录下')
for sjmx in jmxs:
jmx = sjmx['jmx']
jmx_id = sjmx['id']
thread_base_info = Jmxs.objects.get(id=jmx_id).thread_base_info
num_threads = json.loads(thread_base_info)['num_threads']
samplers_info = JmxThreadGroup.objects.values(
'id', 'child_info').filter(jmx_id=jmx_id, child_type="sampler")
jmx_name = Tools.filename(jmx)
shutil.copy(jmx, temp_dir)
# 在这里查找替换变量{{}}
temp_jmx = temp_dir + os.sep + jmx_name + '.temp.jmx'
temp_temp_jmx = temp_dir + os.sep + jmx_name + '.jmx'
# 替换全局变量参数
logger.debug('替换全局参数')
with open(temp_jmx, 'w') as write:
with open(temp_temp_jmx, 'r') as read:
content = read.readlines()
for line in content:
param_name_list = re.findall(pt, line)
for param_name in param_name_list:
value = UsersParams.objects.get(
param_name=param_name).param_value
logger.debug(f'全局参数:{param_name}\n{value}')
line = line.replace('{{%s}}' % param_name,
str(value))
write.write(line)
os.remove(temp_temp_jmx)
ModifyJMX(temp_jmx).add_backendListener(
influxdb_url=settings.INFLUXDB_URL,
application_name=task_flow_str)
for sampler in samplers_info:
sampler_id = sampler['id']
sampler_info = json.loads(sampler['child_info'])
sampler_xpath = sampler_info['xpath']
# 提取ID和实际名称的对于关系
sampler_id_name_dict[sampler_xpath.split('@testname="')
[1].split('"]')[0]] = sampler_id
# 创建保存取样器响应的目录
save_path = temp_dir + os.sep + str(sampler_id)
os.makedirs(save_path)
save_path_dict[sampler_id] = save_path
if str(num_threads) == '1':
# 线程是1时，保存错误和异常日志
ModifyJMX(temp_jmx).save_rsp_data(sampler_xpath, save_path)
else:
# 线程不为1时，只保存错误日志
ModifyJMX(temp_jmx).save_rsp_data(sampler_xpath,
save_path,
errorsonly=True)
cmd = f"{settings.JMETER} -n -t {temp_jmx} -l {jtl}"
cmds.append(cmd)
logger.debug('开始执行jmx')
for cmd in cmds:
os.system(cmd)
try:
flow_id = TaskFlow.objects.values('id').get(
randomstr=task_flow_str)['id']
js = JtlsSummary(task_id=taskid, flow_id=flow_id, jtl_url=jtl)
js.save()
logger.debug('jtl信息入库成功')
except:
logger.error('jtl信息入库失败')
os.remove(jtl)
raise
logger.debug('将jtl文件转换为csv文件')
summary_csv = settings.TEMP_URL + task_flow_str + os.sep + 'temp.csv'
rt_png = settings.PIC_URL + f"{task_flow_str}_ResponseTimesOverTime.png"
tps_png = settings.PIC_URL + f"{task_flow_str}_TransactionsPerSecond.png"
to_csv_cmd = f'{settings.JMETER_PLUGINS_CMD} --generate-csv {summary_csv} --input-jtl {jtl} --plugin-type AggregateReport'
to_rt_png = f'{settings.JMETER_PLUGINS_CMD} --generate-png {rt_png} --input-jtl {jtl} --plugin-type ResponseTimesOverTime'
to_tps_png = f'{settings.JMETER_PLUGINS_CMD} --generate-png {tps_png} --input-jtl {jtl} --plugin-type TransactionsPerSecond'
os.system(to_csv_cmd)
os.system(to_rt_png)
os.system(to_tps_png)
if os.path.exists(summary_csv):
logger.info('jtl转为csv成功')
csv_info = Tools.read_csv_info(summary_csv)
try:
for idx, info in enumerate(csv_info):
if idx == 0:
continue
try:
sampler_id = sampler_id_name_dict[info[0]]
except KeyError:
sampler_id = -1
try:
tps = str(float(info[10]))
except:
tps = '0/sec'
csv_to_db = FlowTaskAggregateReport(
task_id=taskid,
flow_id=flow_id,
sampler_id=sampler_id,
label=Tools.filename(info[0]),
samplers=info[1],
average_req=info[2],
median_req=info[3],
line90_req=info[4],
line95_req=info[5],
line99_req=info[6],
min_req=info[7],
max_req=info[8],
error_rate=info[9],
tps=tps,
recieved_per=str(float(info[11])))
csv_to_db.save()
for sampler_id, save_path in save_path_dict.items():
count_rsp = Tools.count_rsp(save_path)
for key, value in count_rsp.items():
rr = RspResult(sampler_id=sampler_id,
flow_id=flow_id,
response=key,
count=value)
rr.save()
png = PngResult(flow_id=flow_id,
rt_png_url=os.sep + rt_png,
tps_png_url=os.sep + tps_png)
png.save()
# 更新流水任务的状态为3，完成状态
logger.debug('更新流水任务状态为完成状态')
TaskFlow.objects.filter(randomstr=task_flow_str).update(
task_status=3, end_time=datetime.now())
logger.debug('流水任务执行完成')
except:
logger.error('保存数据失败')
raise
else:
logger.error('jtl转为csvs失败')
raise
except:
# 更新流水任务的状态为2，运行异常
TaskFlow.objects.filter(randomstr=task_flow_str).update(
task_status=2, end_time=datetime.now())
# 这里会自动打印异常信息
logger.exception(f'执行流水任务失败')
finally:
try:
logger.debug('删除任务流水目录')
shutil.rmtree(settings.TEMP_URL + task_flow_str)
except:
logger.debug('任务流水目录不存在')
def post(self, request):
"""
修改csv信息
"""
data = {}
old_jmx_id = request.data.get('jmxId')
user = request.data.get('userId')
if old_jmx_id:
jmx_info = Jmxs.objects.get(id=old_jmx_id)
jmx_path = str(jmx_info.jmx)
jmx_alias = str(jmx_info.jmx_alias)
temp_jmx_path = settings.JMX_URL + Tools.random_str(9) + '.jmx'
shutil.copyfile(jmx_path, temp_jmx_path)
new_jmx_name = jmx_alias + '.' + Tools.random_str(9)
new_jmx_path = settings.JMX_URL + new_jmx_name + '.jmx'
# 解析JMX
jmxinfo = ReadJmx(temp_jmx_path).analysis_jmx()
if not jmxinfo:
return APIRsp(code=400,
msg='添加失败，jmx文件错误',
status=status.HTTP_400_BAD_REQUEST)
samplers_info = jmxinfo[0]
csvs_info = jmxinfo[1]
thread_info = jmxinfo[2]
# jmx路径
data['jmx'] = new_jmx_path
# jmx别名
data['jmx_alias'] = new_jmx_name
# user的id不存在时，会校验失败
data['add_user'] = user
# 线程组基础信息
data['thread_base_info'] = json.dumps(thread_info)
obj = JmxsSerializer(data=data)
# 校验数据格式
if obj.is_valid():
obj.save()
new_jmx_id = obj.data['id']
for sampler in samplers_info:
# 保存sampler信息
sampler_children = sampler['children']
del sampler['children']
sp = JmxThreadGroup(jmx_id=new_jmx_id,
child_name=sampler['name'],
child_info=json.dumps(sampler),
child_thread=sampler['thread_type'])
sp.save()
# 获取保存后得到的id
sampler_id = sp.id
for child in sampler_children:
sc = SamplersChildren(sampler_id=sampler_id,
child_name=child['child_name'],
child_type=child['child_type'],
child_info=json.dumps(child))
sc.save()
if csvs_info:
for csv in csvs_info:
old_csv_name = csv['name']
new_csv_name = Tools.filename(
Tools.filename(
csv['name'])) + '.' + Tools.random_str(9)
new_csv_path = settings.CSV_URL + new_csv_name + '.csv'
with open(new_jmx_path, 'w') as fw:
with open(temp_jmx_path, 'r') as fr:
for line in fr:
if old_csv_name in line:
line = line.replace(
csv['name'], new_csv_name)
fw.write(line)
shutil.copyfile(csv['filename'], new_csv_path)
csv['filename'] = new_csv_path
c = JmxThreadGroup(jmx_id=new_jmx_id,
child_name=csv['name'],
child_type='csv',
child_info=json.dumps(csv),
child_thread=csv['thread_type'])
c.save()
# 保存csv到数据库
co = Csvs(csv=new_csv_path,
jmx_id=new_jmx_id,
add_user_id=user)
co.save()
else:
shutil.copyfile(temp_jmx_path, new_jmx_path)
os.remove(temp_jmx_path)
return APIRsp()
os.remove(new_jmx_path)
os.remove(temp_jmx_path)
return APIRsp(code=400,
msg='添加失败，参数校验未通过',
status=status.HTTP_400_BAD_REQUEST)
else:
return APIRsp(code=400,
msg='修改失败，参数不完整',
status=status.HTTP_400_BAD_REQUEST)
def post(self, request):
"""
:param request: :param request: {'csv': ,'jmx': , 'user': 1}
:return:
"""
data = {}
# request.POST.get适用于form-data请求获取参数
csv = request.FILES.get('csv')
name = request.POST.get('name')
jmx_id = request.POST.get('jmxId')
user = request.POST.get('userId')
variableNames = request.POST.get('variableNames')
delimiter = request.POST.get('delimiter')
# mult-form-data会将json中的true或者false转换为字符串
ignoreFirstLine = request.POST.get('ignoreFirstLine')
recycle = request.POST.get('recycle')
stopThread = request.POST.get('stopThread')
threadType = request.POST.get('threadType')
if csv and jmx_id and user and variableNames and delimiter and ignoreFirstLine and recycle and stopThread and threadType:
csv_name_ext = os.path.splitext(csv.name)
csv_name = csv_name_ext[0]
csv_ext = csv_name_ext[1]
if csv_ext not in settings.CSV_ALLOWED_FILE_TYPE:
return APIRsp(code=205,
msg='无效的格式，请上传.csv格式的文件',
status=status.HTTP_205_RESET_CONTENT)
csvfile = csv_name + "." + Tools.random_str(9) + csv_ext
path = settings.CSV_URL + csvfile
with open(path, 'wb') as f:
for i in csv.chunks():
f.write(i)
data['csv'] = csvfile
# csv不存在时，接口会报错
data['jmx'] = jmx_id
# user不存在时，接口会报错
data['add_user'] = user
obj = CsvSerializer(data=data)
if obj.is_valid():
obj.save()
jmx_path = Jmxs.objects.values('jmx').get(id=jmx_id)['jmx']
csv_info = ModifyJMX(jmx_path).add_csv(
name,
path,
variableNames,
ignoreFirstLine=Tools.strToBool(ignoreFirstLine),
delimiter=delimiter,
recycle=Tools.strToBool(recycle),
stopThread=Tools.strToBool(stopThread),
accord=threadType)
# 保存csv信息
s = JmxThreadGroup(jmx_id=jmx_id,
child_name=name,
child_type='csv',
child_info=json.dumps(csv_info),
child_thread=threadType)
s.save()
return APIRsp()
return APIRsp(code=500,
msg='添加失败，校验未通过',
status=status.HTTP_400_BAD_REQUEST)
else:
return APIRsp(code=400,
msg='添加失败，参数不完整',
status=status.HTTP_400_BAD_REQUEST)
def post(self, request):
"""
:param request: {'jmx': 'name': , 'add_user': 1}
:return:
"""
data = {}
jmx = request.FILES.get('jmx')
jmx_alias = request.POST.get('jmxName')
user = request.POST.get('addUser')
if jmx and user:
jmx_name_ext = os.path.splitext(jmx.name)
jmx_name = jmx_name_ext[0]
jmx_ext = jmx_name_ext[1]
if jmx_ext not in settings.JMX_ALLOWED_FILE_TYPE:
return APIRsp(code=205,
msg='无效的格式，请上传.jmx格式的文件',
status=status.HTTP_205_RESET_CONTENT)
jmxfile = jmx_name + "." + Tools.random_str(9) + jmx_ext
jmxpath = settings.JMX_URL + jmxfile
with open(jmxpath, 'wb') as f:
for i in jmx.chunks():
f.write(i)
jmxinfo = ReadJmx(jmxpath).analysis_jmx()
if not jmxinfo:
return APIRsp(code=400,
msg='添加失败，jmx文件错误',
status=status.HTTP_400_BAD_REQUEST)
samplers_info = jmxinfo[0]
csvs_info = jmxinfo[1]
thread_info = jmxinfo[2]
# jmx路径
data['jmx'] = jmxpath
# jmx名称
if jmx_alias:
data['jmx_alias'] = jmx_alias
else:
data['jmx_alias'] = jmx_name
# user的id不存在时，会校验失败
data['add_user'] = user
# 线程组基础信息
data['thread_base_info'] = json.dumps(thread_info)
obj = JmxsSerializer(data=data)
# 校验数据格式
if obj.is_valid():
obj.save()
jmx_id = obj.data['id']
for sampler in samplers_info:
# 保存sampler信息
sampler_children = sampler['children']
del sampler['children']
sp = JmxThreadGroup(jmx_id=jmx_id,
child_name=sampler['name'],
child_info=json.dumps(sampler),
child_thread=sampler['thread_type'])
sp.save()
# 获取保存后得到的id
sampler_id = sp.id
for child in sampler_children:
sc = SamplersChildren(sampler_id=sampler_id,
child_name=child['child_name'],
child_type=child['child_type'],
child_info=json.dumps(child))
sc.save()
if csvs_info:
for csv in csvs_info:
# 保存csv信息
c = JmxThreadGroup(jmx_id=jmx_id,
child_name=csv['name'],
child_type='csv',
child_info=json.dumps(csv),
child_thread=csv['thread_type'])
c.save()
return APIRsp()
os.remove(jmxpath)
return APIRsp(code=400,
msg='添加失败，参数校验未通过',
status=status.HTTP_400_BAD_REQUEST)
else:
return APIRsp(code=400,
msg='添加失败，未传入文件或用户id',
status=status.HTTP_400_BAD_REQUEST)
def post(self, request):
data = {}
jmx_name = request.data.get('jmxName')
sampler_name = request.data.get('samplerName')
method = request.data.get('method')
url = request.data.get('url')
param_type = request.data.get('paramType')
params = request.data.get('params')
user = request.data.get('addUser')
if not sampler_name or not method or not url:
return APIRsp(code=400,
msg='创建jmx失败，接口名称、方法、url必传',
status=status.HTTP_400_BAD_REQUEST)
template_path = settings.JMX_URL + 'template.jmx'
new_jmxpath = settings.JMX_URL + jmx_name + "." + Tools.random_str(
9) + '.jmx'
shutil.copyfile(template_path, new_jmxpath)
try:
ModifyJMX(new_jmxpath).add_sampler(sampler_name,
url,
method,
param_type=param_type,
params=params)
except:
os.remove(new_jmxpath)
return APIRsp(code=400,
msg='创建jmx失败，参数错误！',
status=status.HTTP_400_BAD_REQUEST)
jmxinfo = ReadJmx(new_jmxpath).analysis_jmx(upload=False)
if not jmxinfo:
return APIRsp(code=400,
msg='添加失败，jmx文件错误！',
status=status.HTTP_400_BAD_REQUEST)
sampler_info = jmxinfo[0][0]
thread_info = jmxinfo[2]
# jmx路径
data['jmx'] = new_jmxpath
# jmx名称
data['jmx_alias'] = jmx_name
# user的id不存在时，会校验失败
data['add_user'] = user
# 线程组基础信息
data['thread_base_info'] = json.dumps(thread_info)
obj = JmxsSerializer(data=data)
# 校验数据格式
if obj.is_valid():
obj.save()
jmx_id = obj.data['id']
if sampler_info:
# 保存sampler信息
del sampler_info['children']
s = JmxThreadGroup(jmx_id=jmx_id,
child_name=sampler_name,
child_info=json.dumps(sampler_info),
child_thread=sampler_info['thread_type'])
s.save()
return APIRsp(data={'samplerId': s.id}) # sampler的id
os.remove(new_jmxpath)
return APIRsp(code=400,
msg='添加失败，参数校验未通过',
status=status.HTTP_400_BAD_REQUEST)