def test_pt_archiver_generate_args2cmd(self):
"""
测试pt_archiver参数转换
:return:
"""
args = {
"no-version-check": True,
"source": '',
"where": '',
"progress": 5000,
"statistics": True,
"charset": 'UTF8',
"limit": 10000,
"txn-size": 1000,
"sleep": 1
}
pt_archiver = PtArchiver()
cmd_args = pt_archiver.generate_args2cmd(args, False)
self.assertIsInstance(cmd_args, list)
cmd_args = pt_archiver.generate_args2cmd(args, True)
self.assertIsInstance(cmd_args, str)
def archive(archive_id):
"""
执行数据库归档
:return:
"""
archive_info = ArchiveConfig.objects.get(id=archive_id)
s_ins = archive_info.src_instance
src_db_name = archive_info.src_db_name
src_table_name = archive_info.src_table_name
condition = archive_info.condition
no_delete = archive_info.no_delete
sleep = archive_info.sleep
mode = archive_info.mode
# 获取归档表的字符集信息
s_engine = get_engine(s_ins)
db = s_engine.schema_object.databases[src_db_name]
tb = db.tables[src_table_name]
charset = tb.options['charset'].value
if charset is None:
charset = db.options['charset'].value
pt_archiver = PtArchiver()
# 准备参数
source = fr"h={s_ins.host},u={s_ins.user},p={s_ins.password},P={s_ins.port},D={src_db_name},t={src_table_name}"
args = {
"no-version-check": True,
"source": source,
"where": condition,
"progress": 5000,
"statistics": True,
"charset": charset,
"limit": 10000,
"txn-size": 1000,
"sleep": sleep
}
# 归档到目标实例
if mode == 'dest':
d_ins = archive_info.dest_instance
dest_db_name = archive_info.dest_db_name
dest_table_name = archive_info.dest_table_name
dest = fr"h={d_ins.host},u={d_ins.user},p={d_ins.password},P={d_ins.port},D={dest_db_name},t={dest_table_name}"
args['dest'] = dest
args['bulk-insert'] = True
if no_delete:
args['no-delete'] = True
else:
args['bulk-delete'] = True
elif mode == 'file':
output_directory = os.path.join(settings.BASE_DIR,
'downloads/archiver')
os.makedirs(output_directory, exist_ok=True)
args[
'file'] = f'{output_directory}/{s_ins.instance_name}-{src_db_name}-{src_table_name}.txt'
if no_delete:
args['no-delete'] = True
else:
args['bulk-delete'] = True
elif mode == 'purge':
args['purge'] = True
# 参数检查
args_check_result = pt_archiver.check_args(args)
if args_check_result['status'] == 1:
return JsonResponse(args_check_result)
# 参数转换
cmd_args = pt_archiver.generate_args2cmd(args, shell=True)
# 执行命令，获取结果
select_cnt = 0
insert_cnt = 0
delete_cnt = 0
with FuncTimer() as t:
p = pt_archiver.execute_cmd(cmd_args, shell=True)
stdout = ''
for line in iter(p.stdout.readline, ''):
if re.match(r'^SELECT\s(\d+)$', line, re.I):
select_cnt = re.findall(r'^SELECT\s(\d+)$', line)
elif re.match(r'^INSERT\s(\d+)$', line, re.I):
insert_cnt = re.findall(r'^INSERT\s(\d+)$', line)
elif re.match(r'^DELETE\s(\d+)$', line, re.I):
delete_cnt = re.findall(r'^DELETE\s(\d+)$', line)
stdout += f'{line}\n'
statistics = stdout
# 获取异常信息
stderr = p.stderr.read()
if stderr:
statistics = stdout + stderr
# 判断归档结果
select_cnt = int(select_cnt[0]) if select_cnt else 0
insert_cnt = int(insert_cnt[0]) if insert_cnt else 0
delete_cnt = int(delete_cnt[0]) if delete_cnt else 0
error_info = ''
success = True
if stderr:
error_info = f'命令执行报错:{stderr}'
success = False
if mode == 'dest':
# 删除源数据，判断删除数量和写入数量
if not no_delete and (insert_cnt != delete_cnt):
error_info = f"删除和写入数量不一致:{insert_cnt}!={delete_cnt}"
success = False
elif mode == 'file':
# 删除源数据，判断查询数量和删除数量
if not no_delete and (select_cnt != delete_cnt):
error_info = f"查询和删除数量不一致:{select_cnt}!={delete_cnt}"
success = False
elif mode == 'purge':
# 直接删除。判断查询数量和删除数量
if select_cnt != delete_cnt:
error_info = f"查询和删除数量不一致:{select_cnt}!={delete_cnt}"
success = False
# 执行信息保存到数据库
if connection.connection and not connection.is_usable():
close_old_connections()
# 更新最后归档时间
ArchiveConfig(
id=archive_id,
last_archive_time=t.end).save(update_fields=['last_archive_time'])
# 替换密码信息后保存
ArchiveLog.objects.create(
archive=archive_info,
cmd=cmd_args.replace(s_ins.password, '***').replace(
d_ins.password, '***') if mode == 'dest' else cmd_args.replace(
s_ins.password, '***'),
condition=condition,
mode=mode,
no_delete=no_delete,
sleep=sleep,
select_cnt=select_cnt,
insert_cnt=insert_cnt,
delete_cnt=delete_cnt,
statistics=statistics,
success=success,
error_info=error_info,
start_time=t.start,
end_time=t.end)
if not success:
raise Exception(f'{error_info}\n{statistics}')