def test_pt_archiver_generate_args2cmd(self): """ 测试pt_archiver参数转换 :return: """ args = { "no-version-check": True, "source": '', "where": '', "progress": 5000, "statistics": True, "charset": 'UTF8', "limit": 10000, "txn-size": 1000, "sleep": 1 } pt_archiver = PtArchiver() cmd_args = pt_archiver.generate_args2cmd(args, False) self.assertIsInstance(cmd_args, list) cmd_args = pt_archiver.generate_args2cmd(args, True) self.assertIsInstance(cmd_args, str)
def archive(archive_id): """ 执行数据库归档 :return: """ archive_info = ArchiveConfig.objects.get(id=archive_id) s_ins = archive_info.src_instance src_db_name = archive_info.src_db_name src_table_name = archive_info.src_table_name condition = archive_info.condition no_delete = archive_info.no_delete sleep = archive_info.sleep mode = archive_info.mode # 获取归档表的字符集信息 s_engine = get_engine(s_ins) db = s_engine.schema_object.databases[src_db_name] tb = db.tables[src_table_name] charset = tb.options['charset'].value if charset is None: charset = db.options['charset'].value pt_archiver = PtArchiver() # 准备参数 source = fr"h={s_ins.host},u={s_ins.user},p={s_ins.password},P={s_ins.port},D={src_db_name},t={src_table_name}" args = { "no-version-check": True, "source": source, "where": condition, "progress": 5000, "statistics": True, "charset": charset, "limit": 10000, "txn-size": 1000, "sleep": sleep } # 归档到目标实例 if mode == 'dest': d_ins = archive_info.dest_instance dest_db_name = archive_info.dest_db_name dest_table_name = archive_info.dest_table_name dest = fr"h={d_ins.host},u={d_ins.user},p={d_ins.password},P={d_ins.port},D={dest_db_name},t={dest_table_name}" args['dest'] = dest args['bulk-insert'] = True if no_delete: args['no-delete'] = True else: args['bulk-delete'] = True elif mode == 'file': output_directory = os.path.join(settings.BASE_DIR, 'downloads/archiver') os.makedirs(output_directory, exist_ok=True) args[ 'file'] = f'{output_directory}/{s_ins.instance_name}-{src_db_name}-{src_table_name}.txt' if no_delete: args['no-delete'] = True else: args['bulk-delete'] = True elif mode == 'purge': args['purge'] = True # 参数检查 args_check_result = pt_archiver.check_args(args) if args_check_result['status'] == 1: return JsonResponse(args_check_result) # 参数转换 cmd_args = pt_archiver.generate_args2cmd(args, shell=True) # 执行命令,获取结果 select_cnt = 0 insert_cnt = 0 delete_cnt = 0 with FuncTimer() as t: p = pt_archiver.execute_cmd(cmd_args, shell=True) stdout = '' for line in iter(p.stdout.readline, ''): if re.match(r'^SELECT\s(\d+)$', line, re.I): select_cnt = re.findall(r'^SELECT\s(\d+)$', line) elif re.match(r'^INSERT\s(\d+)$', line, re.I): insert_cnt = re.findall(r'^INSERT\s(\d+)$', line) elif re.match(r'^DELETE\s(\d+)$', line, re.I): delete_cnt = re.findall(r'^DELETE\s(\d+)$', line) stdout += f'{line}\n' statistics = stdout # 获取异常信息 stderr = p.stderr.read() if stderr: statistics = stdout + stderr # 判断归档结果 select_cnt = int(select_cnt[0]) if select_cnt else 0 insert_cnt = int(insert_cnt[0]) if insert_cnt else 0 delete_cnt = int(delete_cnt[0]) if delete_cnt else 0 error_info = '' success = True if stderr: error_info = f'命令执行报错:{stderr}' success = False if mode == 'dest': # 删除源数据,判断删除数量和写入数量 if not no_delete and (insert_cnt != delete_cnt): error_info = f"删除和写入数量不一致:{insert_cnt}!={delete_cnt}" success = False elif mode == 'file': # 删除源数据,判断查询数量和删除数量 if not no_delete and (select_cnt != delete_cnt): error_info = f"查询和删除数量不一致:{select_cnt}!={delete_cnt}" success = False elif mode == 'purge': # 直接删除。判断查询数量和删除数量 if select_cnt != delete_cnt: error_info = f"查询和删除数量不一致:{select_cnt}!={delete_cnt}" success = False # 执行信息保存到数据库 if connection.connection and not connection.is_usable(): close_old_connections() # 更新最后归档时间 ArchiveConfig( id=archive_id, last_archive_time=t.end).save(update_fields=['last_archive_time']) # 替换密码信息后保存 ArchiveLog.objects.create( archive=archive_info, cmd=cmd_args.replace(s_ins.password, '***').replace( d_ins.password, '***') if mode == 'dest' else cmd_args.replace( s_ins.password, '***'), condition=condition, mode=mode, no_delete=no_delete, sleep=sleep, select_cnt=select_cnt, insert_cnt=insert_cnt, delete_cnt=delete_cnt, statistics=statistics, success=success, error_info=error_info, start_time=t.start, end_time=t.end) if not success: raise Exception(f'{error_info}\n{statistics}')