def _check_archive_schedule_valid(backup_task_schedule_id): try: return ArchiveSchedule.objects.get(id=backup_task_schedule_id) except ArchiveSchedule.DoesNotExist: xlogging.raise_and_logging_error('不存在的导出计划:{}'.format(backup_task_schedule_id), 'invalid ArchiveTaskSchedule:{}'.format(backup_task_schedule_id), status.HTTP_404_NOT_FOUND)
def getDiskInfo(ident, cdp_time): disk_snapshot_object = '' try: _logger.debug("start get disk snapshot ident {}".format(ident)) disk_snapshot_object = DiskSnapshot.objects.get(ident=ident) except DiskSnapshot.DoesNotExist: _logger.error("invalid disk snapshot ident {}".format(ident)) xlogging.raise_and_logging_error('不存在的disk快照:{}'.format(ident), 'invalid disk snapshot ident', status.HTTP_404_NOT_FOUND) _logger.debug("get disk snapshot success,ident {} id {}".format( disk_snapshot_object.id, disk_snapshot_object.ident)) validator_list = [ GetSnapshotList.is_disk_snapshot_object_exist, GetSnapshotList.is_disk_snapshot_file_exist ] disk_snapshots = GetSnapshotList.query_snapshots_by_snapshot_object( disk_snapshot_object, validator_list, cdp_time) if len(disk_snapshots) == 0: xlogging.raise_and_logging_error( '获取硬盘快照信息失败:ident {},time {}'.format(ident, cdp_time), r'get disk info failed name {} time {}'.format(ident, cdp_time), status.HTTP_404_NOT_FOUND) _logger.debug("disk id {}".format(disk_snapshot_object.id)) _logger.debug('image path {}'.format(disk_snapshot_object.image_path)) _logger.debug("disk_snapshots {}".format(disk_snapshots)) return disk_snapshot_object, disk_snapshots
def get_snapshot_inc_bitmap(curr_snapshot, prev_snapshot): log_info_msg("[get_snapshot_inc_bitmap] begin") curr_path = get_map_file_path(curr_snapshot) prev_path = get_map_file_path(prev_snapshot) log_info_msg("[get_snapshot_inc_bitmap] curr_path={}".format(curr_path)) log_info_msg("[get_snapshot_inc_bitmap] prev_path={}".format(prev_path)) if not curr_path: xlogging.raise_and_logging_error( r'找不到备份位图文件 1', r'[get_snapshot_inc_bitmap] get curr_path failed') return [] if prev_snapshot and (not prev_path): xlogging.raise_and_logging_error( r'找不到备份位图文件 2', r'[get_snapshot_inc_bitmap] get curr_path failed') return [] bmp_list = get_snapshot_inc_bitmap_imp(curr_path, prev_path) if not bmp_list: log_warm_msg( "[get_snapshot_inc_bitmap] get_snapshot_inc_bitmap_imp return None" ) return [] log_info_msg("[get_snapshot_inc_bitmap] success") return bmp_list
def _get_need_lock_snapshots(self): self.log_info('_get_need_lock_snapshots start') if self.task_type == self.QEMU_WORK: last_snapshot = self.snap_shots[-1] last_disk_snapshot = get_disk_snapshot_from_info( last_snapshot.path, last_snapshot.snapshot) if last_disk_snapshot.is_cdp: timestamp = RemoteBackupHelperRemote._from_new_disk_snapshot_info_get_last_timestamp( last_snapshot.snapshot) if timestamp == -1: restore_timestamp = None else: restore_timestamp = timestamp else: restore_timestamp = None validator_list = [ GetSnapshotList.is_disk_snapshot_object_exist, GetSnapshotList.is_disk_snapshot_file_exist ] snapshots = GetSnapshotList.query_snapshots_by_snapshot_object( last_disk_snapshot, validator_list, restore_timestamp) if not snapshots: xlogging.raise_and_logging_error( '获取快照链失败', 'get snapshots failed {}'.format(last_disk_snapshot.ident)) self.need_lock_snapshots = snapshots else: self.need_lock_snapshots = self.snap_shots self.log_info('_get_need_lock_snapshots end')
def qcow2_to_lba(qcow2_tree, lba_tree): for qcow2 in qcow2_tree: q_data = qcow2.data q_bgn = q_data[0] q_end = q_data[1] l_bgn = q_data[2] l_end = q_data[3] is_lower = q_data[4] if (qcow2.begin < q_bgn) or (qcow2.end > q_end): xlogging.raise_and_logging_error( r'位图数据不正常', r'[qcow2_to_lba] not equal begin, begin={} g_bgn={} end={} q_end={}' .format(hex(qcow2.begin), hex(q_bgn), hex(qcow2.end), hex(q_end))) return False offset = qcow2.begin - q_bgn l_bgn += offset offset = q_end - qcow2.end l_end -= offset lba_tree.add(Interval(l_bgn, l_end, qcow2.begin)) return True
def get_bitmap_iv_tree(curr_path, prev_path): if not prev_path: return get_single_iv_tree(curr_path) curr_tree = IntervalTree() retval = read_snapshot_bitmap(curr_path, add_by_qcow2_cb, curr_tree) if not retval: xlogging.raise_and_logging_error( r'读取位图文件 1 失败', r'[get_snapshot_inc_bitmap] get curr_path failed') return None retval = read_snapshot_bitmap(prev_path, exclude_by_qcow2, curr_tree) if not retval: xlogging.raise_and_logging_error( r'读取位图文件 2 失败', r'[get_snapshot_inc_bitmap] get curr_path failed') return None lba_tree = IntervalTree() retval = qcow2_to_lba(curr_tree, lba_tree) curr_tree.clear() if not retval: log_err_msg("[get_bitmap_iv_tree] qcow2_to_lba failed") return None return lba_tree
def deleteNormalDiskSnapshot(self, path, ident, enable_fake=True): if self.__check_fake_call('deleteNormalDiskSnapshot') and enable_fake: _logger.warning(r'deleteNormalDiskSnapshot enable fake') return try: config_path = '/dev/shm/delete_normal_disk_snapshot_sleep' if os.path.exists(config_path): with open(config_path) as f: sleep_seconds = int(f.read()) _logger.debug( r'deleteNormalDiskSnapshot will sleep : {}'.format( sleep_seconds)) time.sleep(sleep_seconds) except Exception: pass returned = self.getImgPrx().DelSnaport( IMG.ImageSnapshotIdent(path, ident)) if returned == -2: xlogging.raise_and_logging_error( r'快照磁盘镜像({})正在使用中,无法回收'.format(ident), r'delete snapshot {} - {} failed, using'.format(path, ident)) elif returned != 0: xlogging.raise_and_logging_error( r'回收快照磁盘镜像({})失败'.format(ident), r'delete snapshot {} - {} failed, {}'.format( path, ident, returned))
def start(self): if self._engine: super().start() else: xlogging.raise_and_logging_error( '内部异常,无效的调用', r'start without _engine :{}'.format(self.name), status.HTTP_501_NOT_IMPLEMENTED)
def restart_ntp_server(): cmd = 'systemctl restart ntpd' returned_code, lines = boxService.box_service.runCmd(cmd, False) if returned_code != 0: xlogging.raise_and_logging_error( r'重启NTP失败', 'restart ntpd failed : {} {}'.format(returned_code, lines))
def http_close_remote_backup_logic(self, task_uuid, disk_token): payload = {'type': 'close_remote_backup_logic', 'task_uuid': task_uuid, 'disk_token': disk_token} res = self.http_post_to_url(payload) if status.is_success(res.status_code): return json.loads(res.content.decode('utf-8')) xlogging.raise_and_logging_error('http_close_remote_backup_logic', res.reason)
def http_query_is_host_cdp_back_end(self, host_snapshot_id): payload = {'type': 'query_is_host_cdp_back_end', 'host_snapshot_id': host_snapshot_id} res = self.http_post_to_url(payload) if status.is_success(res.status_code): return json.loads(res.content.decode('utf-8')) xlogging.raise_and_logging_error('http_query_is_host_cdp_back_end', res.reason)
def http_check_qcow_file_exists(self, snapshots): payload = {'type': 'check_qcow_file_exists', 'snapshots': snapshots} res = self.http_post_to_url(payload) if status.is_success(res.status_code): return json.loads(res.content.decode('utf-8')) xlogging.raise_and_logging_error('http_check_qcow_file_exists', res.reason)
def _get_ethers_info_line(): cmd_result_code, ethers_info = boxService.box_service.runCmd( 'ifconfig', False) if cmd_result_code != 0: xlogging.raise_and_logging_error( '查询网卡信息失败', 'run cmd ifconfig failed, return: {}'.format(ethers_info)) return ethers_info
def _check_strategy_valid(strategy_id): try: return WebGuardStrategy.objects.get(id=strategy_id) except WebGuardStrategy.DoesNotExist: xlogging.raise_and_logging_error( '不存在的检测计划:{}'.format(strategy_id), 'invalid WebGuardStrategy:{}'.format(strategy_id), status.HTTP_404_NOT_FOUND)
def http_query_new_disk_backup(self, host_snapshot_id, last_disk_snapshot_ident): payload = {'type': 'query_new_disk_backup', 'host_snapshot_id': host_snapshot_id, 'last_disk_snapshot_ident': last_disk_snapshot_ident} res = self.http_post_to_url(payload) if status.is_success(res.status_code): return json.loads(res.content.decode('utf-8')) xlogging.raise_and_logging_error('http_query_new_disk_backup error', res.reason)
def _check_user_valid(user_id): try: return User.objects.get(id=user_id) except User.DoesNotExist: xlogging.raise_and_logging_error( '不存在的用户:{}'.format(user_id), 'invalid EmergencyPlan:{}'.format(user_id), status.HTTP_404_NOT_FOUND)
def _check_emergency_plan_valid(plan_id): try: return EmergencyPlan.objects.get(id=plan_id) except EmergencyPlan.DoesNotExist: xlogging.raise_and_logging_error( '不存在的自动应急计划:{}'.format(plan_id), 'invalid EmergencyPlan:{}'.format(plan_id), status.HTTP_404_NOT_FOUND)
def GetPeHostNetAdapterInfo(self, pe_host_ident): returned, result = self.getBoxPrx().GetPeHostNetAdapterInfo( pe_host_ident) if returned != 0: xlogging.raise_and_logging_error( r'获取网络适配器失败', r'call BoxPrx GetPeHostNetAdapterInfo {} failed. returned {}'. format(pe_host_ident, returned), returned, _logger) return result
def get_cdp_file_io_info(cdp_file_path, is_use_flush_flag=0): cmd = '/sbin/aio/cdp_wrapper -print_info {flag} {path} {start} {size}'.format( path=cdp_file_path, start=0, size=1000000000, flag=is_use_flush_flag) returned_code, lines = boxService.box_service.runCmd(cmd, False) if returned_code != 0: xlogging.raise_and_logging_error( r'查看CDP文件失败', 'cdp_wrapper failed : {} {}'.format(returned_code, lines)) return lines
def GetPeHostClassHWInfo(self, pe_host_ident, class_name, parent_level=8): returned, result = self.getBoxPrx().GetPeHostClassHWInfo( pe_host_ident, class_name, parent_level) if returned != 0: xlogging.raise_and_logging_error( r'获取硬件信息失败', r'call BoxPrx GetPeHostClassHWInfo {} - {} - {} failed. returned {}' .format(pe_host_ident, class_name, parent_level, returned)) return result
def http_start_remote_backup_logic(self, task_uuid, disk_token, disk_snapshot_ident, disk_snapshot_list, start_time): payload = {'type': 'start_remote_backup_logic', 'task_uuid': task_uuid, 'disk_token': disk_token, 'disk_snapshot_ident': disk_snapshot_ident, 'disk_snapshot_list': disk_snapshot_list, 'start_time': start_time} res = self.http_post_to_url(payload) if status.is_success(res.status_code): return json.loads(res.content.decode('utf-8')) xlogging.raise_and_logging_error('http_start_remote_backup_logic', res.reason)
def get_storage_node_detail(node_id, refresh_device): nodes_info = StorageNodeLogic.get_all_nodes(refresh_device) if not nodes_info: xlogging.raise_and_logging_error('没有任何已添加的存储节点', 'there is not any storage node') for node_info in nodes_info: if node_info['id'] == node_id: return node_info xlogging.raise_and_logging_error('没有找到指定的存储节点', 'there is not storage node (id={})'.format(node_id))
def get_total_and_used_and_available_by_mount_point(mount_point_path): cmd = 'df | grep "{}" | awk {{\'print $2,$3,$4\'}}'.format( mount_point_path) returned_code, lines = boxService.box_service.runCmd(cmd, True) if returned_code != 0 or len(lines) != 1: xlogging.raise_and_logging_error( r'查询存储空间失败', r'get_total_and_used_and_available_by_mount_point failed : {} {}'. format(returned_code, lines)) values = lines[0].split() return int(values[0]) * 1024, int(values[1]) * 1024, int(values[2]) * 1024
def get_storage_node_base_path(node_id): try: node_base_path = StorageNode.objects.get(id=node_id).path if not boxService.box_service.isFolderExist(node_base_path): xlogging.raise_and_logging_error(r'节点挂载路径不存在', r'{path} do not exist'.format(path=node_base_path)) return node_base_path except StorageNode.DoesNotExist: xlogging.raise_and_logging_error('数据库查询存储节点失败', 'StorageNode(id={}) do not exist')
def generate_task_object(uni_ident, is_auto, plan_id): if is_auto: other_object = WGRestoreTask.objects.filter( finish_datetime__isnull=True, uni_ident=uni_ident).first() if other_object is not None: xlogging.raise_and_logging_error( r'还原目标正在执行还原任务中', r'other_object running : {}'.format(other_object.id), http_status.HTTP_501_NOT_IMPLEMENTED) return WGRestoreTask.objects.create(uni_ident=uni_ident, plan_id=plan_id)
def queryCdpTimestamp(self, path, timestamp, mode='forwards'): # mode: 'forwards', 'backwards' result = self.getLogicPrx().queryCdpTimestamp( path, '{0:f}|{1}'.format(timestamp, mode)) result_list = result.split() if len(result_list) != 1: xlogging.raise_and_logging_error( r'读取CDP数据失败,获取{}的时间点{}失败'.format(path, timestamp), r'get {} snapshot time {} error'.format(path, timestamp)) return float(result_list[0])
def get_host_snapshots_file_size(files_dir): if not boxService.box_service.isFolderExist(files_dir): xlogging.raise_and_logging_error( r'快照路径不存在', r'{path} do not exist'.format(path=files_dir)) cmd = 'du -ms {dir}'.format(dir=files_dir) returned_code, lines = boxService.box_service.runCmd(cmd, False) if returned_code != 0 or len(lines) != 1: xlogging.raise_and_logging_error( r'查询客户端快照文件夹大小失败', 'get_host_snapshot_files_size failed : {} {}'.format( returned_code, lines)) return int(lines[0].split('\t')[0].strip())
def createNormalDiskSnapshot(self, ident, last_snapshot, disk_bytes, flag): handle = self.getImgPrx().create(ident, last_snapshot, disk_bytes, flag) if handle == 0 or handle == -1: xlogging.raise_and_logging_error( r'创建快照磁盘镜像失败'.format(ident), r'create snapshot {} - {} failed, {} {} {}'.format( ident, last_snapshot, disk_bytes, handle, flag)) else: _logger.info(r'createNormalDiskSnapshot ok {} {} {} {} {}'.format( handle, ident, last_snapshot, disk_bytes, flag)) return handle
def _get_kvm_used_params(self, host_snapshot, snapshot_datetime): tmp_files = list() ice_snapshots = list() kvm_used_params = { 'logic': 'linux', 'disk_ctl_type': 'scsi-hd', 'aio_server_ip': '172.29.16.2', 'ip_prefix': '172.29.140', 'tap_name_prefix': 'filesync', 'memory_mbytes': 128, 'qcow_files': list(), 'disksnapshots': list() } tmp_qcow_dir = r'/tmp/tmp_qcow/' os.makedirs(tmp_qcow_dir, exist_ok=True) tmp_qcow_file = os.path.join( tmp_qcow_dir, 'filesync{}.qcow2'.format(uuid.uuid4().hex)) kvm_used_params['qcow_files'].append({ 'base': '/home/kvm_rpc/Clerware-7-x86_64-1611.mini.loader.qcow2', 'new': tmp_qcow_file, 'qcow_type': 'with_base' }) tmp_files.append(tmp_qcow_file) for index, disk_snapshot in enumerate( host_snapshot.disk_snapshots.all()): timestamp = self.task.snapshot_datetime.timestamp( ) if snapshot_datetime else None if timestamp: disk_snapshot, timestamp = self.get_disk_snapshot( host_snapshot, disk_snapshot.disk.ident, timestamp) validator_list = [ GetSnapshotList.is_disk_snapshot_object_exist, GetSnapshotList.is_disk_snapshot_file_exist ] snapshots_ice = GetSnapshotList.query_snapshots_by_snapshot_object( disk_snapshot, validator_list, timestamp=timestamp) if not snapshots_ice: xlogging.raise_and_logging_error('获取快照链失败', 'get snapshot failed', 169) snapshots = [{ 'path': snapshot.path, 'ident': snapshot.snapshot } for snapshot in snapshots_ice] kvm_used_params['disksnapshots'].append({ 'images': snapshots, 'nbd_type': 'gznbd', 'scsi_id': disk_snapshot.disk.ident, }) ice_snapshots.extend(snapshots_ice) return tmp_files, ice_snapshots, kvm_used_params
def post(self, request, plan_id, api_request=None): if api_request is None: api_request = request.data serializer = EmergencyPlanExecuteSerializer(data=api_request) serializer.is_valid(True) plan = _check_emergency_plan_valid(plan_id) running_tasks = json.loads(plan.running_tasks) current_level = serializer.validated_data['level'] current_type = serializer.validated_data['type'] if current_level in running_tasks.keys( ) and running_tasks[current_level] is not None: xlogging.raise_and_logging_error( r'自动应急任务正在执行中', r'EmergencyPlan.running_task [{}] is not None : {}'.format( current_level, running_tasks[current_level]), status.HTTP_501_NOT_IMPLEMENTED) if current_type == EmergencyPlan.EM_MANUAL: pass # do nothing elif current_type == EmergencyPlan.EM_AUTO: for host in plan.hosts.all(): api_request = { 'host_ident': host.ident, 'plan_id': plan.id, 'is_auto': True } rsp = WGRPreLogic().post(None, api_request) if rsp.status_code == status.HTTP_201_CREATED: WGRPreLogic.clear_strategy_last_404(plan.id) _logger.info('start WGRLogic id:{} name:{} ok'.format( plan.id, plan.name)) else: _logger.warning( 'start WGRLogic id:{} name:{} failed {}'.format( plan.id, plan.name, rsp.status_code)) return Response( status=status.HTTP_500_INTERNAL_SERVER_ERROR, data=rsp.data) elif current_type == EmergencyPlan.EM_MAINTAIN: for host in plan.hosts.all(): MaintainStatus().put( None, host.ident, {'status': xdata.MAINTAIN_STATUS_TAKEOVER}) else: xlogging.raise_and_logging_error( r'未定义的处理类型', r'EmergencyPlan em type [{}] is unknown'.format(current_level), status.HTTP_400_BAD_REQUEST) return Response(status=status.HTTP_201_CREATED)