def check_qcow_file_exists(snapshots): """ snapshots [ base, inc1, inc2] :param snapshots: :return: """ snapshots = json.loads(snapshots) snapshots.reverse() _logger.info('check_qcow_file_exists snapshots:{}'.format(snapshots)) rs = {'is_valid': True} try: for index, snapshot in enumerate(snapshots): disk_snapshot = get_disk_snapshot_from_info( snapshot['path'], snapshot['snapshot']) if index == 0: if not GetSnapshotList.is_schedule_valid(disk_snapshot): rs['is_valid'] = False break if not GetSnapshotList.is_disk_snapshot_object_exist( disk_snapshot): rs['is_valid'] = False break if not GetSnapshotList.is_disk_snapshot_file_exist( disk_snapshot): rs['is_valid'] = False break except Exception as e: rs['is_valid'] = False _logger.error('check_qcow_file_exists error:{}'.format(e), exc_info=True) _logger.info('check_qcow_file_exists is_valid:{}'.format( rs['is_valid'])) return rs
def _get_need_lock_snapshots(self): self.log_info('_get_need_lock_snapshots start') if self.task_type == self.QEMU_WORK: last_snapshot = self.snap_shots[-1] last_disk_snapshot = get_disk_snapshot_from_info( last_snapshot.path, last_snapshot.snapshot) if last_disk_snapshot.is_cdp: timestamp = RemoteBackupHelperRemote._from_new_disk_snapshot_info_get_last_timestamp( last_snapshot.snapshot) if timestamp == -1: restore_timestamp = None else: restore_timestamp = timestamp else: restore_timestamp = None validator_list = [ GetSnapshotList.is_disk_snapshot_object_exist, GetSnapshotList.is_disk_snapshot_file_exist ] snapshots = GetSnapshotList.query_snapshots_by_snapshot_object( last_disk_snapshot, validator_list, restore_timestamp) if not snapshots: xlogging.raise_and_logging_error( '获取快照链失败', 'get snapshots failed {}'.format(last_disk_snapshot.ident)) self.need_lock_snapshots = snapshots else: self.need_lock_snapshots = self.snap_shots self.log_info('_get_need_lock_snapshots end')
def _get_kvm_used_params(self, host_snapshot, snapshot_datetime): tmp_files = list() ice_snapshots = list() kvm_used_params = { 'logic': 'linux', 'disk_ctl_type': 'scsi-hd', 'aio_server_ip': '172.29.16.2', 'ip_prefix': '172.29.140', 'tap_name_prefix': 'filesync', 'memory_mbytes': 128, 'qcow_files': list(), 'disksnapshots': list() } tmp_qcow_dir = r'/tmp/tmp_qcow/' os.makedirs(tmp_qcow_dir, exist_ok=True) tmp_qcow_file = os.path.join( tmp_qcow_dir, 'filesync{}.qcow2'.format(uuid.uuid4().hex)) kvm_used_params['qcow_files'].append({ 'base': '/home/kvm_rpc/Clerware-7-x86_64-1611.mini.loader.qcow2', 'new': tmp_qcow_file, 'qcow_type': 'with_base' }) tmp_files.append(tmp_qcow_file) for index, disk_snapshot in enumerate( host_snapshot.disk_snapshots.all()): timestamp = self.task.snapshot_datetime.timestamp( ) if snapshot_datetime else None if timestamp: disk_snapshot, timestamp = self.get_disk_snapshot( host_snapshot, disk_snapshot.disk.ident, timestamp) validator_list = [ GetSnapshotList.is_disk_snapshot_object_exist, GetSnapshotList.is_disk_snapshot_file_exist ] snapshots_ice = GetSnapshotList.query_snapshots_by_snapshot_object( disk_snapshot, validator_list, timestamp=timestamp) if not snapshots_ice: xlogging.raise_and_logging_error('获取快照链失败', 'get snapshot failed', 169) snapshots = [{ 'path': snapshot.path, 'ident': snapshot.snapshot } for snapshot in snapshots_ice] kvm_used_params['disksnapshots'].append({ 'images': snapshots, 'nbd_type': 'gznbd', 'scsi_id': disk_snapshot.disk.ident, }) ice_snapshots.extend(snapshots_ice) return tmp_files, ice_snapshots, kvm_used_params
def _get_host_snapshot_latest_disk_snapshots(host_snapshot, inc_call): disk_snapshot_chain = {} if host_snapshot.is_cdp: disks_snapshots = host_snapshot.disk_snapshots.all() for disk_snapshot in disks_snapshots: disk_ident = disk_snapshot.disk.ident timestamp = host_snapshot.cdp_info.last_datetime.timestamp() disk_snapshot_ident, restore_timestamp = \ GetDiskSnapshot.query_cdp_disk_snapshot_ident(host_snapshot, disk_ident, timestamp) if disk_snapshot_ident is None or restore_timestamp is None: disk_snapshot_ident = GetDiskSnapshot.query_normal_disk_snapshot_ident( host_snapshot, disk_ident) disk_snapshot_object = DiskSnapshot.objects.get( ident=disk_snapshot_ident) validator_list = [ GetSnapshotList.is_disk_snapshot_object_exist, GetSnapshotList.is_disk_snapshot_file_exist ] disk_chain = GetSnapshotList.query_snapshots_by_snapshot_object( disk_snapshot_object, validator_list, restore_timestamp, include_all_node=inc_call) assert disk_chain, '无效的快照链 {}'.format(disk_snapshot) disk_snapshot_chain[disk_snapshot.disk.ident] = disk_chain else: disks_snapshots = host_snapshot.disk_snapshots.all() for disk_snapshot in disks_snapshots: validators = [ GetSnapshotList.is_disk_snapshot_object_exist, GetSnapshotList.is_disk_snapshot_file_exist ] disk_chain = GetSnapshotList.query_snapshots_by_snapshot_object( disk_snapshot, validators, include_all_node=inc_call) assert disk_chain, '无效的快照链 {}'.format(disk_snapshot) disk_snapshot_chain[disk_snapshot.disk.ident] = disk_chain return disk_snapshot_chain
def _modify_cdp_disk_snapshot_info_with_last_timestamp( cdp_disk_snapshot_info, last_timestamp): assert isinstance(last_timestamp, float), 'not invalid info:{}'.format(last_timestamp) cdp_timestamp = RemoteBackupHelperRemote._from_new_disk_snapshot_info_get_last_timestamp( cdp_disk_snapshot_info.snapshot) if cdp_timestamp == -1: if GetSnapshotList.is_cdp_timestamp_exist( cdp_disk_snapshot_info.path, last_timestamp): return GetSnapshotList.format_timestamp(last_timestamp, None), False else: return None, True elif last_timestamp < cdp_timestamp: if GetSnapshotList.is_cdp_timestamp_exist( cdp_disk_snapshot_info.path, last_timestamp): return GetSnapshotList.format_timestamp( last_timestamp, cdp_timestamp), False else: return None, True else: return None, False
def _convert_host_snapshot_2_new_host_backup_info(host_ident, last_host_snapshot, first_snapshot=False): disk_snapshot_chain = [] # 每个磁盘的链: [[disk1_chain],[disk2_chain]] disks_snapshots = last_host_snapshot.disk_snapshots.all() for disk_snapshot in disks_snapshots: if first_snapshot: validators = [ GetSnapshotList.is_disk_snapshot_object_exist, GetSnapshotList.is_disk_snapshot_file_exist ] disk_chain = GetSnapshotList.query_snapshots_by_snapshot_object( disk_snapshot, validators) disk_chain = [ get_disk_snapshot_from_info(img.path, img.snapshot) for img in disk_chain ] else: disk_chain = [disk_snapshot] assert disk_chain, '无效的快照链 {}'.format(disk_snapshot) disk_chain = RemoteBackupHelperRemote.convert_objs_to_json( disk_chain) disk_chain = RemoteBackupHelperRemote.add_disk_ident_to_disk_snapshot_chain( disk_chain) disk_snapshot_chain.append(disk_chain) host_snapshot = RemoteBackupHelperRemote.convert_objs_to_json( [last_host_snapshot])[0] # 修正时间,serializers.serialize 会把时间变成 YYYY-MM-DDTHH:mm:ss.sss 损失精度 host_snapshot['fields'][ 'start_datetime'] = last_host_snapshot.start_datetime.strftime( xdatetime.FORMAT_WITH_MICROSECOND) host_snapshot = RemoteBackupHelperRemote.add_boot_map_to_host_snapshot_sys_info( host_ident, host_snapshot) host_snapshot[ 'disks_idents'] = RemoteBackupHelperRemote.query_disks_idents( disks_snapshots) host_snapshot['disks_chains'] = disk_snapshot_chain return host_snapshot
def get_times_scores_by_host_snapshot(host_snapshot_id, cdp_slice_end_time, query_start_time, query_end_time): query_start_datetime = datetime.strptime(query_start_time, '%Y-%m-%d %H:%M:%S.%f') query_end_datetime = datetime.strptime(query_end_time, '%Y-%m-%d %H:%M:%S.%f') disks_idents = query_current_disks_ident_by_host_snapshot(host_snapshot_id) host_snapshot = HostSnapshot.objects.get(id=host_snapshot_id) cdp_end_timestamp = xdatetime.string2datetime( cdp_slice_end_time).timestamp() validator_list = [ GetSnapshotList.is_disk_snapshot_object_exist, GetSnapshotList.is_disk_snapshot_file_exist ] # 该cdp片段的所有磁盘的: cdps image disks_cdps_image_4slice = list() _disks_cdps_image_4slice = list() for disk_ident in disks_idents: cur_ident, rstamp = GetDiskSnapshot.query_cdp_disk_snapshot_ident( host_snapshot, disk_ident, cdp_end_timestamp) if cur_ident and rstamp: disk_snapshot = DiskSnapshot.objects.get(ident=cur_ident) disk_all_image_until_slice = GetSnapshotList.query_snapshots_by_snapshot_object( disk_snapshot, validator_list, rstamp) is_exist_chain(disk_all_image_until_slice, cur_ident) disk_cdps_image_4slice = get_disk_cdps_image_for_slice( disk_all_image_until_slice) disks_cdps_image_4slice += disk_cdps_image_4slice _disks_cdps_image_4slice.append(disk_cdps_image_4slice) # 保留与查询存在交集的: disk_cdp_image disks_cdps_image_within_query = filter_disk_cdp_image_within_query_time( query_start_datetime, query_end_datetime, disks_cdps_image_4slice) disks_cdps_image_path = [ disk_image_obj.path for disk_image_obj in disks_cdps_image_within_query ] disks_cdps_image_info = get_cdp_images_infos(disks_cdps_image_path) times_iovals = images_infos_convert_to_struct_time_ioval( disks_cdps_image_info) times_iovals = filter_time_ioval_within_query_time(query_start_datetime, query_end_datetime, times_iovals) times_scores = convert_times_iovals_to_times_scores(times_iovals) times_scores = get_constant_elems(times_scores, 200) # 在查询时间内,没有找到时间点,则往窗口左侧找,若无则启用该cdp片段的host_snapshot时间 if len(times_scores) == 0: disks_cdps_image_before_query = get_disks_cdps_image_lt_query_start_datetime( _disks_cdps_image_4slice, query_start_datetime) disks_cdps_image_path = [ disk_cdp_image.path for disk_cdp_image in disks_cdps_image_before_query ] disks_cdps_image_info = get_cdp_images_infos(disks_cdps_image_path) times_iovals = images_infos_convert_to_struct_time_ioval( disks_cdps_image_info) time_ioval = get_closed_time_ioval(times_iovals, query_start_datetime) closed_time = host_snapshot.start_datetime if time_ioval is None else time_ioval[ 'time'] times_scores = [{'time': closed_time, 'score': -1}] return times_scores
def _get_disk_snapshot(self, host_snapshot, snapshot_time, folder_path, logic): disksnapshot = dict() data_devices = list() boot_devices = list() mdisk_snapshots = DiskSnapshot.objects.filter( host_snapshot=host_snapshot.id) if len(mdisk_snapshots) == 0: xlogging.raise_and_logging_error('不存在的客户端快照', 'invalid host snapshot id', status.HTTP_404_NOT_FOUND) host_snapshot_ext_info = json.loads(host_snapshot.ext_info) disk_index_info = host_snapshot_ext_info['disk_index_info'] for index_info in disk_index_info: index_info['disk_ident'] = \ DiskSnapshot.objects.get(ident=index_info['snapshot_disk_ident']).disk.ident for disk_snapshot in mdisk_snapshots: boot_device = False device_profile = dict() device_profile['wwid'] = uuid.uuid4().hex disks = list() qcow2path = os.path.join(folder_path, '{}.qcow2'.format(time.time())) save_disk_ident = None for info in disk_index_info: if disk_snapshot.disk.ident == info['disk_ident']: save_disk_ident = info['disk_ident'] device_profile['snapshot_disk_index'] = info[ 'snapshot_disk_index'] device_profile['DiskSize'] = str(disk_snapshot.bytes) boot_device = info['boot_device'] break device_profile['qcow2path'] = qcow2path device_profile['nbd'] = json.loads( boxService.box_service.NbdFindUnusedReverse()) if snapshot_time is None: disk_snapshot_object = DiskSnapshot.objects.get( ident=disk_snapshot.ident) restore_timestamp = None else: disk_ident = disk_snapshot.disk.ident disk_snapshot_ident, restore_timestamp = \ GetDiskSnapshot.query_cdp_disk_snapshot_ident(host_snapshot, disk_ident, snapshot_time) if disk_snapshot_ident is None or restore_timestamp is None: disk_snapshot_ident = GetDiskSnapshot.query_normal_disk_snapshot_ident( host_snapshot, disk_ident) if disk_snapshot_ident is None: _logger.error( 'no valid cdp disk snapshot,and get normal failed {} {} {}' .format(host_snapshot.id, disk_ident, snapshot_time)) continue _logger.warning( 'no valid cdp disk snapshot use normal snapshot : {} {} {} {}' .format(host_snapshot.id, disk_ident, snapshot_time, disk_snapshot_ident)) else: _logger.debug( 'get valid cdp disk snapshot {} {} {} {}'.format( host_snapshot.id, disk_ident, snapshot_time, disk_snapshot_ident)) disk_snapshot_object = DiskSnapshot.objects.get( ident=disk_snapshot_ident) if disk_snapshot_object is None: xlogging.raise_and_logging_error( '获取硬盘快照信息失败', r'get disk info failed disk_snapshot_object is None') validator_list = [ GetSnapshotList.is_disk_snapshot_object_exist, GetSnapshotList.is_disk_snapshot_file_exist ] disk_snapshots = GetSnapshotList.query_snapshots_by_snapshot_object( disk_snapshot_object, validator_list, restore_timestamp) if len(disk_snapshots) == 0: xlogging.raise_and_logging_error( '获取硬盘快照信息失败', r'get disk info failed name {} time {}'.format( disk_snapshot.ident, restore_timestamp)) for disk_snapshot_m in disk_snapshots: disks.append({ "path": disk_snapshot_m.path, "ident": disk_snapshot_m.snapshot }) if boot_device == True: if logic == 'linux': device_profile[ 'boot_device_normal_snapshot_ident'] = disk_snapshot.ident boot_devices.append({ "device_profile": device_profile, "disk_snapshots": disks, "disk_ident": save_disk_ident }) else: data_devices.append({ "device_profile": device_profile, "disk_snapshots": disks, "disk_ident": save_disk_ident }) disksnapshot['data_devices'] = data_devices disksnapshot['boot_devices'] = boot_devices return disksnapshot
def _start_create_empty_qcow(self): local_snapshot = self._sub_task_object.local_snapshot GetSnapshotList.create_empty_snapshot(local_snapshot, r'内部异常,代码2312', self._task_name, self._task_name)
def _get_args(self): args_list = list() aio_info = json.loads(self._host_object.aio_info) aio_ip = aio_info['ip'] # for dataq_ip_port if self._is_schedule_enable_ssl(): ip_port = '{}:{}'.format(aio_ip, '20001') args_list.extend( ['--dataq_ip_port', ip_port, '--dataq_protocol', 'ssl']) else: ip_port = '{}:{}'.format(aio_ip, '20000') args_list.extend( ['--dataq_ip_port', ip_port, '--dataq_protocol', 'tcp']) if self._get_schedule_bandwidth_config_file(): args_list.extend( ['--rate_limit', self._get_schedule_bandwidth_config_file()]) # for qcow_img_ip_port args_list.extend(['--qcow_img_ip_port', '127.0.0.1:21101']) # for sanpshot_curr local_snapshot = self._sub_task_object.local_snapshot snapshot = self._get_snapshot_str(local_snapshot) args_list.extend(['--sanpshot_curr', snapshot]) # for snapshot_dep last_disk_snapshot_object = local_snapshot.parent_snapshot if last_disk_snapshot_object: validator_list = [ GetSnapshotList.is_disk_snapshot_object_exist, GetSnapshotList.is_disk_snapshot_file_exist ] disk_snapshots = GetSnapshotList.query_snapshots_by_snapshot_object( last_disk_snapshot_object, validator_list) if len(disk_snapshots) == 0: raise SubTaskFailed( '同步子任务失败,获取快照链失败', 'not found dep snapshots', SubTaskFailed.ERROR_CODE_LOCAL_SNAPSHOT_MISS) for disk_snapshot in disk_snapshots: _snapshot = self._get_snapshot_str_img(disk_snapshot) args_list.extend(['--snapshot_dep', _snapshot]) else: disk_snapshots = list() if disk_snapshots: self._need_lock_snapshots = disk_snapshots else: self._need_lock_snapshots = list() # for task_id args_list.extend(['--task_id', self.task_uuid]) # for snapshot_type args_list.extend( ['--snapshot_type', 'cdp' if self.is_cdp else 'qcow2']) # for dataq_number args_list.extend(['--dataq_number', '10']) # for disk_token args_list.extend(['--disk_token', self.task_uuid]) # for disk_bytesize args_list.extend(['--disk_bytesize', str(local_snapshot.bytes)]) return args_list
def query_latest_disk_backup(host_snapshot_id, last_disk_snapshot_ident, last_timestamp): # 查询最新: "磁盘快照" (.cdp) _logger.info( 'query_latest_disk_backup host_snapshot_id:{}, last_disk_snapshot_ident:{}, last_timestamp:{}' .format(host_snapshot_id, last_disk_snapshot_ident, last_timestamp)) host_snapshot_id = int(host_snapshot_id) last_timestamp = float(last_timestamp) host = HostSnapshot.objects.get(id=host_snapshot_id).host latest_host_snapshot, latest_disk_snapshots = RemoteBackupHelperRemote.get_host_latest_snapshots( host.ident, inc_all=True) if not latest_disk_snapshots: _logger.error( 'query_latest_disk_backup latest_disk_snapshots is empty!') return None last_disk_snapshot = DiskSnapshot.objects.get( ident=last_disk_snapshot_ident) # 在快照链中 求取差异部分 new_backup_list = list() is_family = True if last_disk_snapshot.disk.ident in latest_disk_snapshots: disk_snapshots = latest_disk_snapshots[ last_disk_snapshot.disk.ident] find_key = False for disk_snapshot_info in disk_snapshots: disk_snapshot_object = get_disk_snapshot_from_info( disk_snapshot_info.path, disk_snapshot_info.snapshot) if disk_snapshot_object.ident == last_disk_snapshot.ident: find_key = True if disk_snapshot_object.is_cdp: if last_timestamp == -1: continue new_time_str, need_all_backup = \ RemoteBackupHelperRemote._modify_cdp_disk_snapshot_info_with_last_timestamp( disk_snapshot_info, last_timestamp) _logger.info( 'query_latest_disk_backup disk_snapshot_info:{} ' 'last_timestamp:{} ' 'new_time_str:{},' 'need_all_backup:{}'.format( disk_snapshot_info, last_timestamp, new_time_str, need_all_backup)) # 需要完整备份 if need_all_backup: is_family = False new_backup_list = disk_snapshots break if new_time_str: disk_snapshot_info.snapshot = new_time_str new_backup_list.append(disk_snapshot_info) else: return { 'is_family': True, 'disk_snapshot_info': [] } # 不需要备份 else: if find_key: new_backup_list.append(disk_snapshot_info) if not find_key and not disk_snapshots: _logger.warning('query_latest_disk_backup not find_key!') is_family = False new_backup_list = disk_snapshots else: # 没有在一个链中 需要根据native guid 获取 最新的链 is_family = False last_host_snapshot = GetSnapshotList.get_host_snapshot_by_disk_snapshot( last_disk_snapshot) native_guid = RemoteBackupHelperRemote.get_disk_native_guid( last_disk_snapshot_ident, last_host_snapshot) if not native_guid: _logger.error( 'query_latest_disk_backup not find native guid, last_host_snapshot:{} ' 'last_disk_snapshot:{}'.format(last_host_snapshot, last_disk_snapshot)) return None disk_snapshot = GetDiskSnapshot.get_by_native_guid_and_host_snapshot( native_guid, latest_host_snapshot) disk_ident = disk_snapshot.disk.ident new_backup_list = latest_disk_snapshots[disk_ident] new_backup_object_list = [ get_disk_snapshot_from_info(item.path, item.snapshot) for item in GetSnapshotList.clean_not_key_snapshots(new_backup_list) ] disk_snapshot_info = RemoteBackupHelperRemote.convert_objs_to_json( new_backup_object_list) RemoteBackupHelperRemote.add_snapshot_info(disk_snapshot_info, new_backup_list) RemoteBackupHelperRemote.add_disk_ident_to_disk_snapshot_chain( disk_snapshot_info) result = { 'is_family': is_family, 'disk_snapshot_info': disk_snapshot_info } _logger.info('query_latest_disk_backup out:{}'.format( pprint.pformat(result))) return result