def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.CONNECT_PATH, self.connect)
http_server.register_async_uri(self.DISCONNECT_PATH, self.disconnect)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH,
self.create_root_volume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete_bits)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH,
self.create_template_from_volume)
http_server.register_async_uri(self.UPLOAD_BITS_TO_IMAGESTORE_PATH,
self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_BITS_TO_IMAGESTORE_PATH,
self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_IMAGESTORE_PATH,
self.download_from_imagestore)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH,
self.create_empty_volume)
http_server.register_async_uri(self.CONVERT_IMAGE_TO_VOLUME,
self.convert_image_to_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH,
self.resize_volume)
http_server.register_async_uri(self.CHANGE_VOLUME_ACTIVE_PATH,
self.active_lv)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH,
self.get_volume_size)
http_server.register_async_uri(self.CHECK_DISKS_PATH, self.check_disks)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH,
self.revert_volume_from_snapshot)
http_server.register_async_uri(self.GET_QCOW2_REFERENCE,
self.get_qcow2_reference)
self.imagestore_client = ImageStoreClient()
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.CONNECT_PATH, self.connect)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH, self.create_root_volume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete_bits)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH, self.create_template_from_volume)
http_server.register_async_uri(self.UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH, self.upload_to_sftp)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH, self.download_from_sftp)
http_server.register_async_uri(self.UPLOAD_BITS_TO_IMAGESTORE_PATH, self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_BITS_TO_IMAGESTORE_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_IMAGESTORE_PATH, self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH, self.revert_volume_from_snapshot)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.merge_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_SNAPSHOT_PATH, self.offline_merge_snapshots)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.create_empty_volume)
http_server.register_async_uri(self.CREATE_FOLDER_PATH, self.create_folder)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH, self.get_volume_size)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH, self.resize_volume)
http_server.register_async_uri(self.REINIT_IMAGE_PATH, self.reinit_image)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_KVM_HOST_PATH, self.download_from_kvmhost)
http_server.register_async_uri(self.CANCEL_DOWNLOAD_BITS_FROM_KVM_HOST_PATH, self.cancel_download_from_kvmhost)
http_server.register_async_uri(self.GET_DOWNLOAD_BITS_FROM_KVM_HOST_PROGRESS_PATH, self.get_download_bits_from_kvmhost_progress)
self.imagestore_client = ImageStoreClient()
self.id_files = {}
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.INIT_PATH, self.init)
http_server.register_async_uri(self.GET_PHYSICAL_CAPACITY_PATH,
self.get_physical_capacity)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH,
self.create_empty_volume)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH,
self.create_root_volume_from_template)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete)
http_server.register_async_uri(self.DELETE_DIR_PATH, self.deletedir)
http_server.register_async_uri(self.GET_LIST_PATH, self.list)
http_server.register_async_uri(self.DOWNLOAD_BIT_PATH,
self.download_from_sftp)
http_server.register_async_uri(self.UPLOAD_BIT_PATH,
self.upload_to_sftp)
http_server.register_async_uri(self.UPLOAD_TO_IMAGESTORE_PATH,
self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_TO_IMAGESTORE_PATH,
self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_FROM_IMAGESTORE_PATH,
self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_SNAPSHOT_PATH,
self.revert_snapshot)
http_server.register_async_uri(self.REINIT_IMAGE_PATH,
self.reinit_image)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH,
self.merge_snapshot)
http_server.register_async_uri(self.MERGE_AND_REBASE_SNAPSHOT_PATH,
self.merge_and_rebase_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_PATH,
self.offline_merge_snapshot)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME,
self.create_template_from_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(
self.REBASE_ROOT_VOLUME_TO_BACKING_FILE_PATH,
self.rebase_root_volume_to_backing_file)
http_server.register_async_uri(self.VERIFY_SNAPSHOT_CHAIN_PATH,
self.verify_backing_file_chain)
http_server.register_async_uri(self.REBASE_SNAPSHOT_BACKING_FILES_PATH,
self.rebase_backing_files)
http_server.register_async_uri(self.COPY_TO_REMOTE_BITS_PATH,
self.copy_bits_to_remote)
http_server.register_async_uri(self.GET_MD5_PATH, self.get_md5)
http_server.register_async_uri(self.CHECK_MD5_PATH, self.check_md5)
http_server.register_async_uri(self.GET_BACKING_FILE_PATH,
self.get_backing_file_path)
http_server.register_async_uri(self.GET_VOLUME_SIZE,
self.get_volume_size)
http_server.register_async_uri(self.GET_BASE_IMAGE_PATH,
self.get_volume_base_image_path)
http_server.register_async_uri(self.GET_QCOW2_REFERENCE,
self.get_qcow2_reference)
http_server.register_async_uri(self.CONVERT_QCOW2_TO_RAW,
self.convert_qcow2_to_raw)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH,
self.resize_volume)
self.imagestore_client = ImageStoreClient()
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.MOUNT_PATH, self.mount)
http_server.register_async_uri(self.IS_MOUNT_PATH, self.ismount)
http_server.register_async_uri(self.MOUNT_DATA_PATH, self.mountdata)
http_server.register_async_uri(self.INIT_PATH, self.init)
http_server.register_async_uri(self.PING_PATH, self.ping)
http_server.register_async_uri(self.UPDATE_MOUNT_POINT_PATH, self.updateMount)
http_server.register_async_uri(self.REMOUNT_PATH, self.remount)
http_server.register_async_uri(self.UNMOUNT_PATH, self.umount)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.checkbits)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.createempty)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH, self.createvolume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.deletebits)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH, self.getvolumesize)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH, self.revertvolume)
http_server.register_async_uri(self.REINIT_VOLUME_PATH, self.reinit)
http_server.register_async_uri(self.UPLOAD_BIT_TO_IMAGESTORE__PATH, self.uploadtoimagestore)
http_server.register_async_uri(self.DOWNLOAD_BIT_TO_IMAGESTORE_PATH, self.downloadfromimagestore)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH, self.resize)
http_server.register_async_uri(self.COMMIT_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH, self.createtemplate)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.mergesnapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_SNAPSHOT_PATH, self.offlinemerge)
http_server.register_async_uri(self.GET_CAPACITY_PATH, self.getcapacity)
http_server.register_async_uri(self.CHECK_MOUNT_PATH, self.checkmountpath)
self.mount_path = {}
self.uuid = None
self.imagestore_client = ImageStoreClient()
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_sync_uri(self.MOUNT_PATH, self.mount)
http_server.register_sync_uri(self.UNMOUNT_PATH, self.umount)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_TEMPLATE_PATH, self.create_root_volume_from_template)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.create_empty_volume)
http_server.register_async_uri(self.DOWNLOAD_FROM_SFTP_PATH, self.download_from_sftp)
http_server.register_async_uri(self.GET_CAPACITY_PATH, self.get_capacity)
http_server.register_async_uri(self.DELETE_PATH, self.delete)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH, self.create_template_from_root_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH, self.revert_volume_from_snapshot)
http_server.register_async_uri(self.REINIT_IMAGE_PATH, self.reinit_image)
http_server.register_async_uri(self.UPLOAD_TO_SFTP_PATH, self.upload_to_sftp)
http_server.register_async_uri(self.UPLOAD_TO_IMAGESTORE_PATH, self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_TO_IMAGESTORE_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_FROM_IMAGESTORE_PATH, self.download_from_imagestore)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.merge_snapshot)
http_server.register_async_uri(self.REBASE_MERGE_SNAPSHOT_PATH, self.rebase_and_merge_snapshot)
http_server.register_async_uri(self.MOVE_BITS_PATH, self.move_bits)
http_server.register_async_uri(self.OFFLINE_SNAPSHOT_MERGE, self.merge_snapshot_to_volume)
http_server.register_async_uri(self.REMOUNT_PATH, self.remount)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH, self.get_volume_size)
http_server.register_async_uri(self.PING_PATH, self.ping)
http_server.register_async_uri(self.GET_VOLUME_BASE_IMAGE_PATH, self.get_volume_base_image_path)
http_server.register_async_uri(self.UPDATE_MOUNT_POINT_PATH, self.update_mount_point)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH, self.resize_volume)
http_server.register_async_uri(self.NFS_TO_NFS_MIGRATE_BITS_PATH, self.migrate_bits)
http_server.register_async_uri(self.NFS_REBASE_VOLUME_BACKING_FILE_PATH, self.rebase_volume_backing_file)
self.mount_path = {}
self.image_cache = None
self.imagestore_client = ImageStoreClient()
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.CONNECT_PATH, self.connect)
http_server.register_async_uri(self.DISCONNECT_PATH, self.disconnect)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH, self.create_root_volume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete_bits)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH, self.create_template_from_volume)
http_server.register_async_uri(self.UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH, self.upload_to_sftp)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH, self.download_from_sftp)
http_server.register_async_uri(self.UPLOAD_BITS_TO_IMAGESTORE_PATH, self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_BITS_TO_IMAGESTORE_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_IMAGESTORE_PATH, self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH, self.revert_volume_from_snapshot)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.merge_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_SNAPSHOT_PATH, self.offline_merge_snapshots)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.create_empty_volume)
http_server.register_async_uri(self.CONVERT_IMAGE_TO_VOLUME, self.convert_image_to_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH, self.resize_volume)
http_server.register_async_uri(self.CHANGE_VOLUME_ACTIVE_PATH, self.active_lv)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH, self.get_volume_size)
http_server.register_async_uri(self.CHECK_DISKS_PATH, self.check_disks)
http_server.register_async_uri(self.ADD_SHARED_BLOCK, self.add_disk)
http_server.register_async_uri(self.MIGRATE_DATA_PATH, self.migrate_volumes)
http_server.register_async_uri(self.GET_BLOCK_DEVICES_PATH, self.get_block_devices)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_KVM_HOST_PATH, self.download_from_kvmhost)
http_server.register_async_uri(self.CANCEL_DOWNLOAD_BITS_FROM_KVM_HOST_PATH, self.cancel_download_from_kvmhost)
self.imagestore_client = ImageStoreClient()
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.CONNECT_PATH, self.connect)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH,
self.create_root_volume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete_bits)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH,
self.create_template_from_volume)
http_server.register_async_uri(
self.UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH, self.upload_to_sftp)
http_server.register_async_uri(
self.DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH,
self.download_from_sftp)
http_server.register_async_uri(self.UPLOAD_BITS_TO_IMAGESTORE_PATH,
self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_BITS_TO_IMAGESTORE_PATH,
self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_IMAGESTORE_PATH,
self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH,
self.revert_volume_from_snapshot)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH,
self.merge_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_SNAPSHOT_PATH,
self.offline_merge_snapshots)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH,
self.create_empty_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH,
self.get_volume_size)
self.mount_point = None
self.imagestore_client = ImageStoreClient()
def start(self):
http_server = kvmagent.get_http_server()
# http_server.register_async_uri(self.INIT_PATH, self.init)
# http_server.register_async_uri(self.DELETE_PATH, self.delete)
# http_server.register_async_uri(self.CREATE_VOLUME_PATH, self.create)
# http_server.register_async_uri(self.CLONE_PATH, self.clone)
# http_server.register_async_uri(self.COMMIT_IMAGE_PATH, self.commit_image)
# http_server.register_async_uri(self.CREATE_SNAPSHOT_PATH, self.create_snapshot)
# http_server.register_async_uri(self.DELETE_SNAPSHOT_PATH, self.delete_snapshot)
# http_server.register_async_uri(self.PURGE_SNAPSHOT_PATH, self.purge_snapshots)
# http_server.register_async_uri(self.PROTECT_SNAPSHOT_PATH, self.protect_snapshot)
# http_server.register_async_uri(self.UNPROTECT_SNAPSHOT_PATH, self.unprotect_snapshot)
# http_server.register_async_uri(self.ROLLBACK_SNAPSHOT_PATH, self.rollback_snapshot)
# http_server.register_async_uri(self.FLATTEN_PATH, self.flatten)
# http_server.register_async_uri(self.CP_PATH, self.cp)
# http_server.register_async_uri(self.UPLOAD_IMAGESTORE_PATH, self.upload_imagestore)
# http_server.register_async_uri(self.DOWNLOAD_IMAGESTORE_PATH, self.download_imagestore)
# http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH, self.get_volume_size)
# http_server.register_async_uri(self.PING_PATH, self.ping)
# http_server.register_async_uri(self.GET_FACTS, self.get_facts)
# http_server.register_async_uri(self.DELETE_IMAGE_CACHE, self.delete_image_cache)
# http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
# http_server.register_async_uri(self.RESIZE_VOLUME_PATH, self.resize_volume)
# http_server.register_sync_uri(self.ECHO_PATH, self.echo)
# http_server.register_async_uri(self.MIGRATE_VOLUME_PATH, self.migrate_volume)
# http_server.register_async_uri(self.MIGRATE_VOLUME_SNAPSHOT_PATH, self.migrate_volume_snapshot)
# http_server.register_async_uri(self.GET_VOLUME_SNAPINFOS_PATH, self.get_volume_snapinfos)
self.image_cache = None
self.imagestore_client = ImageStoreClient()
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.CONNECT_PATH, self.connect)
http_server.register_async_uri(self.DISCONNECT_PATH, self.disconnect)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH, self.create_root_volume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete_bits)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH, self.create_template_from_volume)
http_server.register_async_uri(self.UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH, self.upload_to_sftp)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH, self.download_from_sftp)
http_server.register_async_uri(self.UPLOAD_BITS_TO_IMAGESTORE_PATH, self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_BITS_TO_IMAGESTORE_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_IMAGESTORE_PATH, self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH, self.revert_volume_from_snapshot)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.merge_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_SNAPSHOT_PATH, self.offline_merge_snapshots)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.create_empty_volume)
http_server.register_async_uri(self.CONVERT_IMAGE_TO_VOLUME, self.convert_image_to_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH, self.resize_volume)
http_server.register_async_uri(self.CHANGE_VOLUME_ACTIVE_PATH, self.active_lv)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH, self.get_volume_size)
http_server.register_async_uri(self.CHECK_DISKS_PATH, self.check_disks)
http_server.register_async_uri(self.ADD_SHARED_BLOCK, self.add_disk)
http_server.register_async_uri(self.MIGRATE_DATA_PATH, self.migrate_volumes)
http_server.register_async_uri(self.GET_BLOCK_DEVICES_PATH, self.get_block_devices)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_KVM_HOST_PATH, self.download_from_kvmhost)
http_server.register_async_uri(self.CANCEL_DOWNLOAD_BITS_FROM_KVM_HOST_PATH, self.cancel_download_from_kvmhost)
http_server.register_async_uri(self.GET_BACKING_CHAIN_PATH, self.get_backing_chain)
http_server.register_async_uri(self.CONVERT_VOLUME_PROVISIONING_PATH, self.convert_volume_provisioning)
self.imagestore_client = ImageStoreClient()
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_sync_uri(self.MOUNT_PATH, self.mount)
http_server.register_sync_uri(self.UNMOUNT_PATH, self.umount)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_TEMPLATE_PATH, self.create_root_volume_from_template)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.create_empty_volume)
http_server.register_async_uri(self.DOWNLOAD_FROM_SFTP_PATH, self.download_from_sftp)
http_server.register_async_uri(self.GET_CAPACITY_PATH, self.get_capacity)
http_server.register_async_uri(self.DELETE_PATH, self.delete)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH, self.create_template_from_root_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH, self.revert_volume_from_snapshot)
http_server.register_async_uri(self.UPLOAD_TO_SFTP_PATH, self.upload_to_sftp)
http_server.register_async_uri(self.UPLOAD_TO_IMAGESTORE_PATH, self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_TO_IMAGESTORE_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_FROM_IMAGESTORE_PATH, self.download_from_imagestore)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.merge_snapshot)
http_server.register_async_uri(self.REBASE_MERGE_SNAPSHOT_PATH, self.rebase_and_merge_snapshot)
http_server.register_async_uri(self.MOVE_BITS_PATH, self.move_bits)
http_server.register_async_uri(self.OFFLINE_SNAPSHOT_MERGE, self.merge_snapshot_to_volume)
http_server.register_async_uri(self.REMOUNT_PATH, self.remount)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH, self.get_volume_size)
http_server.register_async_uri(self.PING_PATH, self.ping)
http_server.register_async_uri(self.GET_VOLUME_BASE_IMAGE_PATH, self.get_volume_base_image_path)
http_server.register_async_uri(self.UPDATE_MOUNT_POINT_PATH, self.update_mount_point)
self.mount_path = {}
self.image_cache = None
self.imagestore_client = ImageStoreClient()
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.INIT_PATH, self.init)
http_server.register_async_uri(self.GET_PHYSICAL_CAPACITY_PATH, self.get_physical_capacity)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.create_empty_volume)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH, self.create_root_volume_from_template)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete)
http_server.register_async_uri(self.DELETE_DIR_PATH, self.deletedir)
http_server.register_async_uri(self.UPLOAD_TO_IMAGESTORE_PATH, self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_TO_IMAGESTORE_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_FROM_IMAGESTORE_PATH, self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_SNAPSHOT_PATH, self.revert_snapshot)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.merge_snapshot)
http_server.register_async_uri(self.MERGE_AND_REBASE_SNAPSHOT_PATH, self.merge_and_rebase_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_PATH, self.offline_merge_snapshot)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME, self.create_template_from_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.REBASE_ROOT_VOLUME_TO_BACKING_FILE_PATH, self.rebase_root_volume_to_backing_file)
http_server.register_async_uri(self.VERIFY_SNAPSHOT_CHAIN_PATH, self.verify_backing_file_chain)
http_server.register_async_uri(self.REBASE_SNAPSHOT_BACKING_FILES_PATH, self.rebase_backing_files)
http_server.register_async_uri(self.COPY_TO_REMOTE_BITS_PATH, self.copy_bits_to_remote)
http_server.register_async_uri(self.GET_MD5_PATH, self.get_md5)
http_server.register_async_uri(self.CHECK_MD5_PATH, self.check_md5)
http_server.register_async_uri(self.GET_BACKING_FILE_PATH, self.get_backing_file_path)
http_server.register_async_uri(self.GET_VOLUME_SIZE, self.get_volume_size)
http_server.register_async_uri(self.GET_BASE_IMAGE_PATH, self.get_volume_base_image_path)
http_server.register_async_uri(self.GET_QCOW2_REFERENCE, self.get_qcow2_reference)
http_server.register_async_uri(self.CONVERT_QCOW2_TO_RAW, self.convert_qcow2_to_raw)
self.imagestore_client = ImageStoreClient()
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.INSTALL_TDC_PATH, self.installtdc)
http_server.register_async_uri(self.DETACH_VOLUME_PATH,
self.detachvolume)
self.imagestore_client = ImageStoreClient()
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.MOUNT_PATH, self.mount)
http_server.register_async_uri(self.IS_MOUNT_PATH, self.ismount)
http_server.register_async_uri(self.MOUNT_DATA_PATH, self.mountdata)
http_server.register_async_uri(self.INIT_PATH, self.init)
http_server.register_async_uri(self.PING_PATH, self.ping)
http_server.register_async_uri(self.LIST_PATH, self.list)
http_server.register_async_uri(self.UPDATE_MOUNT_POINT_PATH, self.updateMount)
http_server.register_async_uri(self.REMOUNT_PATH, self.remount)
http_server.register_async_uri(self.UNMOUNT_PATH, self.umount)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.checkbits)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.createempty)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH, self.createvolume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.deletebits)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH, self.getvolumesize)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH, self.revertvolume)
http_server.register_async_uri(self.REINIT_VOLUME_PATH, self.reinit)
http_server.register_async_uri(self.UPLOAD_BIT_TO_IMAGESTORE__PATH, self.uploadtoimagestore)
http_server.register_async_uri(self.DOWNLOAD_BIT_TO_IMAGESTORE_PATH, self.downloadfromimagestore)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH, self.resize)
http_server.register_async_uri(self.COMMIT_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH, self.createtemplate)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.mergesnapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_SNAPSHOT_PATH, self.offlinemerge)
http_server.register_async_uri(self.GET_CAPACITY_PATH, self.getcapacity)
http_server.register_async_uri(self.CHECK_MOUNT_PATH, self.checkmountpath)
self.mount_path = {}
self.uuid = None
self.imagestore_client = ImageStoreClient()
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.CONNECT_PATH, self.connect)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH, self.create_root_volume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete_bits)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH, self.create_template_from_volume)
http_server.register_async_uri(self.UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH, self.upload_to_sftp)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH, self.download_from_sftp)
http_server.register_async_uri(self.UPLOAD_BITS_TO_IMAGESTORE_PATH, self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_BITS_TO_IMAGESTORE_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_IMAGESTORE_PATH, self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH, self.revert_volume_from_snapshot)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.merge_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_SNAPSHOT_PATH, self.offline_merge_snapshots)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.create_empty_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH, self.get_volume_size)
self.mount_point = None
self.imagestore_client = ImageStoreClient()
class LocalStoragePlugin(kvmagent.KvmAgent):
INIT_PATH = "/localstorage/init"
GET_PHYSICAL_CAPACITY_PATH = "/localstorage/getphysicalcapacity"
CREATE_EMPTY_VOLUME_PATH = "/localstorage/volume/createempty"
CREATE_VOLUME_FROM_CACHE_PATH = "/localstorage/volume/createvolumefromcache"
DELETE_BITS_PATH = "/localstorage/delete"
DELETE_DIR_PATH = "/localstorage/deletedir"
UPLOAD_BIT_PATH = "/localstorage/sftp/upload"
DOWNLOAD_BIT_PATH = "/localstorage/sftp/download"
UPLOAD_TO_IMAGESTORE_PATH = "/localstorage/imagestore/upload"
COMMIT_TO_IMAGESTORE_PATH = "/localstorage/imagestore/commit"
DOWNLOAD_FROM_IMAGESTORE_PATH = "/localstorage/imagestore/download"
REVERT_SNAPSHOT_PATH = "/localstorage/snapshot/revert"
MERGE_SNAPSHOT_PATH = "/localstorage/snapshot/merge"
MERGE_AND_REBASE_SNAPSHOT_PATH = "/localstorage/snapshot/mergeandrebase"
OFFLINE_MERGE_PATH = "/localstorage/snapshot/offlinemerge"
CREATE_TEMPLATE_FROM_VOLUME = "/localstorage/volume/createtemplate"
CHECK_BITS_PATH = "/localstorage/checkbits"
REBASE_ROOT_VOLUME_TO_BACKING_FILE_PATH = "/localstorage/volume/rebaserootvolumetobackingfile"
VERIFY_SNAPSHOT_CHAIN_PATH = "/localstorage/snapshot/verifychain"
REBASE_SNAPSHOT_BACKING_FILES_PATH = "/localstorage/snapshot/rebasebackingfiles"
COPY_TO_REMOTE_BITS_PATH = "/localstorage/copytoremote"
GET_MD5_PATH = "/localstorage/getmd5"
CHECK_MD5_PATH = "/localstorage/checkmd5"
GET_BACKING_FILE_PATH = "/localstorage/volume/getbackingfile"
GET_VOLUME_SIZE = "/localstorage/volume/getsize"
GET_BASE_IMAGE_PATH = "/localstorage/volume/getbaseimagepath"
GET_QCOW2_REFERENCE = "/localstorage/getqcow2reference"
CONVERT_QCOW2_TO_RAW = "/localstorage/imagestore/convert/raw"
RESIZE_VOLUME_PATH = "/localstorage/volume/resize"
REINIT_IMAGE_PATH = "/localstorage/reinit/image"
CHECK_INITIALIZED_FILE = "/localstorage/check/initializedfile"
CREATE_INITIALIZED_FILE = "/localstorage/create/initializedfile"
LOCAL_NOT_ROOT_USER_MIGRATE_TMP_PATH = "primary_storage_tmp_dir"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.INIT_PATH, self.init)
http_server.register_async_uri(self.GET_PHYSICAL_CAPACITY_PATH,
self.get_physical_capacity)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH,
self.create_empty_volume)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH,
self.create_root_volume_from_template)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete)
http_server.register_async_uri(self.DELETE_DIR_PATH, self.deletedir)
http_server.register_async_uri(self.DOWNLOAD_BIT_PATH,
self.download_from_sftp)
http_server.register_async_uri(self.UPLOAD_BIT_PATH,
self.upload_to_sftp)
http_server.register_async_uri(self.UPLOAD_TO_IMAGESTORE_PATH,
self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_TO_IMAGESTORE_PATH,
self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_FROM_IMAGESTORE_PATH,
self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_SNAPSHOT_PATH,
self.revert_snapshot)
http_server.register_async_uri(self.REINIT_IMAGE_PATH,
self.reinit_image)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH,
self.merge_snapshot)
http_server.register_async_uri(self.MERGE_AND_REBASE_SNAPSHOT_PATH,
self.merge_and_rebase_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_PATH,
self.offline_merge_snapshot)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME,
self.create_template_from_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(
self.REBASE_ROOT_VOLUME_TO_BACKING_FILE_PATH,
self.rebase_root_volume_to_backing_file)
http_server.register_async_uri(self.VERIFY_SNAPSHOT_CHAIN_PATH,
self.verify_backing_file_chain)
http_server.register_async_uri(self.REBASE_SNAPSHOT_BACKING_FILES_PATH,
self.rebase_backing_files)
http_server.register_async_uri(self.COPY_TO_REMOTE_BITS_PATH,
self.copy_bits_to_remote)
http_server.register_async_uri(self.GET_MD5_PATH, self.get_md5)
http_server.register_async_uri(self.CHECK_MD5_PATH, self.check_md5)
http_server.register_async_uri(self.GET_BACKING_FILE_PATH,
self.get_backing_file_path)
http_server.register_async_uri(self.GET_VOLUME_SIZE,
self.get_volume_size)
http_server.register_async_uri(self.GET_BASE_IMAGE_PATH,
self.get_volume_base_image_path)
http_server.register_async_uri(self.GET_QCOW2_REFERENCE,
self.get_qcow2_reference)
http_server.register_async_uri(self.CONVERT_QCOW2_TO_RAW,
self.convert_qcow2_to_raw)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH,
self.resize_volume)
http_server.register_async_uri(self.CHECK_INITIALIZED_FILE,
self.check_initialized_file)
http_server.register_async_uri(self.CREATE_INITIALIZED_FILE,
self.create_initialized_file)
self.imagestore_client = ImageStoreClient()
def stop(self):
pass
@kvmagent.replyerror
def check_initialized_file(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
file_path = cmd.filePath
rsp = CheckInitializedFileRsp()
if file_path is None:
rsp.success = False
rsp.error = "input file path is None"
else:
rsp.existed = os.path.exists(file_path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_initialized_file(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
file_path = cmd.filePath
rsp = AgentResponse()
if file_path is None:
rsp.success = False
rsp.error = "input file path is None"
else:
if not os.path.exists(file_path):
f = open(file_path, 'w')
f.close()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def resize_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
install_path = cmd.installPath
rsp = ResizeVolumeRsp()
shell.call("qemu-img resize %s %s" % (install_path, cmd.size))
ret = linux.qcow2_virtualsize(install_path)
rsp.size = ret
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def convert_qcow2_to_raw(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.convert_image_raw(cmd)
@kvmagent.replyerror
def get_qcow2_reference(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
out = shell.call('find %s -type f' % cmd.searchingDir)
rsp = GetQCOW2ReferenceRsp()
rsp.referencePaths = []
real_path = os.path.realpath(cmd.path)
for f in out.splitlines():
backing_file = linux.qcow2_get_backing_file(f)
if os.path.realpath(backing_file) == real_path:
rsp.referencePaths.append(f)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_volume_base_image_path(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeBaseImagePathRsp()
if not os.path.basename(cmd.volumeInstallDir).endswith(cmd.volumeUuid):
raise Exception('maybe you pass a wrong install dir')
path = linux.get_qcow2_base_image_recusively(cmd.volumeInstallDir,
cmd.imageCacheDir)
if not path:
return jsonobject.dumps(rsp)
rsp.path = path
rsp.size = linux.get_qcow2_file_chain_size(path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_backing_file_path(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
out = linux.qcow2_get_backing_file(cmd.path)
rsp = GetBackingFileRsp()
if out:
rsp.backingFilePath = out
rsp.size = os.path.getsize(out)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_md5(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetMd5Rsp()
rsp.md5s = []
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
report = Report(cmd.threadContext, cmd.threadContextStack)
report.processType = "LocalStorageMigrateVolume"
PFILE = shell.call('mktemp /tmp/tmp-XXXXXX').strip()
total = 0
written = 0
for to in cmd.md5s:
total = total + os.path.getsize(to.path)
start = 0
end = 10
if cmd.stage:
start, end = get_scale(cmd.stage)
def _get_progress(synced):
logger.debug("getProgress in get_md5")
if not os.path.exists(PFILE):
return synced
last = linux.tail_1(PFILE).strip()
if not last or not last.isdigit():
return synced
percent = int(
round((float(written) * 100 +
os.path.getsize(to.path) * float(last)) / total *
(end - start) / 100) + start)
report.progress_report(str(percent), "report")
return synced
report.resourceUuid = cmd.volumeUuid
if start == 0:
report.progress_report("0", "start")
else:
report.progress_report(str(start), "report")
for to in cmd.md5s:
_, md5, _ = bash_progress_1(
"pv -n %s 2>%s | md5sum | cut -d ' ' -f 1" % (to.path, PFILE),
_get_progress)
rsp.md5s.append({
'resourceUuid': to.resourceUuid,
'path': to.path,
'md5': md5
})
written += os.path.getsize(to.path)
percent = int(
round(float(written) / float(total) * (end - start) + start))
report.progress_report(percent, "report")
if os.path.exists(PFILE):
os.remove(PFILE)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def check_md5(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
report = Report(cmd.threadContext, cmd.threadContextStack)
report.processType = "LocalStorageMigrateVolume"
PFILE = shell.call('mktemp /tmp/tmp-XXXXXX').strip()
total = 0
written = 0
start = 90
end = 100
if cmd.stage:
start, end = get_scale(cmd.stage)
for to in cmd.md5s:
total = total + os.path.getsize(to.path)
def _get_progress(synced):
logger.debug("getProgress in check_md5")
if not os.path.exists(PFILE):
return synced
last = linux.tail_1(PFILE).strip()
if not last or not last.isdigit():
return synced
percent = int(
round((float(written) * 100 +
os.path.getsize(to.path) * float(last)) / total *
(end - start) / 100) + start)
report.progress_report(percent, "report")
return synced
report.resourceUuid = cmd.volumeUuid
for to in cmd.md5s:
_, dst_md5, _ = bash_progress_1(
"pv -n %s 2>%s | md5sum | cut -d ' ' -f 1" % (to.path, PFILE),
_get_progress)
if dst_md5 != to.md5:
raise Exception(
"MD5 unmatch. The file[uuid:%s, path:%s]'s md5 (src host:%s, dst host:%s)"
% (to.resourceUuid, to.path, to.md5, dst_md5))
written += os.path.getsize(to.path)
percent = int(
round(float(written) / float(total) * (end - start) + start))
report.progress_report(percent, "report")
if os.path.exists(PFILE):
os.remove(PFILE)
rsp = AgentResponse()
if end == 100:
report.progress_report("100", "finish")
else:
report.progress_report(str(end), "report")
return jsonobject.dumps(rsp)
@staticmethod
def _get_disk_capacity(path):
if not path:
raise Exception('storage path cannot be None')
return linux.get_disk_capacity_by_df(path)
@kvmagent.replyerror
@in_bash
def copy_bits_to_remote(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
if cmd.dstUsername != 'root':
raise Exception("cannot support migrate to non-root user host")
chain = sum([linux.qcow2_get_file_chain(p) for p in cmd.paths], [])
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
report = Report(cmd.threadContext, cmd.threadContextStack)
report.processType = "LocalStorageMigrateVolume"
report.resourceUuid = cmd.volumeUuid
PFILE = shell.call('mktemp /tmp/tmp-XXXXXX').strip()
PASSWORD_FILE = linux.write_to_temp_file(cmd.dstPassword)
start = 10
end = 90
if cmd.stage:
start, end = get_scale(cmd.stage)
total = 0
for path in set(chain):
total = total + os.path.getsize(path)
written = 0
def _get_progress(synced):
logger.debug(
"getProgress in localstorage-agent, synced: %s, total: %s" %
(synced, total))
if not os.path.exists(PFILE):
return synced
fpread = open(PFILE, 'r')
lines = fpread.readlines()
if not lines:
fpread.close()
return synced
last = str(lines[-1]).strip().split('\r')[-1]
if not last or len(last.split()) < 1:
fpread.close()
return synced
line = last.split()[0]
if not line.isdigit():
return synced
if total > 0:
synced = long(line)
if synced < total:
percent = int(
round(
float(written + synced) / float(total) *
(end - start) + start))
report.progress_report(percent, "report")
synced = written
fpread.close()
return synced
for path in set(chain):
PATH = path
USER = cmd.dstUsername
IP = cmd.dstIp
PORT = (cmd.dstPort and cmd.dstPort or "22")
DIR = os.path.dirname(path)
_, _, err = bash_progress_1(
# Fixes ZSTAC-13430: handle extremely complex password like ~ ` !@#$%^&*()_+-=[]{}|?<>;:'"/ .
'rsync -av --progress --relative {{PATH}} --rsh="/usr/bin/sshpass -f{{PASSWORD_FILE}} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p {{PORT}} -l {{USER}}" {{IP}}:/ 1>{{PFILE}}',
_get_progress,
False)
if err:
linux.rm_file_force(PASSWORD_FILE)
linux.rm_file_force(PFILE)
raise Exception('fail to migrate vm to host, because %s' %
str(err))
written += os.path.getsize(path)
bash_errorout(
'/usr/bin/sshpass -f{{PASSWORD_FILE}} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p {{PORT}} {{USER}}@{{IP}} "/bin/sync {{PATH}}"'
)
percent = int(
round(float(written) / float(total) * (end - start) + start))
report.progress_report(percent, "report")
linux.rm_file_force(PASSWORD_FILE)
linux.rm_file_force(PFILE)
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def verify_backing_file_chain(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
for sp in cmd.snapshots:
if not os.path.exists(sp.path):
raise Exception('cannot find the file[%s]' % sp.path)
if sp.parentPath and not os.path.exists(sp.parentPath):
raise Exception('cannot find the backing file[%s]' %
sp.parentPath)
if sp.parentPath:
out = linux.qcow2_get_backing_file(sp.path)
if sp.parentPath != out:
raise Exception(
"resource[Snapshot or Volume, uuid:%s, path:%s]'s backing file[%s] is not equal to %s"
% (sp.snapshotUuid, sp.path, out, sp.parentPath))
return jsonobject.dumps(AgentResponse())
@kvmagent.replyerror
def rebase_backing_files(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
for sp in cmd.snapshots:
if sp.parentPath:
linux.qcow2_rebase_no_check(sp.parentPath, sp.path)
return jsonobject.dumps(AgentResponse())
@kvmagent.replyerror
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckBitsRsp()
rsp.existing = os.path.exists(cmd.path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_template_from_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
linux.create_template(cmd.volumePath, cmd.installPath)
logger.debug('successfully created template[%s] from volume[%s]' %
(cmd.installPath, cmd.volumePath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def revert_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotRsp()
install_path = cmd.snapshotInstallPath
new_volume_path = os.path.join(os.path.dirname(install_path),
'{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone_with_cmd(install_path, new_volume_path, cmd)
size = linux.qcow2_virtualsize(new_volume_path)
rsp.newVolumeInstallPath = new_volume_path
rsp.size = size
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def reinit_image(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ReinitImageRsp()
install_path = cmd.imagePath
dirname = os.path.dirname(cmd.volumePath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
new_volume_path = os.path.join(dirname,
'{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone_with_cmd(install_path, new_volume_path, cmd)
rsp.newVolumeInstallPath = new_volume_path
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotRsp()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
linux.create_template(cmd.snapshotInstallPath,
cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
cmd.workspaceInstallPath)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_and_rebase_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
snapshots = cmd.snapshotInstallPaths
count = len(snapshots)
for i in range(count):
if i + 1 < count:
target = snapshots[i]
backing_file = snapshots[i + 1]
linux.qcow2_rebase_no_check(backing_file, target)
latest = snapshots[0]
rsp = RebaseAndMergeSnapshotsRsp()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
linux.create_template(latest, cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
cmd.workspaceInstallPath)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def offline_merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
if not cmd.fullRebase:
linux.qcow2_rebase(cmd.srcPath, cmd.destPath)
else:
tmp = os.path.join(os.path.dirname(cmd.destPath),
'%s.qcow2' % uuidhelper.uuid())
linux.create_template(cmd.destPath, tmp)
shell.call("mv %s %s" % (tmp, cmd.destPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_physical_capacity(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def rebase_root_volume_to_backing_file(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
linux.qcow2_rebase_no_check(cmd.backingFilePath, cmd.rootVolumePath)
return jsonobject.dumps(AgentResponse())
@kvmagent.replyerror
def init(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
if not os.path.exists(cmd.path):
os.makedirs(cmd.path, 0755)
if cmd.initFilePath:
if not os.path.exists(cmd.initFilePath):
f = open(cmd.initFilePath, 'w')
f.close()
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_empty_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
try:
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname)
if cmd.backingFile:
linux.qcow2_create_with_backing_file_and_cmd(
cmd.backingFile, cmd.installUrl, cmd)
else:
linux.qcow2_create_with_cmd(cmd.installUrl, cmd.size, cmd)
except Exception as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = 'unable to create empty volume[uuid:%s, name:%s], %s' % (
cmd.volumeUuid, cmd.name, str(e))
rsp.success = False
return jsonobject.dumps(rsp)
logger.debug(
'successfully create empty volume[uuid:%s, size:%s] at %s' %
(cmd.volumeUuid, cmd.size, cmd.installUrl))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_root_volume_from_template(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
if not os.path.exists(cmd.templatePathInCache):
rsp.error = "unable to find image in cache"
rsp.success = False
logger.debug('error: %s: %s' %
(rsp.error, cmd.templatePathInCache))
return jsonobject.dumps(rsp)
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
linux.qcow2_clone_with_cmd(cmd.templatePathInCache, cmd.installUrl,
cmd)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def delete(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
kvmagent.deleteImage(cmd.path)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def deletedir(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
linux.rm_dir_checked(cmd.path)
logger.debug('successfully delete %s' % cmd.path)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
def upload():
if not os.path.exists(cmd.primaryStorageInstallPath):
raise kvmagent.KvmError('cannot find %s' %
cmd.primaryStorageInstallPath)
linux.scp_upload(cmd.hostname, cmd.sshKey,
cmd.primaryStorageInstallPath,
cmd.backupStorageInstallPath, cmd.username,
cmd.sshPort)
try:
upload()
except kvmagent.KvmError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(cmd, req)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
@kvmagent.replyerror
def download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
try:
linux.scp_download(cmd.hostname, cmd.sshKey,
cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath, cmd.username,
cmd.sshPort)
logger.debug('successfully download %s/%s to %s' %
(cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath))
except Exception as e:
content = traceback.format_exc()
logger.warn(content)
err = "unable to download %s/%s, because %s" % (
cmd.hostname, cmd.backupStorageInstallPath, str(e))
rsp.error = err
rsp.success = False
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def download_from_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
cachedir = None if cmd.isData else cmd.storagePath
self.imagestore_client.download_from_imagestore(
cachedir, cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath)
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.storagePath)
return jsonobject.dumps(rsp)
class LocalStoragePlugin(kvmagent.KvmAgent):
INIT_PATH = "/localstorage/init"
GET_PHYSICAL_CAPACITY_PATH = "/localstorage/getphysicalcapacity"
CREATE_EMPTY_VOLUME_PATH = "/localstorage/volume/createempty"
CREATE_VOLUME_FROM_CACHE_PATH = "/localstorage/volume/createvolumefromcache"
DELETE_BITS_PATH = "/localstorage/delete"
UPLOAD_BIT_PATH = "/localstorage/sftp/upload"
DOWNLOAD_BIT_PATH = "/localstorage/sftp/download"
UPLOAD_TO_IMAGESTORE_PATH = "/localstorage/imagestore/upload"
COMMIT_TO_IMAGESTORE_PATH = "/localstorage/imagestore/commit"
DOWNLOAD_FROM_IMAGESTORE_PATH = "/localstorage/imagestore/download"
REVERT_SNAPSHOT_PATH = "/localstorage/snapshot/revert"
MERGE_SNAPSHOT_PATH = "/localstorage/snapshot/merge"
MERGE_AND_REBASE_SNAPSHOT_PATH = "/localstorage/snapshot/mergeandrebase"
OFFLINE_MERGE_PATH = "/localstorage/snapshot/offlinemerge"
CREATE_TEMPLATE_FROM_VOLUME = "/localstorage/volume/createtemplate"
CHECK_BITS_PATH = "/localstorage/checkbits"
REBASE_ROOT_VOLUME_TO_BACKING_FILE_PATH = "/localstorage/volume/rebaserootvolumetobackingfile"
VERIFY_SNAPSHOT_CHAIN_PATH = "/localstorage/snapshot/verifychain"
REBASE_SNAPSHOT_BACKING_FILES_PATH = "/localstorage/snapshot/rebasebackingfiles"
COPY_TO_REMOTE_BITS_PATH = "/localstorage/copytoremote"
GET_MD5_PATH = "/localstorage/getmd5"
CHECK_MD5_PATH = "/localstorage/checkmd5"
GET_BACKING_FILE_PATH = "/localstorage/volume/getbackingfile"
GET_VOLUME_SIZE = "/localstorage/volume/getsize"
GET_BASE_IMAGE_PATH = "/localstorage/volume/getbaseimagepath"
GET_QCOW2_REFERENCE = "/localstorage/getqcow2reference"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.INIT_PATH, self.init)
http_server.register_async_uri(self.GET_PHYSICAL_CAPACITY_PATH,
self.get_physical_capacity)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH,
self.create_empty_volume)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH,
self.create_root_volume_from_template)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete)
http_server.register_async_uri(self.DOWNLOAD_BIT_PATH,
self.download_from_sftp)
http_server.register_async_uri(self.UPLOAD_BIT_PATH,
self.upload_to_sftp)
http_server.register_async_uri(self.UPLOAD_TO_IMAGESTORE_PATH,
self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_TO_IMAGESTORE_PATH,
self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_FROM_IMAGESTORE_PATH,
self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_SNAPSHOT_PATH,
self.revert_snapshot)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH,
self.merge_snapshot)
http_server.register_async_uri(self.MERGE_AND_REBASE_SNAPSHOT_PATH,
self.merge_and_rebase_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_PATH,
self.offline_merge_snapshot)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME,
self.create_template_from_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(
self.REBASE_ROOT_VOLUME_TO_BACKING_FILE_PATH,
self.rebase_root_volume_to_backing_file)
http_server.register_async_uri(self.VERIFY_SNAPSHOT_CHAIN_PATH,
self.verify_backing_file_chain)
http_server.register_async_uri(self.REBASE_SNAPSHOT_BACKING_FILES_PATH,
self.rebase_backing_files)
http_server.register_async_uri(self.COPY_TO_REMOTE_BITS_PATH,
self.copy_bits_to_remote)
http_server.register_async_uri(self.GET_MD5_PATH, self.get_md5)
http_server.register_async_uri(self.CHECK_MD5_PATH, self.check_md5)
http_server.register_async_uri(self.GET_BACKING_FILE_PATH,
self.get_backing_file_path)
http_server.register_async_uri(self.GET_VOLUME_SIZE,
self.get_volume_size)
http_server.register_async_uri(self.GET_BASE_IMAGE_PATH,
self.get_volume_base_image_path)
http_server.register_async_uri(self.GET_QCOW2_REFERENCE,
self.get_qcow2_reference)
self.path = None
self.imagestore_client = ImageStoreClient()
def stop(self):
pass
@kvmagent.replyerror
def get_qcow2_reference(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
out = shell.call('find %s -type f' % cmd.searchingDir)
rsp = GetQCOW2ReferenceRsp()
rsp.referencePaths = []
for f in out.split('\n'):
f = f.strip(' \t\r\n')
if not f: continue
backing_file = shell.call(
"qemu-img info %s | grep 'backing file:' | cut -d ':' -f 2" %
f)
backing_file = backing_file.strip(' \t\r\n')
if backing_file == cmd.path:
rsp.referencePaths.append(f)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_volume_base_image_path(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeBaseImagePathRsp()
rsp.path = linux.get_qcow2_base_image_path_recusively(cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_backing_file_path(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
out = shell.call(
"qemu-img info %s | grep 'backing file:' | cut -d ':' -f 2" %
cmd.path)
out = out.strip(' \t\r\n')
rsp = GetBackingFileRsp()
if out:
rsp.backingFilePath = out
rsp.size = os.path.getsize(out)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_md5(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetMd5Rsp()
rsp.md5s = []
for to in cmd.md5s:
md5 = shell.call("md5sum %s | cut -d ' ' -f 1" % to.path)
rsp.md5s.append({
'resourceUuid': to.resourceUuid,
'path': to.path,
'md5': md5
})
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def check_md5(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
for to in cmd.md5s:
dst_md5 = shell.call("md5sum %s | cut -d ' ' -f 1" % to.path)
if dst_md5 != to.md5:
raise Exception(
"MD5 unmatch. The file[uuid:%s, path:%s]'s md5 (src host:%s, dst host:%s)"
% (to.resourceUuid, to.path, to.md5, dst_md5))
rsp = AgentResponse()
return jsonobject.dumps(rsp)
def _get_disk_capacity(self):
return linux.get_disk_capacity_by_df(self.path)
@kvmagent.replyerror
def copy_bits_to_remote(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
for path in cmd.paths:
shell.call(
'rsync -a --relative %s --rsh="/usr/bin/sshpass -p %s ssh -o StrictHostKeyChecking=no -l %s" %s:/'
% (path, cmd.dstPassword, cmd.dstUsername, cmd.dstIp))
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def verify_backing_file_chain(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
for sp in cmd.snapshots:
if not os.path.exists(sp.path):
raise Exception('cannot find the file[%s]' % sp.path)
if sp.parentPath and not os.path.exists(sp.parentPath):
raise Exception('cannot find the backing file[%s]' %
sp.parentPath)
if sp.parentPath:
out = shell.call(
"qemu-img info %s | grep 'backing file' | cut -d ':' -f 2"
% sp.path)
out = out.strip(' \t\r\n')
if sp.parentPath != out:
raise Exception(
"resource[Snapshot or Volume, uuid:%s, path:%s]'s backing file[%s] is not equal to %s"
% (sp.snapshotUuid, sp.path, out, sp.parentPath))
return jsonobject.dumps(AgentResponse())
@kvmagent.replyerror
def rebase_backing_files(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
for sp in cmd.snapshots:
if sp.parentPath:
linux.qcow2_rebase_no_check(sp.parentPath, sp.path)
return jsonobject.dumps(AgentResponse())
@kvmagent.replyerror
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckBitsRsp()
rsp.existing = os.path.exists(cmd.path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_template_from_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
linux.qcow2_create_template(cmd.volumePath, cmd.installPath)
logger.debug('successfully created template[%s] from volume[%s]' %
(cmd.installPath, cmd.volumePath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def revert_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotRsp()
install_path = cmd.snapshotInstallPath
new_volume_path = os.path.join(os.path.dirname(install_path),
'{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone(install_path, new_volume_path)
rsp.newVolumeInstallPath = new_volume_path
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotRsp()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
linux.qcow2_create_template(cmd.snapshotInstallPath,
cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
cmd.workspaceInstallPath)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_and_rebase_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
snapshots = cmd.snapshotInstallPaths
count = len(snapshots)
for i in range(count):
if i + 1 < count:
target = snapshots[i]
backing_file = snapshots[i + 1]
linux.qcow2_rebase_no_check(backing_file, target)
latest = snapshots[0]
rsp = RebaseAndMergeSnapshotsRsp()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
linux.qcow2_create_template(latest, cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
cmd.workspaceInstallPath)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def offline_merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
if not cmd.fullRebase:
linux.qcow2_rebase(cmd.srcPath, cmd.destPath)
else:
tmp = os.path.join(os.path.dirname(cmd.destPath),
'%s.qcow2' % uuidhelper.uuid())
linux.qcow2_create_template(cmd.destPath, tmp)
shell.call("mv %s %s" % (tmp, cmd.destPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_physical_capacity(self, req):
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def rebase_root_volume_to_backing_file(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
linux.qcow2_rebase_no_check(cmd.backingFilePath, cmd.rootVolumePath)
return jsonobject.dumps(AgentResponse())
@kvmagent.replyerror
def init(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.path = cmd.path
if not os.path.exists(self.path):
os.makedirs(self.path, 0755)
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_empty_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
try:
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname)
if cmd.backingFile:
linux.qcow2_create_with_backing_file(cmd.backingFile,
cmd.installUrl)
else:
linux.qcow2_create(cmd.installUrl, cmd.size)
except Exception as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = 'unable to create empty volume[uuid:%s, name:%s], %s' % (
cmd.uuid, cmd.name, str(e))
rsp.success = False
return jsonobject.dumps(rsp)
logger.debug(
'successfully create empty volume[uuid:%s, size:%s] at %s' %
(cmd.volumeUuid, cmd.size, cmd.installUrl))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_root_volume_from_template(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
if not os.path.exists(cmd.templatePathInCache):
rsp.error = "UNABLE_TO_FIND_IMAGE_IN_CACHE"
rsp.success = False
logger.debug('error: %s: %s' %
(rsp.error, cmd.templatePathInCache))
return jsonobject.dumps(rsp)
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
linux.qcow2_clone(cmd.templatePathInCache, cmd.installUrl)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def delete(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
shell.call('rm -f %s' % cmd.path)
pdir = os.path.dirname(cmd.path)
linux.rmdir_if_empty(pdir)
logger.debug('successfully delete %s' % cmd.path)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
def upload():
if not os.path.exists(cmd.primaryStorageInstallPath):
raise kvmagent.KvmError('cannot find %s' %
cmd.primaryStorageInstallPath)
linux.scp_upload(cmd.hostname, cmd.sshKey,
cmd.primaryStorageInstallPath,
cmd.backupStorageInstallPath, cmd.username,
cmd.sshPort)
try:
upload()
except kvmagent.KvmError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(
cmd.hostname, cmd.primaryStorageInstallPath)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(
cmd.primaryStorageInstallPath)
@kvmagent.replyerror
def download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
try:
linux.scp_download(cmd.hostname, cmd.sshKey,
cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath, cmd.username,
cmd.sshPort)
logger.debug('successfully download %s/%s to %s' %
(cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath))
except Exception as e:
content = traceback.format_exc()
logger.warn(content)
err = "unable to download %s/%s, because %s" % (
cmd.hostname, cmd.backupStorageInstallPath, str(e))
rsp.error = err
rsp.success = False
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def download_from_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.imagestore_client.download_from_imagestore(
self.path, cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath)
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
class SharedMountPointPrimaryStoragePlugin(kvmagent.KvmAgent):
CONNECT_PATH = "/sharedmountpointprimarystorage/connect"
CREATE_VOLUME_FROM_CACHE_PATH = "/sharedmountpointprimarystorage/createrootvolume"
DELETE_BITS_PATH = "/sharedmountpointprimarystorage/bits/delete"
GET_SUBPATH_PATH = "/sharedmountpointprimarystorage/sub/path"
CREATE_TEMPLATE_FROM_VOLUME_PATH = "/sharedmountpointprimarystorage/createtemplatefromvolume"
UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH = "/sharedmountpointprimarystorage/sftp/upload"
DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH = "/sharedmountpointprimarystorage/sftp/download"
UPLOAD_BITS_TO_IMAGESTORE_PATH = "/sharedmountpointprimarystorage/imagestore/upload"
COMMIT_BITS_TO_IMAGESTORE_PATH = "/sharedmountpointprimarystorage/imagestore/commit"
DOWNLOAD_BITS_FROM_IMAGESTORE_PATH = "/sharedmountpointprimarystorage/imagestore/download"
REVERT_VOLUME_FROM_SNAPSHOT_PATH = "/sharedmountpointprimarystorage/volume/revertfromsnapshot"
MERGE_SNAPSHOT_PATH = "/sharedmountpointprimarystorage/snapshot/merge"
OFFLINE_MERGE_SNAPSHOT_PATH = "/sharedmountpointprimarystorage/snapshot/offlinemerge"
CREATE_EMPTY_VOLUME_PATH = "/sharedmountpointprimarystorage/volume/createempty"
CHECK_BITS_PATH = "/sharedmountpointprimarystorage/bits/check"
GET_VOLUME_SIZE_PATH = "/sharedmountpointprimarystorage/volume/getsize"
RESIZE_VOLUME_PATH = "/sharedmountpointprimarystorage/volume/resize"
REINIT_IMAGE_PATH = "/sharedmountpointprimarystorage/volume/reinitimage"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.CONNECT_PATH, self.connect)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH, self.create_root_volume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete_bits)
http_server.register_async_uri(self.GET_SUBPATH_PATH, self.get_sub_path)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH, self.create_template_from_volume)
http_server.register_async_uri(self.UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH, self.upload_to_sftp)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH, self.download_from_sftp)
http_server.register_async_uri(self.UPLOAD_BITS_TO_IMAGESTORE_PATH, self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_BITS_TO_IMAGESTORE_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_IMAGESTORE_PATH, self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH, self.revert_volume_from_snapshot)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.merge_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_SNAPSHOT_PATH, self.offline_merge_snapshots)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.create_empty_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH, self.get_volume_size)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH, self.resize_volume)
http_server.register_async_uri(self.REINIT_IMAGE_PATH, self.reinit_image)
self.imagestore_client = ImageStoreClient()
self.id_files = {}
def stop(self):
pass
@kvmagent.replyerror
def resize_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
install_path = cmd.installPath
rsp = ResizeVolumeRsp()
shell.call("qemu-img resize %s %s" % (install_path, cmd.size))
ret = linux.qcow2_virtualsize(install_path)
rsp.size = ret
return jsonobject.dumps(rsp)
@staticmethod
def _get_disk_capacity(mount_point):
if not mount_point:
raise Exception('storage mount point cannot be None')
return linux.get_disk_capacity_by_df(mount_point)
@kvmagent.replyerror
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def connect(self, req):
none_shared_mount_fs_type = ['xfs', 'ext2', 'ext3', 'ext4', 'vfat', 'tmpfs', 'btrfs']
cmd = jsonobject.loads(req[http.REQUEST_BODY])
if not linux.timeout_isdir(cmd.mountPoint):
raise kvmagent.KvmError('%s is not a directory, the mount point seems not setup' % cmd.mountPoint)
folder_fs_type = shell.call("df -T %s|tail -1|awk '{print $2}'" % cmd.mountPoint).strip()
if folder_fs_type in none_shared_mount_fs_type:
raise kvmagent.KvmError(
'%s filesystem is %s, which is not a shared mount point type.' % (cmd.mountPoint, folder_fs_type))
id_dir = os.path.join(cmd.mountPoint, "zstack_smp_id_file")
shell.call("mkdir -p %s" % id_dir)
lock_file = os.path.join(id_dir, "uuid.lock")
@lock.file_lock(lock_file, locker=lock.Flock())
def check_other_smp_and_set_id_file(uuid, existUuids):
o = shell.ShellCmd('''\
ls %s | grep -v %s | grep -o "[0-9a-f]\{8\}[0-9a-f]\{4\}[1-5][0-9a-f]\{3\}[89ab][0-9a-f]\{3\}[0-9a-f]\{12\}"\
''' % (id_dir, uuid))
o(False)
if o.return_code != 0:
file_uuids = []
else:
file_uuids = o.stdout.splitlines()
for file_uuid in file_uuids:
if file_uuid in existUuids:
raise Exception(
"the mount point [%s] has been occupied by other SMP[uuid:%s], Please attach this directly"
% (cmd.mountPoint, file_uuid))
logger.debug("existing id files: %s" % file_uuids)
self.id_files[uuid] = os.path.join(id_dir, uuid)
if not os.path.exists(self.id_files[uuid]):
# check if hosts in the same cluster mount the same path but different storages.
rsp.isFirst = True
for file_uuid in file_uuids:
linux.rm_file_force(os.path.join(id_dir, file_uuid))
linux.touch_file(self.id_files[uuid])
linux.sync()
rsp = ConnectRsp()
check_other_smp_and_set_id_file(cmd.uuid, cmd.existUuids)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_root_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if not os.path.exists(cmd.templatePathInCache):
rsp.error = "unable to find image in cache"
rsp.success = False
return jsonobject.dumps(rsp)
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
linux.qcow2_clone_with_cmd(cmd.templatePathInCache, cmd.installPath, cmd)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def delete_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if cmd.folder:
linux.rm_dir_checked(cmd.path)
else:
kvmagent.deleteImage(cmd.path)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_sub_path(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetSubPathRsp()
rsp.paths = kvmagent.listPath(cmd.path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_template_from_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
linux.create_template(cmd.volumePath, cmd.installPath)
logger.debug('successfully created template[%s] from volume[%s]' % (cmd.installPath, cmd.volumePath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
def upload():
if not os.path.exists(cmd.primaryStorageInstallPath):
raise kvmagent.KvmError('cannot find %s' % cmd.primaryStorageInstallPath)
linux.scp_upload(cmd.hostname, cmd.sshKey, cmd.primaryStorageInstallPath, cmd.backupStorageInstallPath, cmd.username, cmd.sshPort)
upload()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
linux.scp_download(cmd.hostname, cmd.sshKey, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath, cmd.username, cmd.sshPort)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
logger.debug('successfully download %s/%s to %s' % (cmd.hostname, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(cmd, req)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
@kvmagent.replyerror
def download_from_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
cachedir = None if cmd.isData else cmd.mountPoint
self.imagestore_client.download_from_imagestore(cachedir, cmd.hostname, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath)
rsp = AgentRsp()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def reinit_image(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ReinitImageRsp()
install_path = cmd.imageInstallPath
dirname = os.path.dirname(cmd.volumeInstallPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
new_volume_path = os.path.join(dirname, '{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone_with_cmd(install_path, new_volume_path, cmd)
rsp.newVolumeInstallPath = new_volume_path
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def revert_volume_from_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotRsp()
install_path = cmd.snapshotInstallPath
new_volume_path = os.path.join(os.path.dirname(install_path), '{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone_with_cmd(install_path, new_volume_path, cmd)
size = linux.qcow2_virtualsize(new_volume_path)
rsp.newVolumeInstallPath = new_volume_path
rsp.size = size
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotRsp()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
linux.create_template(cmd.snapshotInstallPath, cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.workspaceInstallPath)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def offline_merge_snapshots(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if not cmd.fullRebase:
linux.qcow2_rebase(cmd.srcPath, cmd.destPath)
else:
tmp = os.path.join(os.path.dirname(cmd.destPath), '%s.qcow2' % uuidhelper.uuid())
linux.create_template(cmd.destPath, tmp)
shell.call("mv %s %s" % (tmp, cmd.destPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_empty_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname)
if cmd.backingFile:
linux.qcow2_create_with_backing_file_and_cmd(cmd.backingFile, cmd.installPath, cmd)
else:
linux.qcow2_create_with_cmd(cmd.installPath, cmd.size, cmd)
logger.debug('successfully create empty volume[uuid:%s, size:%s] at %s' % (cmd.volumeUuid, cmd.size, cmd.installPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckBitsRsp()
rsp.existing = os.path.exists(cmd.path)
return jsonobject.dumps(rsp)
class ZsesStoragePlugin(kvmagent.KvmAgent):
INIT_PATH = "/zses/init"
GET_PHYSICAL_CAPACITY_PATH = "/zses/getphysicalcapacity"
CREATE_EMPTY_VOLUME_PATH = "/zses/volume/createempty"
CREATE_VOLUME_FROM_CACHE_PATH = "/zses/volume/createvolumefromcache"
DELETE_BITS_PATH = "/zses/delete"
DELETE_DIR_PATH = "/zses/deletedir"
UPLOAD_TO_IMAGESTORE_PATH = "/zses/imagestore/upload"
COMMIT_TO_IMAGESTORE_PATH = "/zses/imagestore/commit"
DOWNLOAD_FROM_IMAGESTORE_PATH = "/zses/imagestore/download"
REVERT_SNAPSHOT_PATH = "/zses/snapshot/revert"
MERGE_SNAPSHOT_PATH = "/zses/snapshot/merge"
MERGE_AND_REBASE_SNAPSHOT_PATH = "/zses/snapshot/mergeandrebase"
OFFLINE_MERGE_PATH = "/zses/snapshot/offlinemerge"
CREATE_TEMPLATE_FROM_VOLUME = "/zses/volume/createtemplate"
CHECK_BITS_PATH = "/zses/checkbits"
REBASE_ROOT_VOLUME_TO_BACKING_FILE_PATH = "/zses/volume/rebaserootvolumetobackingfile"
VERIFY_SNAPSHOT_CHAIN_PATH = "/zses/snapshot/verifychain"
REBASE_SNAPSHOT_BACKING_FILES_PATH = "/zses/snapshot/rebasebackingfiles"
COPY_TO_REMOTE_BITS_PATH = "/zses/copytoremote"
GET_MD5_PATH = "/zses/getmd5"
CHECK_MD5_PATH = "/zses/checkmd5"
GET_BACKING_FILE_PATH = "/zses/volume/getbackingfile"
GET_VOLUME_SIZE = "/zses/volume/getsize"
GET_BASE_IMAGE_PATH = "/zses/volume/getbaseimagepath"
GET_QCOW2_REFERENCE = "/zses/getqcow2reference"
CONVERT_QCOW2_TO_RAW = "/zses/imagestore/convert/raw"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.INIT_PATH, self.init)
http_server.register_async_uri(self.GET_PHYSICAL_CAPACITY_PATH, self.get_physical_capacity)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.create_empty_volume)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH, self.create_root_volume_from_template)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete)
http_server.register_async_uri(self.DELETE_DIR_PATH, self.deletedir)
http_server.register_async_uri(self.UPLOAD_TO_IMAGESTORE_PATH, self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_TO_IMAGESTORE_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_FROM_IMAGESTORE_PATH, self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_SNAPSHOT_PATH, self.revert_snapshot)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.merge_snapshot)
http_server.register_async_uri(self.MERGE_AND_REBASE_SNAPSHOT_PATH, self.merge_and_rebase_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_PATH, self.offline_merge_snapshot)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME, self.create_template_from_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.REBASE_ROOT_VOLUME_TO_BACKING_FILE_PATH, self.rebase_root_volume_to_backing_file)
http_server.register_async_uri(self.VERIFY_SNAPSHOT_CHAIN_PATH, self.verify_backing_file_chain)
http_server.register_async_uri(self.REBASE_SNAPSHOT_BACKING_FILES_PATH, self.rebase_backing_files)
http_server.register_async_uri(self.COPY_TO_REMOTE_BITS_PATH, self.copy_bits_to_remote)
http_server.register_async_uri(self.GET_MD5_PATH, self.get_md5)
http_server.register_async_uri(self.CHECK_MD5_PATH, self.check_md5)
http_server.register_async_uri(self.GET_BACKING_FILE_PATH, self.get_backing_file_path)
http_server.register_async_uri(self.GET_VOLUME_SIZE, self.get_volume_size)
http_server.register_async_uri(self.GET_BASE_IMAGE_PATH, self.get_volume_base_image_path)
http_server.register_async_uri(self.GET_QCOW2_REFERENCE, self.get_qcow2_reference)
http_server.register_async_uri(self.CONVERT_QCOW2_TO_RAW, self.convert_qcow2_to_raw)
self.imagestore_client = ImageStoreClient()
def stop(self):
pass
@kvmagent.replyerror
def convert_qcow2_to_raw(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.convert_image_raw(cmd)
@kvmagent.replyerror
def get_qcow2_reference(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
out = shell.call('find %s -type f' % cmd.searchingDir)
rsp = GetQCOW2ReferenceRsp()
rsp.referencePaths = []
real_path = os.path.realpath(cmd.path)
for f in out.splitlines():
backing_file = linux.qcow2_get_backing_file(f)
if os.path.realpath(backing_file) == real_path:
rsp.referencePaths.append(f)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_volume_base_image_path(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeBaseImagePathRsp()
if not os.path.basename(cmd.volumeInstallDir).endswith(cmd.volumeUuid):
raise Exception('maybe you pass a wrong install dir')
path = linux.get_qcow2_base_image_recusively(cmd.volumeInstallDir, cmd.imageCacheDir)
if not path:
return jsonobject.dumps(rsp)
rsp.path = path
rsp.size = linux.get_qcow2_file_chain_size(path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_backing_file_path(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
out = linux.qcow2_get_backing_file(cmd.path)
rsp = GetBackingFileRsp()
if out:
rsp.backingFilePath = out
rsp.size = os.path.getsize(out)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_md5(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetMd5Rsp()
rsp.md5s = []
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
report = Report(cmd.threadContext, cmd.threadContextStack)
report.processType = "LocalStorageMigrateVolume"
PFILE = shell.call('mktemp /tmp/tmp-XXXXXX').strip()
total = 0
written = 0
for to in cmd.md5s:
total = total + os.path.getsize(to.path)
start = 0
end = 10
if cmd.stage:
start, end = get_scale(cmd.stage)
def _get_progress(synced):
logger.debug("getProgress in get_md5")
if not os.path.exists(PFILE):
return synced
last = linux.tail_1(PFILE).strip()
if not last or not last.isdigit():
return synced
percent = int(round((float(written) * 100 + os.path.getsize(to.path) * float(last)) / total * (end - start) / 100) + start)
report.progress_report(str(percent), "report")
return synced
report.resourceUuid = cmd.volumeUuid
if start == 0:
report.progress_report("0", "start")
else:
report.progress_report(str(start), "report")
for to in cmd.md5s:
_, md5, _ = bash_progress_1("pv -n %s 2>%s | md5sum | cut -d ' ' -f 1" % (to.path, PFILE), _get_progress)
rsp.md5s.append({
'resourceUuid': to.resourceUuid,
'path': to.path,
'md5': md5
})
written += os.path.getsize(to.path)
percent = int(round(float(written) / float(total) * (end - start) + start))
report.progress_report(percent, "report")
if os.path.exists(PFILE):
os.remove(PFILE)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def check_md5(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
report = Report(cmd.threadContext, cmd.threadContextStack)
report.processType = "LocalStorageMigrateVolume"
PFILE = shell.call('mktemp /tmp/tmp-XXXXXX').strip()
total = 0
written = 0
start = 90
end = 100
if cmd.stage:
start, end = get_scale(cmd.stage)
for to in cmd.md5s:
total = total + os.path.getsize(to.path)
def _get_progress(synced):
logger.debug("getProgress in check_md5")
if not os.path.exists(PFILE):
return synced
last = linux.tail_1(PFILE).strip()
if not last or not last.isdigit():
return synced
percent = int(round((float(written) * 100 + os.path.getsize(to.path) * float(last)) / total * (end - start) / 100) + start)
report.progress_report(percent, "report")
return synced
report.resourceUuid = cmd.volumeUuid
for to in cmd.md5s:
_, dst_md5, _ = bash_progress_1("pv -n %s 2>%s | md5sum | cut -d ' ' -f 1" % (to.path, PFILE), _get_progress)
if dst_md5 != to.md5:
raise Exception("MD5 unmatch. The file[uuid:%s, path:%s]'s md5 (src host:%s, dst host:%s)" %
(to.resourceUuid, to.path, to.md5, dst_md5))
written += os.path.getsize(to.path)
percent = int(round(float(written) / float(total) * (end - start) + start))
report.progress_report(percent, "report")
if os.path.exists(PFILE):
os.remove(PFILE)
rsp = AgentResponse()
if end == 100:
report.progress_report("100", "finish")
else:
report.progress_report(str(end), "report")
return jsonobject.dumps(rsp)
@staticmethod
def _get_disk_capacity(path):
if not path:
raise Exception('storage path cannot be None')
return linux.get_disk_capacity_by_df(path)
@kvmagent.replyerror
@in_bash
def copy_bits_to_remote(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
chain = sum([linux.qcow2_get_file_chain(p) for p in cmd.paths], [])
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
report = Report(cmd.threadContext, cmd.threadContextStack)
report.processType = "LocalStorageMigrateVolume"
report.resourceUuid = cmd.uuid
PFILE = shell.call('mktemp /tmp/tmp-XXXXXX').strip()
start = 10
end = 90
if cmd.stage:
start, end = get_scale(cmd.stage)
total = 0
for path in set(chain):
total = total + os.path.getsize(path)
written = 0
def _get_progress(synced):
logger.debug("getProgress in localstorage-agent, synced: %s, total: %s" % (synced, total))
if not os.path.exists(PFILE):
return synced
fpread = open(PFILE, 'r')
lines = fpread.readlines()
if not lines:
fpread.close()
return synced
last = str(lines[-1]).strip().split('\r')[-1]
if not last or len(last.split()) < 1:
fpread.close()
return synced
line = last.split()[0]
if not line.isdigit():
return synced
if total > 0:
synced = long(line)
if synced < total:
percent = int(round(float(written + synced) / float(total) * (end - start) + start))
report.progress_report(percent, "report")
synced = written
fpread.close()
return synced
for path in set(chain):
PATH = path
PASSWORD = linux.shellquote(cmd.dstPassword)
USER = cmd.dstUsername
IP = cmd.dstIp
PORT = (cmd.dstPort and cmd.dstPort or "22")
DIR = os.path.dirname(path)
if cmd.dstUsername == 'root':
_, _, err = bash_progress_1(
'rsync -av --progress --relative {{PATH}} --rsh="/usr/bin/sshpass -p {{PASSWORD}} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p {{PORT}} -l {{USER}}" {{IP}}:/ 1>{{PFILE}}', _get_progress, False)
if err:
raise Exception('fail to migrate vm to host, because %s' % str(err))
else:
raise Exception("cannot support migrate to non-root user host")
written += os.path.getsize(path)
bash_errorout('/usr/bin/sshpass -p {{PASSWORD}} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p {{PORT}} {{USER}}@{{IP}} "/bin/sync {{PATH}}"')
percent = int(round(float(written) / float(total) * (end - start) + start))
report.progress_report(percent, "report")
if os.path.exists(PFILE):
os.remove(PFILE)
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def verify_backing_file_chain(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
for sp in cmd.snapshots:
if not os.path.exists(sp.path):
raise Exception('cannot find the file[%s]' % sp.path)
if sp.parentPath and not os.path.exists(sp.parentPath):
raise Exception('cannot find the backing file[%s]' % sp.parentPath)
if sp.parentPath:
out = linux.qcow2_get_backing_file(sp.path)
if sp.parentPath != out:
raise Exception("resource[Snapshot or Volume, uuid:%s, path:%s]'s backing file[%s] is not equal to %s" %
(sp.snapshotUuid, sp.path, out, sp.parentPath))
return jsonobject.dumps(AgentResponse())
@kvmagent.replyerror
def rebase_backing_files(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
for sp in cmd.snapshots:
if sp.parentPath:
linux.qcow2_rebase_no_check(sp.parentPath, sp.path)
return jsonobject.dumps(AgentResponse())
@kvmagent.replyerror
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckBitsRsp()
rsp.existing = os.path.exists(cmd.path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_template_from_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
linux.create_template(cmd.volumePath, cmd.installPath)
logger.debug('successfully created template[%s] from volume[%s]' % (cmd.installPath, cmd.volumePath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def revert_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotRsp()
install_path = cmd.snapshotInstallPath
new_volume_path = os.path.join(os.path.dirname(install_path), '{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone_with_cmd(install_path, new_volume_path, cmd)
rsp.newVolumeInstallPath = new_volume_path
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotRsp()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
linux.create_template(cmd.snapshotInstallPath, cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.workspaceInstallPath)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_and_rebase_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
snapshots = cmd.snapshotInstallPaths
count = len(snapshots)
for i in range(count):
if i+1 < count:
target = snapshots[i]
backing_file = snapshots[i+1]
linux.qcow2_rebase_no_check(backing_file, target)
latest = snapshots[0]
rsp = RebaseAndMergeSnapshotsRsp()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
linux.create_template(latest, cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.workspaceInstallPath)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def offline_merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
if not cmd.fullRebase:
linux.qcow2_rebase(cmd.srcPath, cmd.destPath)
else:
tmp = os.path.join(os.path.dirname(cmd.destPath), '%s.qcow2' % uuidhelper.uuid())
linux.create_template(cmd.destPath, tmp)
shell.call("mv %s %s" % (tmp, cmd.destPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_physical_capacity(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def rebase_root_volume_to_backing_file(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
linux.qcow2_rebase_no_check(cmd.backingFilePath, cmd.rootVolumePath)
return jsonobject.dumps(AgentResponse())
@kvmagent.replyerror
def init(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
if not os.path.exists(cmd.path):
os.makedirs(cmd.path, 0755)
# make sure that the filesystem is mounted
shell.call("/usr/local/bin/zs-ess-is-mounted")
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_empty_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
try:
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname)
if cmd.backingFile:
linux.qcow2_create_with_backing_file_and_cmd(cmd.backingFile, cmd.installUrl, cmd)
else:
linux.qcow2_create_with_cmd(cmd.installUrl, cmd.size, cmd)
except Exception as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = 'unable to create empty volume[uuid:%s, name:%s], %s' % (cmd.uuid, cmd.name, str(e))
rsp.success = False
return jsonobject.dumps(rsp)
logger.debug('successfully create empty volume[uuid:%s, size:%s] at %s' % (cmd.volumeUuid, cmd.size, cmd.installUrl))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_root_volume_from_template(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
if not os.path.exists(cmd.templatePathInCache):
rsp.error = "unable to find image in cache"
rsp.success = False
logger.debug('error: %s: %s' % (rsp.error, cmd.templatePathInCache))
return jsonobject.dumps(rsp)
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
linux.qcow2_clone_with_cmd(cmd.templatePathInCache, cmd.installUrl, cmd)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def delete(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
kvmagent.deleteImage(cmd.path)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def deletedir(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
linux.rm_dir_checked(cmd.path)
logger.debug('successfully delete %s' % cmd.path)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.storagePath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(cmd, req)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
@kvmagent.replyerror
def download_from_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.imagestore_client.download_from_imagestore(cmd.storagePath, cmd.hostname, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath)
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.storagePath)
return jsonobject.dumps(rsp)
class SharedMountPointPrimaryStoragePlugin(kvmagent.KvmAgent):
CONNECT_PATH = "/sharedmountpointprimarystorage/connect"
CREATE_VOLUME_FROM_CACHE_PATH = "/sharedmountpointprimarystorage/createrootvolume"
DELETE_BITS_PATH = "/sharedmountpointprimarystorage/bits/delete"
CREATE_TEMPLATE_FROM_VOLUME_PATH = "/sharedmountpointprimarystorage/createtemplatefromvolume"
UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH = "/sharedmountpointprimarystorage/sftp/upload"
DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH = "/sharedmountpointprimarystorage/sftp/download"
UPLOAD_BITS_TO_IMAGESTORE_PATH = "/sharedmountpointprimarystorage/imagestore/upload"
COMMIT_BITS_TO_IMAGESTORE_PATH = "/sharedmountpointprimarystorage/imagestore/commit"
DOWNLOAD_BITS_FROM_IMAGESTORE_PATH = "/sharedmountpointprimarystorage/imagestore/download"
REVERT_VOLUME_FROM_SNAPSHOT_PATH = "/sharedmountpointprimarystorage/volume/revertfromsnapshot"
MERGE_SNAPSHOT_PATH = "/sharedmountpointprimarystorage/snapshot/merge"
OFFLINE_MERGE_SNAPSHOT_PATH = "/sharedmountpointprimarystorage/snapshot/offlinemerge"
CREATE_EMPTY_VOLUME_PATH = "/sharedmountpointprimarystorage/volume/createempty"
CHECK_BITS_PATH = "/sharedmountpointprimarystorage/bits/check"
GET_VOLUME_SIZE_PATH = "/sharedmountpointprimarystorage/volume/getsize"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.CONNECT_PATH, self.connect)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH,
self.create_root_volume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete_bits)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH,
self.create_template_from_volume)
http_server.register_async_uri(
self.UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH, self.upload_to_sftp)
http_server.register_async_uri(
self.DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH,
self.download_from_sftp)
http_server.register_async_uri(self.UPLOAD_BITS_TO_IMAGESTORE_PATH,
self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_BITS_TO_IMAGESTORE_PATH,
self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_IMAGESTORE_PATH,
self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH,
self.revert_volume_from_snapshot)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH,
self.merge_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_SNAPSHOT_PATH,
self.offline_merge_snapshots)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH,
self.create_empty_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH,
self.get_volume_size)
self.mount_point = None
self.imagestore_client = ImageStoreClient()
def stop(self):
pass
def _get_disk_capacity(self):
return linux.get_disk_capacity_by_df(self.mount_point)
@kvmagent.replyerror
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def connect(self, req):
none_shared_mount_fs_type = [
'xfs', 'ext2', 'ext3', 'ext4', 'vfat', 'tmpfs', 'btrfs'
]
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.mount_point = cmd.mountPoint
if not os.path.isdir(self.mount_point):
raise kvmagent.KvmError(
'%s is not a directory, the mount point seems not setup' %
self.mount_point)
folder_fs_type = shell.call("df -T %s|tail -1|awk '{print $2}'" %
self.mount_point).strip()
if folder_fs_type in none_shared_mount_fs_type:
raise kvmagent.KvmError(
'%s filesystem is %s, which is not a shared mount point type.'
% (self.mount_point, folder_fs_type))
rsp = AgentRsp()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_root_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if not os.path.exists(cmd.templatePathInCache):
rsp.error = "UNABLE_TO_FIND_IMAGE_IN_CACHE"
rsp.success = False
return jsonobject.dumps(rsp)
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
linux.qcow2_clone(cmd.templatePathInCache, cmd.installPath)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def delete_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
shell.call('rm -f %s' % cmd.path)
pdir = os.path.dirname(cmd.path)
linux.rmdir_if_empty(pdir)
logger.debug('successfully delete %s' % cmd.path)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_template_from_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
linux.qcow2_create_template(cmd.volumePath, cmd.installPath)
logger.debug('successfully created template[%s] from volume[%s]' %
(cmd.installPath, cmd.volumePath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
def upload():
if not os.path.exists(cmd.primaryStorageInstallPath):
raise kvmagent.KvmError('cannot find %s' %
cmd.primaryStorageInstallPath)
linux.scp_upload(cmd.hostname, cmd.sshKey,
cmd.primaryStorageInstallPath,
cmd.backupStorageInstallPath, cmd.username,
cmd.sshPort)
upload()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
linux.scp_download(cmd.hostname, cmd.sshKey,
cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath, cmd.username,
cmd.sshPort)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
logger.debug('successfully download %s/%s to %s' %
(cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(cmd, req)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
@kvmagent.replyerror
def download_from_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.imagestore_client.download_from_imagestore(
self.mount_point, cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath)
rsp = AgentRsp()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def revert_volume_from_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotRsp()
install_path = cmd.snapshotInstallPath
new_volume_path = os.path.join(os.path.dirname(install_path),
'{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone(install_path, new_volume_path)
rsp.newVolumeInstallPath = new_volume_path
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotRsp()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
linux.qcow2_create_template(cmd.snapshotInstallPath,
cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
cmd.workspaceInstallPath)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def offline_merge_snapshots(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if not cmd.fullRebase:
linux.qcow2_rebase(cmd.srcPath, cmd.destPath)
else:
tmp = os.path.join(os.path.dirname(cmd.destPath),
'%s.qcow2' % uuidhelper.uuid())
linux.qcow2_create_template(cmd.destPath, tmp)
shell.call("mv %s %s" % (tmp, cmd.destPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_empty_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname)
if cmd.backingFile:
linux.qcow2_create_with_backing_file(cmd.backingFile,
cmd.installPath)
else:
linux.qcow2_create(cmd.installPath, cmd.size)
logger.debug(
'successfully create empty volume[uuid:%s, size:%s] at %s' %
(cmd.volumeUuid, cmd.size, cmd.installPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckBitsRsp()
rsp.existing = os.path.exists(cmd.path)
return jsonobject.dumps(rsp)
class AliyunNasStoragePlugin(kvmagent.KvmAgent):
MOUNT_PATH = "/aliyun/nas/primarystorage/firstmount"
IS_MOUNT_PATH = "/aliyun/nas/primarystorage/ismount"
MOUNT_DATA_PATH = "/aliyun/nas/primarystorage/mountdata"
INIT_PATH = "/aliyun/nas/primarystorage/init"
PING_PATH = "/aliyun/nas/primarystorage/ping"
GET_CAPACITY_PATH = "/aliyun/nas/primarystorage/getcapacity"
LIST_PATH = "/aliyun/nas/primarystorage/list"
UPDATE_MOUNT_POINT_PATH = "/aliyun/nas/primarystorage/updatemountpoint"
REMOUNT_PATH = "/aliyun/nas/primarystorage/remount"
UNMOUNT_PATH = "/aliyun/nas/primarystorage/unmount"
CHECK_BITS_PATH = "/aliyun/nas/primarystorage/checkbits"
CREATE_EMPTY_VOLUME_PATH = "/aliyun/nas/primarystorage/createempty"
CREATE_VOLUME_FROM_CACHE_PATH = "/aliyun/nas/primarystorage/createvolume"
DELETE_BITS_PATH = "/aliyun/nas/primarystorage/deletebits"
GET_VOLUME_SIZE_PATH = "/aliyun/nas/primarystorage/getvolumesize"
REVERT_VOLUME_FROM_SNAPSHOT_PATH = "/aliyun/nas/primarystorage/revertvolume"
DOWNLOAD_BIT_TO_IMAGESTORE_PATH = "/aliyun/nas/primarystorage/imagestore/download"
UPLOAD_BIT_TO_IMAGESTORE__PATH = "/aliyun/nas/primarystorage/imagestore/upload"
REINIT_VOLUME_PATH = "/aliyun/nas/primarystorage/reinit"
RESIZE_VOLUME_PATH = "/aliyun/nas/primarystorage/resize"
COMMIT_PATH = "/aliyun/nas/primarystorage/commit"
CREATE_TEMPLATE_FROM_VOLUME_PATH = "/aliyun/nas/primarystorage/createtemplatefromvolume"
MERGE_SNAPSHOT_PATH = "/aliyun/nas/primarystorage/mergesnapshot"
OFFLINE_MERGE_SNAPSHOT_PATH = "/aliyun/nas/primarystorage/snapshot/offlinemerge"
CHECK_MOUNT_PATH = "/aliyun/nas/primarystorage/checkmount"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.MOUNT_PATH, self.mount)
http_server.register_async_uri(self.IS_MOUNT_PATH, self.ismount)
http_server.register_async_uri(self.MOUNT_DATA_PATH, self.mountdata)
http_server.register_async_uri(self.INIT_PATH, self.init)
http_server.register_async_uri(self.PING_PATH, self.ping)
http_server.register_async_uri(self.LIST_PATH, self.list)
http_server.register_async_uri(self.UPDATE_MOUNT_POINT_PATH, self.updateMount)
http_server.register_async_uri(self.REMOUNT_PATH, self.remount)
http_server.register_async_uri(self.UNMOUNT_PATH, self.umount)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.checkbits)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.createempty)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH, self.createvolume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.deletebits)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH, self.getvolumesize)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH, self.revertvolume)
http_server.register_async_uri(self.REINIT_VOLUME_PATH, self.reinit)
http_server.register_async_uri(self.UPLOAD_BIT_TO_IMAGESTORE__PATH, self.uploadtoimagestore)
http_server.register_async_uri(self.DOWNLOAD_BIT_TO_IMAGESTORE_PATH, self.downloadfromimagestore)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH, self.resize)
http_server.register_async_uri(self.COMMIT_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH, self.createtemplate)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.mergesnapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_SNAPSHOT_PATH, self.offlinemerge)
http_server.register_async_uri(self.GET_CAPACITY_PATH, self.getcapacity)
http_server.register_async_uri(self.CHECK_MOUNT_PATH, self.checkmountpath)
self.mount_path = {}
self.uuid = None
self.imagestore_client = ImageStoreClient()
def stop(self):
pass
def _set_capacity_to_response(self, uuid, rsp):
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(uuid)
def _get_disk_capacity(self, uuid):
path = self.mount_path.get(uuid)
if not path:
raise Exception('cannot find mount path of primary storage[uuid: %s]' % uuid)
return linux.get_disk_capacity_by_df(path)
@kvmagent.replyerror
def mount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
linux.is_valid_nfs_url(cmd.url)
if not naslinux.is_mounted(cmd.mountPath, cmd.url):
linux.mount(cmd.url, cmd.mountPath, cmd.options)
logger.debug(http.path_msg(self.MOUNT_PATH, 'mounted %s on %s' % (cmd.url, cmd.mountPath)))
rsp.mounted = True
self.mount_path[cmd.uuid] = cmd.mountPath
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def ismount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = IsMountResponse()
linux.is_valid_nfs_url(cmd.url)
if naslinux.is_mounted(cmd.mountPath, cmd.url):
rsp.ismounted = True
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def mountdata(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
naslinux.createCommonPath(cmd.mountPath, cmd.basePath)
if not naslinux.is_mounted(cmd.dataPath, cmd.url):
linux.mount(cmd.url, cmd.dataPath, cmd.options)
logger.debug(http.path_msg(self.MOUNT_DATA_PATH, 'mounted %s on %s' % (cmd.url, cmd.dataPath)))
rsp.mounted = True
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def init(self, req):
'''
cmd.url --> domain:/ps-[uuid]
cmd.mountPath --> /opt/ps
cmd.common --> /opt/ps/commons
cmd.data --> /opt/ps/datas
cmd.dirs --> []
'''
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = InitResponse()
linux.is_valid_nfs_url(cmd.url)
'''
example:
1. mount url: /opt/ps
2. mkdir /opt/ps/ps-[uuid]
3. mkdir /opt/ps/ps-[uuid]/commons/xxx.. (such as heartbeat, cache, ..)
at last we get url:/ps-[uuid] for hosts mount
'''
domain = cmd.url.split(':')[0] + ":/"
psDir = cmd.url.split(':')[1].lstrip('/')
basedir = os.path.join(cmd.mountPath, psDir)
'''
check if mounted {cmd.mountPath}
'''
if linux.is_mounted(path=cmd.mountPath) and not naslinux.is_mounted(cmd.mountPath, cmd.url):
raise Exception('mountPath[%s] already mount to another url' % cmd.mountPath)
linux.mount(domain, cmd.mountPath, cmd.options)
shell.call('mkdir -p %s' % basedir)
for dir in cmd.dirs:
shell.call('mkdir -p %s' % os.path.join(basedir, dir))
linux.umount(cmd.mountPath)
common_dir = os.path.join(cmd.mountPath, cmd.common)
data_dir = os.path.join(cmd.mountPath, cmd.data)
shell.call('mkdir -p %s' % common_dir)
shell.call('mkdir -p %s' % data_dir)
linux.mount(cmd.url, common_dir, cmd.options)
rsp.mounted = True
self.mount_path[cmd.uuid] = common_dir
self._set_capacity_to_response(cmd.uuid, rsp)
self.uuid = cmd.uuid
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@in_bash
def ping(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
mount_path = cmd.mountPath
# if nfs service stop, os.path.isdir will hung
if not linux.timeout_isdir(mount_path) or not linux.is_mounted(path=mount_path):
raise Exception(
'the mount path[%s] of the nas primary storage[uuid:%s] is not existing' % (mount_path, cmd.uuid))
test_file = os.path.join(mount_path, cmd.heartbeat, '%s-ping-test-file' % cmd.uuid)
touch = shell.ShellCmd('timeout 60 touch %s' % test_file)
touch(False)
if touch.return_code == 124:
raise Exception('unable to access the mount path[%s] of the nas primary storage[uuid:%s] in 60s, timeout' %
(mount_path, cmd.uuid))
elif touch.return_code != 0:
touch.raise_error()
linux.rm_file_force(test_file)
return jsonobject.dumps(AliyunNasResponse())
@kvmagent.replyerror
def getvolumesize(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
self._set_capacity_to_response(cmd.uuid, rsp)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def list(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ListResponse()
rsp.paths = kvmagent.listPath(cmd.path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def updateMount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
linux.is_valid_nfs_url(cmd.newUrl)
if not naslinux.is_mounted(cmd.mountPath, cmd.newUrl):
# umount old one
if naslinux.is_mounted(cmd.mountPath, cmd.url):
linux.umount(cmd.mountPath)
# mount new
linux.mount(cmd.newUrl, cmd.mountPath, cmd.options)
self.mount_path[cmd.uuid] = cmd.mountPath
logger.debug('updated the mount path[%s] mounting point from %s to %s' % (
cmd.mountPath, cmd.url, cmd.newUrl))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def remount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
linux.is_valid_nfs_url(cmd.url)
naslinux.remount(cmd.url, cmd.mountPath, cmd.options)
self.mount_path[cmd.uuid] = cmd.mountPath
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def umount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
if naslinux.is_mounted(path=cmd.mountPath):
ret = linux.umount(cmd.mountPath)
if not ret:
logger.warn(http.path_msg(self.UNMOUNT_PATH, 'unmount %s failed' % cmd.mountPath))
logger.debug(http.path_msg(self.UNMOUNT_PATH, 'umounted %s' % cmd.mountPath))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def checkbits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckIsBitsExistingResponse()
rsp.existing = os.path.exists(cmd.path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def createempty(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname)
if cmd.backingFile:
linux.qcow2_create_with_backing_file_and_cmd(cmd.backingFile, cmd.installPath, cmd)
else:
linux.qcow2_create_with_cmd(cmd.installPath, cmd.size, cmd)
logger.debug(
'successfully create empty volume[uuid:%s, size:%s] at %s' % (cmd.volumeUuid, cmd.size, cmd.installPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.uuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def createvolume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
if not os.path.exists(cmd.templatePathInCache):
rsp.error = "unable to find image in cache"
rsp.success = False
return jsonobject.dumps(rsp)
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
linux.qcow2_clone_with_cmd(cmd.templatePathInCache, cmd.installPath, cmd)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.uuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def deletebits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
self.delNasBits(cmd.folder, cmd.path)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.uuid)
return jsonobject.dumps(rsp)
def delNasBits(self, folder, path):
if folder:
linux.rm_dir_checked(path)
else:
kvmagent.deleteImage(path)
# pdir = os.path.dirname(path)
# if os.path.exists(pdir) or not os.listdir(pdir):
# linux.umount(pdir, False)
@kvmagent.replyerror
def revertvolume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotRsp()
install_path = cmd.snapshotInstallPath
new_volume_path = os.path.join(os.path.dirname(install_path), '{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone_with_cmd(install_path, new_volume_path, cmd)
size = linux.qcow2_virtualsize(new_volume_path)
rsp.newVolumeInstallPath = new_volume_path
rsp.size = size
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def reinit(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ReInitImageRsp()
install_path = cmd.imagePath
new_volume_path = os.path.join(os.path.dirname(cmd.volumePath), '{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone_with_cmd(install_path, new_volume_path, cmd)
rsp.newVolumeInstallPath = new_volume_path
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.uuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def uploadtoimagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(cmd, req)
@kvmagent.replyerror
def downloadfromimagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.imagestore_client.download_from_imagestore(cmd.cacheDir, cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath)
rsp = AliyunNasResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.uuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def resize(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
install_path = cmd.installPath
rsp = ResizeVolumeRsp()
shell.call("qemu-img resize %s %s" % (install_path, cmd.size))
ret = linux.qcow2_virtualsize(install_path)
rsp.size = ret
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
@kvmagent.replyerror
def createtemplate(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
linux.create_template(cmd.volumePath, cmd.installPath)
logger.debug('successfully created template[%s] from volume[%s]' % (cmd.installPath, cmd.volumePath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.uuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def mergesnapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotRsp()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
linux.create_template(cmd.snapshotInstallPath, cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.workspaceInstallPath)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.uuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def offlinemerge(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
if not cmd.fullRebase:
linux.qcow2_rebase(cmd.srcPath, cmd.destPath)
else:
tmp = os.path.join(os.path.dirname(cmd.destPath), '%s.qcow2' % uuidhelper.uuid())
linux.create_template(cmd.destPath, tmp)
shell.call("mv %s %s" % (tmp, cmd.destPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.uuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def getcapacity(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
def findMountUrl(self, mount):
if ' on ' not in mount:
raise Exception("' on ' not in mount: %s" % mount)
return mount.split(' on ')[0].strip()
def findMountPath(self, mount):
if ' on ' not in mount:
raise Exception("' on ' not in mount: %s" % mount)
tmp = mount.split(' on ')[1]
return tmp.split(' ')[0].strip()
def findMountInfo(self, mount):
if ' on ' not in mount:
raise Exception("' on ' not in mount: %s" % mount)
tmp = mount.split(' on ')[1]
if ' (' in tmp:
return '(' + tmp.split(' (')[1].strip()
else:
return None
@kvmagent.replyerror
def checkmountpath(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckMountPathRsp()
mounts = naslinux.getMountInfo(cmd.psUrl)
for mount in mounts:
data = MountData()
data.mountUrl = self.findMountUrl(mount)
data.mountPath = self.findMountPath(mount)
try:
info = self.findMountInfo(mount)
naslinux.checkMountStatus(data.mountUrl, data.mountPath, info)
data.status = "Normal"
except naslinux.InvalidMountDomainException as e:
data.status = "RemoteFault"
data.info = str(e)
except naslinux.InvalidMountPathException as e:
data.status = "LocalFault"
data.info = str(e)
rsp.datas.append(data)
return jsonobject.dumps(rsp)
class NfsPrimaryStoragePlugin(kvmagent.KvmAgent):
'''
classdocs
'''
MOUNT_PATH = '/nfsprimarystorage/mount'
UNMOUNT_PATH = '/nfsprimarystorage/unmount'
CREATE_VOLUME_FROM_TEMPLATE_PATH = "/nfsprimarystorage/sftp/createvolumefromtemplate"
CREATE_EMPTY_VOLUME_PATH = "/nfsprimarystorage/createemptyvolume"
GET_CAPACITY_PATH = "/nfsprimarystorage/getcapacity"
CREATE_TEMPLATE_FROM_VOLUME_PATH = "/nfsprimarystorage/sftp/createtemplatefromvolume"
REVERT_VOLUME_FROM_SNAPSHOT_PATH = "/nfsprimarystorage/revertvolumefromsnapshot"
REINIT_IMAGE_PATH = "/nfsprimarystorage/reinitimage"
DELETE_PATH = "/nfsprimarystorage/delete"
LIST_PATH = "/nfsprimarystorage/listpath"
CHECK_BITS_PATH = "/nfsprimarystorage/checkbits"
UPLOAD_TO_SFTP_PATH = "/nfsprimarystorage/uploadtosftpbackupstorage"
DOWNLOAD_FROM_SFTP_PATH = "/nfsprimarystorage/downloadfromsftpbackupstorage"
UPLOAD_TO_IMAGESTORE_PATH = "/nfsprimarystorage/imagestore/upload"
COMMIT_TO_IMAGESTORE_PATH = "/nfsprimarystorage/imagestore/commit"
DOWNLOAD_FROM_IMAGESTORE_PATH = "/nfsprimarystorage/imagestore/download"
MERGE_SNAPSHOT_PATH = "/nfsprimarystorage/mergesnapshot"
REBASE_MERGE_SNAPSHOT_PATH = "/nfsprimarystorage/rebaseandmergesnapshot"
MOVE_BITS_PATH = "/nfsprimarystorage/movebits"
OFFLINE_SNAPSHOT_MERGE = "/nfsprimarystorage/offlinesnapshotmerge"
REMOUNT_PATH = "/nfsprimarystorage/remount"
GET_VOLUME_SIZE_PATH = "/nfsprimarystorage/getvolumesize"
PING_PATH = "/nfsprimarystorage/ping"
GET_VOLUME_BASE_IMAGE_PATH = "/nfsprimarystorage/getvolumebaseimage"
UPDATE_MOUNT_POINT_PATH = "/nfsprimarystorage/updatemountpoint"
RESIZE_VOLUME_PATH = "/nfsprimarystorage/volume/resize"
NFS_TO_NFS_MIGRATE_BITS_PATH = "/nfsprimarystorage/migratebits"
NFS_REBASE_VOLUME_BACKING_FILE_PATH = "/nfsprimarystorage/rebasevolumebackingfile"
ERR_UNABLE_TO_FIND_IMAGE_IN_CACHE = "unable to find image in cache"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_sync_uri(self.MOUNT_PATH, self.mount)
http_server.register_sync_uri(self.UNMOUNT_PATH, self.umount)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_TEMPLATE_PATH, self.create_root_volume_from_template)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.create_empty_volume)
http_server.register_async_uri(self.DOWNLOAD_FROM_SFTP_PATH, self.download_from_sftp)
http_server.register_async_uri(self.GET_CAPACITY_PATH, self.get_capacity)
http_server.register_async_uri(self.DELETE_PATH, self.delete)
http_server.register_async_uri(self.LIST_PATH, self.list)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH, self.create_template_from_root_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH, self.revert_volume_from_snapshot)
http_server.register_async_uri(self.REINIT_IMAGE_PATH, self.reinit_image)
http_server.register_async_uri(self.UPLOAD_TO_SFTP_PATH, self.upload_to_sftp)
http_server.register_async_uri(self.UPLOAD_TO_IMAGESTORE_PATH, self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_TO_IMAGESTORE_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_FROM_IMAGESTORE_PATH, self.download_from_imagestore)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.merge_snapshot)
http_server.register_async_uri(self.REBASE_MERGE_SNAPSHOT_PATH, self.rebase_and_merge_snapshot)
http_server.register_async_uri(self.MOVE_BITS_PATH, self.move_bits)
http_server.register_async_uri(self.OFFLINE_SNAPSHOT_MERGE, self.merge_snapshot_to_volume)
http_server.register_async_uri(self.REMOUNT_PATH, self.remount)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH, self.get_volume_size)
http_server.register_async_uri(self.PING_PATH, self.ping)
http_server.register_async_uri(self.GET_VOLUME_BASE_IMAGE_PATH, self.get_volume_base_image_path)
http_server.register_async_uri(self.UPDATE_MOUNT_POINT_PATH, self.update_mount_point)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH, self.resize_volume)
http_server.register_async_uri(self.NFS_TO_NFS_MIGRATE_BITS_PATH, self.migrate_bits)
http_server.register_async_uri(self.NFS_REBASE_VOLUME_BACKING_FILE_PATH, self.rebase_volume_backing_file)
self.mount_path = {}
self.image_cache = None
self.imagestore_client = ImageStoreClient()
def stop(self):
pass
@kvmagent.replyerror
def migrate_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = NfsToNfsMigrateBitsRsp()
mount_path = cmd.mountPath
dst_folder_path = cmd.dstFolderPath
temp_dir = None
try:
if not cmd.isMounted:
linux.is_valid_nfs_url(cmd.url)
temp_dir = tempfile.mkdtemp()
# dst folder is absolute path
mount_path = temp_dir + mount_path
dst_folder_path = temp_dir + dst_folder_path
if not linux.is_mounted(mount_path, cmd.url):
linux.mount(cmd.url, mount_path, cmd.options, "nfs4")
# Report task progress based on flow chain for now
# To get more accurate progress, we need to report from here someday
# begin migration, then check md5 sums
shell.call("mkdir -p %s; cp -r %s/* %s; sync" % (dst_folder_path, cmd.srcFolderPath, dst_folder_path))
src_md5 = shell.call(
"find %s -type f -exec md5sum {} \; | awk '{ print $1 }' | sort | md5sum" % cmd.srcFolderPath)
dst_md5 = shell.call("find %s -type f -exec md5sum {} \; | awk '{ print $1 }' | sort | md5sum" % dst_folder_path)
if src_md5 != dst_md5:
rsp.error = "failed to copy files from %s to %s, md5sum not match" % (cmd.srcFolderPath, dst_folder_path)
rsp.success = False
if not cmd.isMounted:
linux.umount(mount_path)
finally:
if temp_dir is not None:
return_code = shell.run("mount | grep '%s'" % temp_dir)
if return_code != 0:
# in case dir is not empty
try:
os.rmdir(temp_dir)
except OSError as e:
logger.warn("delete temp_dir %s failed: %s", (temp_dir, str(e)))
else:
logger.warn("temp_dir %s still had mounted destination primary storage, skip cleanup operation" % temp_dir)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def rebase_volume_backing_file(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = NfsRebaseVolumeBackingFileRsp()
if not cmd.dstImageCacheTemplateFolderPath:
qcow2s = shell.call("find %s -type f -regex '.*\.qcow2$'" % cmd.dstVolumeFolderPath)
else:
qcow2s = shell.call("find %s %s -type f -regex '.*\.qcow2$'" % (cmd.dstVolumeFolderPath, cmd.dstImageCacheTemplateFolderPath))
for qcow2 in qcow2s.split():
fmt = shell.call("qemu-img info %s | grep '^file format' | awk -F ': ' '{ print $2 }'" % qcow2)
if fmt.strip() != "qcow2":
continue
backing_file = linux.qcow2_get_backing_file(qcow2)
if backing_file == "":
continue
new_backing_file = backing_file.replace(cmd.srcPsMountPath, cmd.dstPsMountPath)
linux.qcow2_rebase_no_check(new_backing_file, qcow2)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def resize_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
install_path = cmd.installPath
rsp = ResizeVolumeRsp()
shell.call("qemu-img resize %s %s" % (install_path, cmd.size))
ret = linux.qcow2_virtualsize(install_path)
rsp.size = ret
return jsonobject.dumps(rsp)
def _get_disk_capacity(self, uuid):
path = self.mount_path.get(uuid)
if not path:
raise Exception('cannot find mount path of primary storage[uuid: %s]' % uuid)
return linux.get_disk_capacity_by_df(path)
def _json_meta_file_name(self, path):
return path + '.json'
def _set_capacity_to_response(self, uuid, rsp):
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(uuid)
@kvmagent.replyerror
def update_mount_point(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = UpdateMountPointResponse()
linux.is_valid_nfs_url(cmd.newMountPoint)
if not linux.is_mounted(cmd.mountPath, cmd.newMountPoint):
# umount old one
if linux.is_mounted(cmd.mountPath, cmd.oldMountPoint):
linux.umount(cmd.mountPath)
# mount new
linux.mount(cmd.newMountPoint, cmd.mountPath, cmd.options, "nfs4")
self.mount_path[cmd.uuid] = cmd.mountPath
logger.debug('updated the mount path[%s] mounting point from %s to %s' % (cmd.mountPath, cmd.oldMountPoint, cmd.newMountPoint))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_volume_base_image_path(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeBaseImagePathRsp()
if not os.path.basename(cmd.volumeInstallDir).endswith(cmd.volumeUuid):
raise Exception('maybe you pass a wrong install dir')
path = linux.get_qcow2_base_image_recusively(cmd.volumeInstallDir, cmd.imageCacheDir)
if not path:
return jsonobject.dumps(rsp)
rsp.path = path
rsp.size = linux.get_qcow2_file_chain_size(path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@in_bash
def ping(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
mount_path = self.mount_path[cmd.uuid]
# if nfs service stop, os.path.isdir will hung
if not linux.timeout_isdir(mount_path) or not linux.is_mounted(path=mount_path):
raise Exception('the mount path[%s] of the nfs primary storage[uuid:%s] is not existing' % (mount_path, cmd.uuid))
test_file = os.path.join(mount_path, '%s-ping-test-file' % uuidhelper.uuid())
touch = shell.ShellCmd('timeout 60 touch %s' % test_file)
touch(False)
if touch.return_code == 124:
raise Exception('unable to access the mount path[%s] of the nfs primary storage[uuid:%s] in 60s, timeout' %
(mount_path, cmd.uuid))
elif touch.return_code != 0:
touch.raise_error()
linux.rm_file_force(test_file)
return jsonobject.dumps(NfsResponse())
@kvmagent.replyerror
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot_to_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = OfflineMergeSnapshotRsp()
if not cmd.fullRebase:
linux.qcow2_rebase(cmd.srcPath, cmd.destPath)
else:
tmp = os.path.join(os.path.dirname(cmd.destPath), '%s.qcow2' % uuidhelper.uuid())
linux.create_template(cmd.destPath, tmp)
shell.call("mv %s %s" % (tmp, cmd.destPath))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def move_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MoveBitsRsp()
if not os.path.exists(cmd.srcPath):
rsp.error = "%s is not existing" % cmd.srcPath
rsp.success = False
else:
dirname = os.path.dirname(cmd.destPath)
if not os.path.exists(dirname):
os.makedirs(dirname)
shell.call("mv %s %s" % (cmd.srcPath, cmd.destPath))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def rebase_and_merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
snapshots = cmd.snapshotInstallPaths
count = len(snapshots)
for i in range(count):
if i+1 < count:
target = snapshots[i]
backing_file = snapshots[i+1]
linux.qcow2_rebase_no_check(backing_file, target)
latest = snapshots[0]
rsp = RebaseAndMergeSnapshotsResponse()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
try:
linux.create_template(latest, cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = cmd.workspaceInstallPath
self._set_capacity_to_response(cmd.uuid, rsp)
except linux.LinuxError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotResponse()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
try:
linux.create_template(cmd.snapshotInstallPath, cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.workspaceInstallPath)
self._set_capacity_to_response(cmd.uuid, rsp)
except linux.LinuxError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CopyToSftpBackupStorageResponse()
def upload():
if not os.path.exists(cmd.primaryStorageInstallPath):
raise kvmagent.KvmError('cannot find %s' % cmd.primaryStorageInstallPath)
linux.scp_upload(cmd.backupStorageHostName, cmd.backupStorageSshKey, cmd.primaryStorageInstallPath, cmd.backupStorageInstallPath, cmd.backupStorageUserName, cmd.backupStorageSshPort)
try:
upload()
except kvmagent.KvmError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(cmd, req)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
@kvmagent.replyerror
def download_from_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
mount_path = self.mount_path.get(cmd.uuid)
self.check_nfs_mounted(mount_path)
cachedir = None if cmd.isData else mount_path
self.imagestore_client.download_from_imagestore(cachedir, cmd.hostname, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath)
rsp = kvmagent.AgentResponse()
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def reinit_image(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ReInitImageResponse()
install_path = cmd.imagePath
dirname = os.path.dirname(cmd.volumePath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
new_volume_path = os.path.join(dirname, '{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone_with_cmd(install_path, new_volume_path, cmd)
rsp.newVolumeInstallPath = new_volume_path
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def revert_volume_from_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotResponse()
install_path = cmd.snapshotInstallPath
new_volume_path = os.path.join(os.path.dirname(install_path), '{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone_with_cmd(install_path, new_volume_path, cmd)
rsp.newVolumeInstallPath = new_volume_path
size = linux.qcow2_virtualsize(new_volume_path)
rsp.size = size
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckIsBitsExistingRsp()
rsp.existing = os.path.exists(cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def delete(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = DeleteResponse()
if cmd.folder:
linux.rm_dir_checked(cmd.installPath)
else:
kvmagent.deleteImage(cmd.installPath)
logger.debug('successfully delete %s' % cmd.installPath)
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def list(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ListResponse()
rsp.paths = kvmagent.listPath(cmd.path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def remount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MountResponse()
linux.is_valid_nfs_url(cmd.url)
linux.remount(cmd.url, cmd.mountPath, cmd.options)
self.mount_path[cmd.uuid] = cmd.mountPath
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@lock.lock('mount')
def mount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MountResponse()
linux.is_valid_nfs_url(cmd.url)
if not linux.is_mounted(cmd.mountPath, cmd.url):
linux.mount(cmd.url, cmd.mountPath, cmd.options, "nfs4")
self.mount_path[cmd.uuid] = cmd.mountPath
logger.debug(http.path_msg(self.MOUNT_PATH, 'mounted %s on %s' % (cmd.url, cmd.mountPath)))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def umount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = UnmountResponse()
if linux.is_mounted(path=cmd.mountPath):
ret = linux.umount(cmd.mountPath)
if not ret: logger.warn(http.path_msg(self.UNMOUNT_PATH, 'unmount %s from %s failed' % (cmd.mountPath, cmd.url)))
logger.debug(http.path_msg(self.UNMOUNT_PATH, 'umounted %s from %s' % (cmd.mountPath, cmd.url)))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_capacity(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetCapacityResponse()
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_empty_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CreateEmptyVolumeResponse()
try:
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname)
linux.qcow2_create_with_cmd(cmd.installUrl, cmd.size, cmd)
except Exception as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = 'unable to create empty volume[uuid:%s, name:%s], %s' % (cmd.uuid, cmd.name, str(e))
rsp.success = False
return jsonobject.dumps(rsp)
meta = VolumeMeta()
meta.account_uuid = cmd.accountUuid
meta.hypervisor_type = cmd.hypervisorType
meta.name = cmd.name
meta.uuid = cmd.volumeUuid
meta.size = cmd.size
meta_path = self._json_meta_file_name(cmd.installUrl)
with open(meta_path, 'w') as fd:
fd.write(jsonobject.dumps(meta, pretty=True))
self._set_capacity_to_response(cmd.uuid, rsp)
logger.debug('successfully create empty volume[uuid:%s, name:%s, size:%s] at %s' % (cmd.uuid, cmd.name, cmd.size, cmd.installUrl))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_template_from_root_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CreateTemplateFromRootVolumeRsp()
try:
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
linux.create_template(cmd.rootVolumePath, cmd.installPath)
except linux.LinuxError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = 'unable to create image to root@%s:%s from root volume[%s], %s' % (cmd.sftpBackupStorageHostName,
cmd.installPath, cmd.rootVolumePath, str(e))
rsp.success = False
self._set_capacity_to_response(cmd.uuid, rsp)
logger.debug('successfully created template[%s] from root volume[%s]' % (cmd.installPath, cmd.rootVolumePath))
return jsonobject.dumps(rsp)
def check_nfs_mounted(self, mount_path):
if not linux.is_mounted(mount_path):
raise Exception('NFS not mounted on: %s' % mount_path)
@kvmagent.replyerror
def download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.check_nfs_mounted(self.mount_path.get(cmd.uuid))
rsp = DownloadBitsFromSftpBackupStorageResponse()
try:
linux.scp_download(cmd.hostname, cmd.sshKey, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath, cmd.username, cmd.sshPort)
logger.debug('successfully download %s/%s to %s' % (cmd.hostname, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath))
self._set_capacity_to_response(cmd.uuid, rsp)
except Exception as e:
content = traceback.format_exc()
logger.warn(content)
err = "unable to download %s/%s, because %s" % (cmd.hostname, cmd.backupStorageInstallPath, str(e))
rsp.error = err
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_root_volume_from_template(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CreateRootVolumeFromTemplateResponse()
if not os.path.exists(cmd.templatePathInCache):
rsp.error = self.ERR_UNABLE_TO_FIND_IMAGE_IN_CACHE
rsp.success = False
return jsonobject.dumps(rsp)
try:
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
linux.qcow2_clone_with_cmd(cmd.templatePathInCache, cmd.installUrl, cmd)
logger.debug('successfully create root volume[%s] from template in cache[%s]' % (cmd.installUrl, cmd.templatePathInCache))
meta = VolumeMeta()
meta.account_uuid = cmd.accountUuid
meta.hypervisor_type = cmd.hypervisorType
meta.name = cmd.name
meta.uuid = cmd.volumeUuid
meta.size = os.path.getsize(cmd.templatePathInCache)
meta_path = self._json_meta_file_name(cmd.installUrl)
with open(meta_path, 'w') as fd:
fd.write(jsonobject.dumps(meta, pretty=True))
self._set_capacity_to_response(cmd.uuid, rsp)
logger.debug('successfully create root volume[%s] from template in cache[%s]' % (cmd.installUrl, cmd.templatePathInCache))
except Exception as e:
content = traceback.format_exc()
logger.warn(content)
err = 'unable to clone qcow2 template[%s] to %s' % (cmd.templatePathInCache, cmd.installUrl)
rsp.error = err
rsp.success = False
return jsonobject.dumps(rsp)
class LocalStoragePlugin(kvmagent.KvmAgent):
INIT_PATH = "/localstorage/init"
GET_PHYSICAL_CAPACITY_PATH = "/localstorage/getphysicalcapacity"
CREATE_EMPTY_VOLUME_PATH = "/localstorage/volume/createempty"
CREATE_VOLUME_FROM_CACHE_PATH = "/localstorage/volume/createvolumefromcache"
DELETE_BITS_PATH = "/localstorage/delete"
DELETE_DIR_PATH = "/localstorage/deletedir"
UPLOAD_BIT_PATH = "/localstorage/sftp/upload"
DOWNLOAD_BIT_PATH = "/localstorage/sftp/download"
UPLOAD_TO_IMAGESTORE_PATH = "/localstorage/imagestore/upload"
COMMIT_TO_IMAGESTORE_PATH = "/localstorage/imagestore/commit"
DOWNLOAD_FROM_IMAGESTORE_PATH = "/localstorage/imagestore/download"
REVERT_SNAPSHOT_PATH = "/localstorage/snapshot/revert"
MERGE_SNAPSHOT_PATH = "/localstorage/snapshot/merge"
MERGE_AND_REBASE_SNAPSHOT_PATH = "/localstorage/snapshot/mergeandrebase"
OFFLINE_MERGE_PATH = "/localstorage/snapshot/offlinemerge"
CREATE_TEMPLATE_FROM_VOLUME = "/localstorage/volume/createtemplate"
CHECK_BITS_PATH = "/localstorage/checkbits"
REBASE_ROOT_VOLUME_TO_BACKING_FILE_PATH = "/localstorage/volume/rebaserootvolumetobackingfile"
VERIFY_SNAPSHOT_CHAIN_PATH = "/localstorage/snapshot/verifychain"
REBASE_SNAPSHOT_BACKING_FILES_PATH = "/localstorage/snapshot/rebasebackingfiles"
COPY_TO_REMOTE_BITS_PATH = "/localstorage/copytoremote"
GET_MD5_PATH = "/localstorage/getmd5"
CHECK_MD5_PATH = "/localstorage/checkmd5"
GET_BACKING_FILE_PATH = "/localstorage/volume/getbackingfile"
GET_VOLUME_SIZE = "/localstorage/volume/getsize"
GET_BASE_IMAGE_PATH = "/localstorage/volume/getbaseimagepath"
GET_QCOW2_REFERENCE = "/localstorage/getqcow2reference"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.INIT_PATH, self.init)
http_server.register_async_uri(self.GET_PHYSICAL_CAPACITY_PATH,
self.get_physical_capacity)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH,
self.create_empty_volume)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH,
self.create_root_volume_from_template)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete)
http_server.register_async_uri(self.DELETE_DIR_PATH, self.deletedir)
http_server.register_async_uri(self.DOWNLOAD_BIT_PATH,
self.download_from_sftp)
http_server.register_async_uri(self.UPLOAD_BIT_PATH,
self.upload_to_sftp)
http_server.register_async_uri(self.UPLOAD_TO_IMAGESTORE_PATH,
self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_TO_IMAGESTORE_PATH,
self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_FROM_IMAGESTORE_PATH,
self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_SNAPSHOT_PATH,
self.revert_snapshot)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH,
self.merge_snapshot)
http_server.register_async_uri(self.MERGE_AND_REBASE_SNAPSHOT_PATH,
self.merge_and_rebase_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_PATH,
self.offline_merge_snapshot)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME,
self.create_template_from_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(
self.REBASE_ROOT_VOLUME_TO_BACKING_FILE_PATH,
self.rebase_root_volume_to_backing_file)
http_server.register_async_uri(self.VERIFY_SNAPSHOT_CHAIN_PATH,
self.verify_backing_file_chain)
http_server.register_async_uri(self.REBASE_SNAPSHOT_BACKING_FILES_PATH,
self.rebase_backing_files)
http_server.register_async_uri(self.COPY_TO_REMOTE_BITS_PATH,
self.copy_bits_to_remote)
http_server.register_async_uri(self.GET_MD5_PATH, self.get_md5)
http_server.register_async_uri(self.CHECK_MD5_PATH, self.check_md5)
http_server.register_async_uri(self.GET_BACKING_FILE_PATH,
self.get_backing_file_path)
http_server.register_async_uri(self.GET_VOLUME_SIZE,
self.get_volume_size)
http_server.register_async_uri(self.GET_BASE_IMAGE_PATH,
self.get_volume_base_image_path)
http_server.register_async_uri(self.GET_QCOW2_REFERENCE,
self.get_qcow2_reference)
self.path = None
self.imagestore_client = ImageStoreClient()
def stop(self):
pass
@kvmagent.replyerror
def get_qcow2_reference(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
out = shell.call('find %s -type f' % cmd.searchingDir)
rsp = GetQCOW2ReferenceRsp()
rsp.referencePaths = []
for f in out.split('\n'):
f = f.strip(' \t\r\n')
if not f: continue
backing_file = linux.qcow2_get_backing_file(f)
if backing_file == cmd.path:
rsp.referencePaths.append(f)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_volume_base_image_path(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeBaseImagePathRsp()
rsp.path = linux.get_qcow2_base_image_path_recusively(cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_backing_file_path(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
out = linux.qcow2_get_backing_file(cmd.path)
rsp = GetBackingFileRsp()
if out:
rsp.backingFilePath = out
rsp.size = os.path.getsize(out)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_md5(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetMd5Rsp()
rsp.md5s = []
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
report = Report()
report.processType = "LocalStorageMigrateVolume"
PFILE = shell.call('mktemp /tmp/tmp-XXXXXX').strip()
total = 0
written = 0
for to in cmd.md5s:
total = total + os.path.getsize(to.path)
def _get_progress(synced):
logger.debug("getProgress in get_md5")
if not os.path.exists(PFILE):
return synced
last = shell.call('tail -1 %s' % PFILE).strip()
if not last or not last.isdigit():
return synced
percent = int(
(float(written) * 100 + os.path.getsize(to.path) * float(last))
/ total / 10)
report.progress_report(str(percent), "report")
return synced
report.resourceUuid = cmd.volumeUuid
report.progress_report("0", "start")
for to in cmd.md5s:
_, md5, _ = bash_progress_1(
"pv -n %s 2>%s | md5sum | cut -d ' ' -f 1" % (to.path, PFILE),
_get_progress)
rsp.md5s.append({
'resourceUuid': to.resourceUuid,
'path': to.path,
'md5': md5
})
written += os.path.getsize(to.path)
percent = int(round(float(written) / float(total) * 10))
report.progress_report(percent, "report")
if os.path.exists(PFILE):
os.remove(PFILE)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def check_md5(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
report = Report()
report.processType = "LocalStorageMigrateVolume"
PFILE = shell.call('mktemp /tmp/tmp-XXXXXX').strip()
total = 0
written = 0
for to in cmd.md5s:
total = total + os.path.getsize(to.path)
def _get_progress(synced):
logger.debug("getProgress in check_md5")
if not os.path.exists(PFILE):
return synced
last = shell.call('tail -1 %s' % PFILE).strip()
if not last or not last.isdigit():
return synced
percent = int(
round((float(written) * 100 +
os.path.getsize(to.path) * float(last)) / total / 10) +
90)
report.progress_report(percent, "report")
return synced
report.resourceUuid = cmd.volumeUuid
for to in cmd.md5s:
_, dst_md5, _ = bash_progress_1(
"pv -n %s 2>%s | md5sum | cut -d ' ' -f 1" % (to.path, PFILE),
_get_progress)
if dst_md5 != to.md5:
raise Exception(
"MD5 unmatch. The file[uuid:%s, path:%s]'s md5 (src host:%s, dst host:%s)"
% (to.resourceUuid, to.path, to.md5, dst_md5))
written += os.path.getsize(to.path)
percent = int(round(float(written) / float(total) * 10 + 90))
report.progress_report(percent, "report")
if os.path.exists(PFILE):
os.remove(PFILE)
rsp = AgentResponse()
report.progress_report("100", "finish")
return jsonobject.dumps(rsp)
def _get_disk_capacity(self):
return linux.get_disk_capacity_by_df(self.path)
@kvmagent.replyerror
@in_bash
def copy_bits_to_remote(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
chain = sum([linux.qcow2_get_file_chain(p) for p in cmd.paths], [])
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
report = Report()
report.processType = "LocalStorageMigrateVolume"
report.resourceUuid = cmd.uuid
PFILE = shell.call('mktemp /tmp/tmp-XXXXXX').strip()
total = 0
for path in set(chain):
total = total + os.path.getsize(path)
written = 0
def _get_progress(synced):
logger.debug(
"getProgress in localstorage-agent, synced: %s, total: %s" %
(synced, total))
if not os.path.exists(PFILE):
return synced
fpread = open(PFILE, 'r')
lines = fpread.readlines()
if not lines:
fpread.close()
return synced
last = str(lines[-1]).strip().split('\r')[-1]
if not last or len(last.split()) < 1:
fpread.close()
return synced
line = last.split()[0]
if not line.isdigit():
return synced
if total > 0:
synced = long(line)
if synced < total:
percent = int(
round(
float(written + synced) / float(total) * 80 + 10))
report.progress_report(percent, "report")
synced = written
fpread.close()
return synced
err = None
for path in set(chain):
PATH = path
PASSWORD = cmd.dstPassword
USER = cmd.dstUsername
IP = cmd.dstIp
PORT = (cmd.dstPort and cmd.dstPort or "22")
DIR = os.path.dirname(path)
bash_errorout(
'/usr/bin/sshpass -p {{PASSWORD}} ssh -o StrictHostKeyChecking=no -t -t -p {{PORT}} {{USER}}@{{IP}} "sudo mkdir -p {{DIR}}"'
)
_, _, err = bash_progress_1(
'rsync -av --progress --relative {{PATH}} --rsh="/usr/bin/sshpass -p {{PASSWORD}} ssh -o StrictHostKeyChecking=no -p {{PORT}} -l {{USER}}" {{IP}}:/ 1>{{PFILE}}',
_get_progress)
if err:
raise err
written += os.path.getsize(path)
bash_errorout(
'/usr/bin/sshpass -p {{PASSWORD}} ssh -o StrictHostKeyChecking=no -p {{PORT}} {{USER}}@{{IP}} "/bin/sync {{PATH}}"'
)
percent = int(round(float(written) / float(total) * 80 + 10))
report.progress_report(percent, "report")
if os.path.exists(PFILE):
os.remove(PFILE)
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def verify_backing_file_chain(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
for sp in cmd.snapshots:
if not os.path.exists(sp.path):
raise Exception('cannot find the file[%s]' % sp.path)
if sp.parentPath and not os.path.exists(sp.parentPath):
raise Exception('cannot find the backing file[%s]' %
sp.parentPath)
if sp.parentPath:
out = linux.qcow2_get_backing_file(sp.path)
if sp.parentPath != out:
raise Exception(
"resource[Snapshot or Volume, uuid:%s, path:%s]'s backing file[%s] is not equal to %s"
% (sp.snapshotUuid, sp.path, out, sp.parentPath))
return jsonobject.dumps(AgentResponse())
@kvmagent.replyerror
def rebase_backing_files(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
for sp in cmd.snapshots:
if sp.parentPath:
linux.qcow2_rebase_no_check(sp.parentPath, sp.path)
return jsonobject.dumps(AgentResponse())
@kvmagent.replyerror
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckBitsRsp()
rsp.existing = os.path.exists(cmd.path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_template_from_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
linux.qcow2_create_template(cmd.volumePath, cmd.installPath)
logger.debug('successfully created template[%s] from volume[%s]' %
(cmd.installPath, cmd.volumePath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def revert_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotRsp()
install_path = cmd.snapshotInstallPath
new_volume_path = os.path.join(os.path.dirname(install_path),
'{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone(install_path, new_volume_path)
rsp.newVolumeInstallPath = new_volume_path
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotRsp()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
linux.qcow2_create_template(cmd.snapshotInstallPath,
cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
cmd.workspaceInstallPath)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_and_rebase_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
snapshots = cmd.snapshotInstallPaths
count = len(snapshots)
for i in range(count):
if i + 1 < count:
target = snapshots[i]
backing_file = snapshots[i + 1]
linux.qcow2_rebase_no_check(backing_file, target)
latest = snapshots[0]
rsp = RebaseAndMergeSnapshotsRsp()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
linux.qcow2_create_template(latest, cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
cmd.workspaceInstallPath)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def offline_merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
if not cmd.fullRebase:
linux.qcow2_rebase(cmd.srcPath, cmd.destPath)
else:
tmp = os.path.join(os.path.dirname(cmd.destPath),
'%s.qcow2' % uuidhelper.uuid())
linux.qcow2_create_template(cmd.destPath, tmp)
shell.call("mv %s %s" % (tmp, cmd.destPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_physical_capacity(self, req):
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def rebase_root_volume_to_backing_file(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
linux.qcow2_rebase_no_check(cmd.backingFilePath, cmd.rootVolumePath)
return jsonobject.dumps(AgentResponse())
@kvmagent.replyerror
def init(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.path = cmd.path
if not os.path.exists(self.path):
os.makedirs(self.path, 0755)
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_empty_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
try:
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname)
if cmd.backingFile:
linux.qcow2_create_with_backing_file(cmd.backingFile,
cmd.installUrl)
else:
linux.qcow2_create(cmd.installUrl, cmd.size)
except Exception as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = 'unable to create empty volume[uuid:%s, name:%s], %s' % (
cmd.uuid, cmd.name, str(e))
rsp.success = False
return jsonobject.dumps(rsp)
logger.debug(
'successfully create empty volume[uuid:%s, size:%s] at %s' %
(cmd.volumeUuid, cmd.size, cmd.installUrl))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_root_volume_from_template(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
if not os.path.exists(cmd.templatePathInCache):
rsp.error = "UNABLE_TO_FIND_IMAGE_IN_CACHE"
rsp.success = False
logger.debug('error: %s: %s' %
(rsp.error, cmd.templatePathInCache))
return jsonobject.dumps(rsp)
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
linux.qcow2_clone(cmd.templatePathInCache, cmd.installUrl)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def delete(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
shell.call('rm -f %s' % cmd.path)
pdir = os.path.dirname(cmd.path)
linux.rmdir_if_empty(pdir)
logger.debug('successfully delete %s' % cmd.path)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def deletedir(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
shell.call('rm -rf %s' % cmd.path)
logger.debug('successfully delete %s' % cmd.path)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
def upload():
if not os.path.exists(cmd.primaryStorageInstallPath):
raise kvmagent.KvmError('cannot find %s' %
cmd.primaryStorageInstallPath)
linux.scp_upload(cmd.hostname, cmd.sshKey,
cmd.primaryStorageInstallPath,
cmd.backupStorageInstallPath, cmd.username,
cmd.sshPort)
try:
upload()
except kvmagent.KvmError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(cmd, req)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
@kvmagent.replyerror
def download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
try:
linux.scp_download(cmd.hostname, cmd.sshKey,
cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath, cmd.username,
cmd.sshPort)
logger.debug('successfully download %s/%s to %s' %
(cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath))
except Exception as e:
content = traceback.format_exc()
logger.warn(content)
err = "unable to download %s/%s, because %s" % (
cmd.hostname, cmd.backupStorageInstallPath, str(e))
rsp.error = err
rsp.success = False
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def download_from_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.imagestore_client.download_from_imagestore(
self.path, cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath)
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
class NfsPrimaryStoragePlugin(kvmagent.KvmAgent):
'''
classdocs
'''
MOUNT_PATH = '/nfsprimarystorage/mount'
UNMOUNT_PATH = '/nfsprimarystorage/unmount'
CREATE_VOLUME_FROM_TEMPLATE_PATH = "/nfsprimarystorage/sftp/createvolumefromtemplate"
CREATE_EMPTY_VOLUME_PATH = "/nfsprimarystorage/createemptyvolume"
GET_CAPACITY_PATH = "/nfsprimarystorage/getcapacity"
CREATE_TEMPLATE_FROM_VOLUME_PATH = "/nfsprimarystorage/sftp/createtemplatefromvolume"
REVERT_VOLUME_FROM_SNAPSHOT_PATH = "/nfsprimarystorage/revertvolumefromsnapshot"
DELETE_PATH = "/nfsprimarystorage/delete"
CHECK_BITS_PATH = "/nfsprimarystorage/checkbits"
UPLOAD_TO_SFTP_PATH = "/nfsprimarystorage/uploadtosftpbackupstorage"
DOWNLOAD_FROM_SFTP_PATH = "/nfsprimarystorage/downloadfromsftpbackupstorage"
UPLOAD_TO_IMAGESTORE_PATH = "/nfsprimarystorage/imagestore/upload"
COMMIT_TO_IMAGESTORE_PATH = "/nfsprimarystorage/imagestore/commit"
DOWNLOAD_FROM_IMAGESTORE_PATH = "/nfsprimarystorage/imagestore/download"
MERGE_SNAPSHOT_PATH = "/nfsprimarystorage/mergesnapshot"
REBASE_MERGE_SNAPSHOT_PATH = "/nfsprimarystorage/rebaseandmergesnapshot"
MOVE_BITS_PATH = "/nfsprimarystorage/movebits"
OFFLINE_SNAPSHOT_MERGE = "/nfsprimarystorage/offlinesnapshotmerge"
REMOUNT_PATH = "/nfsprimarystorage/remount"
GET_VOLUME_SIZE_PATH = "/nfsprimarystorage/getvolumesize"
PING_PATH = "/nfsprimarystorage/ping"
GET_VOLUME_BASE_IMAGE_PATH = "/nfsprimarystorage/getvolumebaseimage"
UPDATE_MOUNT_POINT_PATH = "/nfsprimarystorage/updatemountpoint"
ERR_UNABLE_TO_FIND_IMAGE_IN_CACHE = "UNABLE_TO_FIND_IMAGE_IN_CACHE"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_sync_uri(self.MOUNT_PATH, self.mount)
http_server.register_sync_uri(self.UNMOUNT_PATH, self.umount)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_TEMPLATE_PATH, self.create_root_volume_from_template)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.create_empty_volume)
http_server.register_async_uri(self.DOWNLOAD_FROM_SFTP_PATH, self.download_from_sftp)
http_server.register_async_uri(self.GET_CAPACITY_PATH, self.get_capacity)
http_server.register_async_uri(self.DELETE_PATH, self.delete)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH, self.create_template_from_root_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH, self.revert_volume_from_snapshot)
http_server.register_async_uri(self.UPLOAD_TO_SFTP_PATH, self.upload_to_sftp)
http_server.register_async_uri(self.UPLOAD_TO_IMAGESTORE_PATH, self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_TO_IMAGESTORE_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_FROM_IMAGESTORE_PATH, self.download_from_imagestore)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.merge_snapshot)
http_server.register_async_uri(self.REBASE_MERGE_SNAPSHOT_PATH, self.rebase_and_merge_snapshot)
http_server.register_async_uri(self.MOVE_BITS_PATH, self.move_bits)
http_server.register_async_uri(self.OFFLINE_SNAPSHOT_MERGE, self.merge_snapshot_to_volume)
http_server.register_async_uri(self.REMOUNT_PATH, self.remount)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH, self.get_volume_size)
http_server.register_async_uri(self.PING_PATH, self.ping)
http_server.register_async_uri(self.GET_VOLUME_BASE_IMAGE_PATH, self.get_volume_base_image_path)
http_server.register_async_uri(self.UPDATE_MOUNT_POINT_PATH, self.update_mount_point)
self.mount_path = {}
self.image_cache = None
self.imagestore_client = ImageStoreClient()
def stop(self):
pass
def _get_disk_capacity(self, uuid):
path = self.mount_path.get(uuid)
if not path:
raise Exception('cannot find mount path of primary storage[uuid: %s]' % uuid)
return linux.get_disk_capacity_by_df(path)
def _json_meta_file_name(self, path):
return path + '.json'
def _set_capacity_to_response(self, uuid, rsp):
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(uuid)
@kvmagent.replyerror
def update_mount_point(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = UpdateMountPointResponse()
linux.is_valid_nfs_url(cmd.newMountPoint)
if not linux.is_mounted(cmd.mountPath, cmd.newMountPoint):
# umount old one
if linux.is_mounted(cmd.mountPath, cmd.oldMountPoint):
linux.umount(cmd.mountPath)
# mount new
linux.mount(cmd.newMountPoint, cmd.mountPath, cmd.options)
self.mount_path[cmd.uuid] = cmd.mountPath
logger.debug('updated the mount path[%s] mounting point from %s to %s' % (cmd.mountPath, cmd.oldMountPoint, cmd.newMountPoint))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_volume_base_image_path(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeBaseImagePathRsp()
rsp.path = linux.get_qcow2_base_image_path_recusively(cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@in_bash
def ping(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
mount_path = self.mount_path[cmd.uuid]
# if nfs service stop, os.path.isdir will hung
if not mount_path or (bash_r("timeout 15 ls %s" % mount_path) != 0):
raise Exception('the mount path[%s] of the nfs primary storage[uuid:%s] is not existing' % (mount_path, cmd.uuid))
test_file = os.path.join(mount_path, '%s-ping-test-file' % uuidhelper.uuid())
touch = shell.ShellCmd('timeout 60 touch %s' % test_file)
touch(False)
if touch.return_code == 124:
raise Exception('unable to access the mount path[%s] of the nfs primary storage[uuid:%s] in 60s, timeout' %
(mount_path, cmd.uuid))
elif touch.return_code != 0:
touch.raise_error()
shell.call('rm -f %s' % test_file)
return jsonobject.dumps(NfsResponse())
@kvmagent.replyerror
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot_to_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = OfflineMergeSnapshotRsp()
if not cmd.fullRebase:
linux.qcow2_rebase(cmd.srcPath, cmd.destPath)
else:
tmp = os.path.join(os.path.dirname(cmd.destPath), '%s.qcow2' % uuidhelper.uuid())
linux.qcow2_create_template(cmd.destPath, tmp)
shell.call("mv %s %s" % (tmp, cmd.destPath))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def move_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MoveBitsRsp()
if not os.path.exists(cmd.srcPath):
rsp.error = "%s is not existing" % cmd.srcPath
rsp.success = False
else:
dirname = os.path.dirname(cmd.destPath)
if not os.path.exists(dirname):
os.makedirs(dirname)
shell.call("mv %s %s" % (cmd.srcPath, cmd.destPath))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def rebase_and_merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
snapshots = cmd.snapshotInstallPaths
count = len(snapshots)
for i in range(count):
if i+1 < count:
target = snapshots[i]
backing_file = snapshots[i+1]
linux.qcow2_rebase_no_check(backing_file, target)
latest = snapshots[0]
rsp = RebaseAndMergeSnapshotsResponse()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
try:
linux.qcow2_create_template(latest, cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = cmd.workspaceInstallPath
self._set_capacity_to_response(cmd.uuid, rsp)
except linux.LinuxError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotResponse()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
try:
linux.qcow2_create_template(cmd.snapshotInstallPath, cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.workspaceInstallPath)
self._set_capacity_to_response(cmd.uuid, rsp)
except linux.LinuxError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CopyToSftpBackupStorageResponse()
def upload():
if not os.path.exists(cmd.primaryStorageInstallPath):
raise kvmagent.KvmError('cannot find %s' % cmd.primaryStorageInstallPath)
linux.scp_upload(cmd.backupStorageHostName, cmd.backupStorageSshKey, cmd.primaryStorageInstallPath, cmd.backupStorageInstallPath, cmd.backupStorageUserName, cmd.backupStorageSshPort)
try:
upload()
except kvmagent.KvmError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(cmd, req)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
@kvmagent.replyerror
def download_from_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.imagestore_client.download_from_imagestore(self.mount_path.get(cmd.uuid), cmd.hostname, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath)
rsp = kvmagent.AgentResponse()
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def revert_volume_from_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotResponse()
install_path = cmd.snapshotInstallPath
new_volume_path = os.path.join(os.path.dirname(install_path), '{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone(install_path, new_volume_path)
rsp.newVolumeInstallPath = new_volume_path
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckIsBitsExistingRsp()
rsp.existing = os.path.exists(cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def delete(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = DeleteResponse()
if cmd.isFolder:
shell.call('rm -rf %s' % cmd.installPath)
else:
shell.call('rm -f %s' % cmd.installPath)
pdir = os.path.dirname(cmd.installPath)
linux.rmdir_if_empty(pdir)
logger.debug('successfully delete %s' % cmd.installPath)
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def remount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MountResponse()
linux.is_valid_nfs_url(cmd.url)
if not linux.is_mounted(cmd.mountPath, cmd.url):
linux.mount(cmd.url, cmd.mountPath, cmd.options)
shell.call('mount -o remount %s' % cmd.mountPath)
self.mount_path[cmd.uuid] = cmd.mountPath
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def mount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MountResponse()
linux.is_valid_nfs_url(cmd.url)
if not linux.is_mounted(cmd.mountPath, cmd.url):
linux.mount(cmd.url, cmd.mountPath, cmd.options)
self.mount_path[cmd.uuid] = cmd.mountPath
logger.debug(http.path_msg(self.MOUNT_PATH, 'mounted %s on %s' % (cmd.url, cmd.mountPath)))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def umount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = UnmountResponse()
if linux.is_mounted(path=cmd.mountPath):
ret = linux.umount(cmd.mountPath)
if not ret: logger.warn(http.path_msg(self.UNMOUNT_PATH, 'unmount %s from %s failed' % (cmd.mountPath, cmd.url)))
logger.debug(http.path_msg(self.UNMOUNT_PATH, 'umounted %s from %s' % (cmd.mountPath, cmd.url)))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_capacity(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetCapacityResponse()
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_empty_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CreateEmptyVolumeResponse()
try:
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname)
linux.qcow2_create(cmd.installUrl, cmd.size)
except Exception as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = 'unable to create empty volume[uuid:%s, name:%s], %s' % (cmd.uuid, cmd.name, str(e))
rsp.success = False
return jsonobject.dumps(rsp)
meta = VolumeMeta()
meta.account_uuid = cmd.accountUuid
meta.hypervisor_type = cmd.hypervisorType
meta.name = cmd.name
meta.uuid = cmd.volumeUuid
meta.size = cmd.size
meta_path = self._json_meta_file_name(cmd.installUrl)
with open(meta_path, 'w') as fd:
fd.write(jsonobject.dumps(meta, pretty=True))
self._set_capacity_to_response(cmd.uuid, rsp)
logger.debug('successfully create empty volume[uuid:%s, name:%s, size:%s] at %s' % (cmd.uuid, cmd.name, cmd.size, cmd.installUrl))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_template_from_root_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CreateTemplateFromRootVolumeRsp()
try:
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
linux.qcow2_create_template(cmd.rootVolumePath, cmd.installPath)
except linux.LinuxError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = 'unable to create image to root@%s:%s from root volume[%s], %s' % (cmd.sftpBackupStorageHostName,
cmd.installPath, cmd.rootVolumePath, str(e))
rsp.success = False
self._set_capacity_to_response(cmd.uuid, rsp)
logger.debug('successfully created template[%s] from root volume[%s]' % (cmd.installPath, cmd.rootVolumePath))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = DownloadBitsFromSftpBackupStorageResponse()
try:
linux.scp_download(cmd.hostname, cmd.sshKey, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath, cmd.username, cmd.sshPort)
logger.debug('successfully download %s/%s to %s' % (cmd.hostname, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath))
self._set_capacity_to_response(cmd.uuid, rsp)
except Exception as e:
content = traceback.format_exc()
logger.warn(content)
err = "unable to download %s/%s, because %s" % (cmd.hostname, cmd.backupStorageInstallPath, str(e))
rsp.error = err
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_root_volume_from_template(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CreateRootVolumeFromTemplateResponse()
if not os.path.exists(cmd.templatePathInCache):
rsp.error = self.ERR_UNABLE_TO_FIND_IMAGE_IN_CACHE
rsp.success = False
return jsonobject.dumps(rsp)
try:
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
linux.qcow2_clone(cmd.templatePathInCache, cmd.installUrl)
logger.debug('successfully create root volume[%s] from template in cache[%s]' % (cmd.installUrl, cmd.templatePathInCache))
meta = VolumeMeta()
meta.account_uuid = cmd.accountUuid
meta.hypervisor_type = cmd.hypervisorType
meta.name = cmd.name
meta.uuid = cmd.volumeUuid
meta.size = os.path.getsize(cmd.templatePathInCache)
meta_path = self._json_meta_file_name(cmd.installUrl)
with open(meta_path, 'w') as fd:
fd.write(jsonobject.dumps(meta, pretty=True))
self._set_capacity_to_response(cmd.uuid, rsp)
logger.debug('successfully create root volume[%s] from template in cache[%s]' % (cmd.installUrl, cmd.templatePathInCache))
except Exception as e:
content = traceback.format_exc()
logger.warn(content)
err = 'unable to clone qcow2 template[%s] to %s' % (cmd.templatePathInCache, cmd.installUrl)
rsp.error = err
rsp.success = False
return jsonobject.dumps(rsp)
class SharedBlockPlugin(kvmagent.KvmAgent):
CONNECT_PATH = "/sharedblock/connect"
DISCONNECT_PATH = "/sharedblock/disconnect"
CREATE_VOLUME_FROM_CACHE_PATH = "/sharedblock/createrootvolume"
DELETE_BITS_PATH = "/sharedblock/bits/delete"
CREATE_TEMPLATE_FROM_VOLUME_PATH = "/sharedblock/createtemplatefromvolume"
UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH = "/sharedblock/sftp/upload"
DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH = "/sharedblock/sftp/download"
UPLOAD_BITS_TO_IMAGESTORE_PATH = "/sharedblock/imagestore/upload"
COMMIT_BITS_TO_IMAGESTORE_PATH = "/sharedblock/imagestore/commit"
DOWNLOAD_BITS_FROM_IMAGESTORE_PATH = "/sharedblock/imagestore/download"
REVERT_VOLUME_FROM_SNAPSHOT_PATH = "/sharedblock/volume/revertfromsnapshot"
MERGE_SNAPSHOT_PATH = "/sharedblock/snapshot/merge"
OFFLINE_MERGE_SNAPSHOT_PATH = "/sharedblock/snapshot/offlinemerge"
CREATE_EMPTY_VOLUME_PATH = "/sharedblock/volume/createempty"
CHECK_BITS_PATH = "/sharedblock/bits/check"
RESIZE_VOLUME_PATH = "/sharedblock/volume/resize"
CONVERT_IMAGE_TO_VOLUME = "/sharedblock/image/tovolume"
CHANGE_VOLUME_ACTIVE_PATH = "/sharedblock/volume/active"
GET_VOLUME_SIZE_PATH = "/sharedblock/volume/getsize"
CHECK_DISKS_PATH = "/sharedblock/disks/check"
ADD_SHARED_BLOCK = "/sharedblock/disks/add"
MIGRATE_DATA_PATH = "/sharedblock/volume/migrate"
GET_BLOCK_DEVICES_PATH = "/sharedblock/blockdevices"
DOWNLOAD_BITS_FROM_KVM_HOST_PATH = "/sharedblock/kvmhost/download"
CANCEL_DOWNLOAD_BITS_FROM_KVM_HOST_PATH = "/sharedblock/kvmhost/download/cancel"
GET_BACKING_CHAIN_PATH = "/sharedblock/volume/backingchain"
CONVERT_VOLUME_PROVISIONING_PATH = "/sharedblock/volume/convertprovisioning"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.CONNECT_PATH, self.connect)
http_server.register_async_uri(self.DISCONNECT_PATH, self.disconnect)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH, self.create_root_volume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete_bits)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH, self.create_template_from_volume)
http_server.register_async_uri(self.UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH, self.upload_to_sftp)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH, self.download_from_sftp)
http_server.register_async_uri(self.UPLOAD_BITS_TO_IMAGESTORE_PATH, self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_BITS_TO_IMAGESTORE_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_IMAGESTORE_PATH, self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH, self.revert_volume_from_snapshot)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.merge_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_SNAPSHOT_PATH, self.offline_merge_snapshots)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.create_empty_volume)
http_server.register_async_uri(self.CONVERT_IMAGE_TO_VOLUME, self.convert_image_to_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH, self.resize_volume)
http_server.register_async_uri(self.CHANGE_VOLUME_ACTIVE_PATH, self.active_lv)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH, self.get_volume_size)
http_server.register_async_uri(self.CHECK_DISKS_PATH, self.check_disks)
http_server.register_async_uri(self.ADD_SHARED_BLOCK, self.add_disk)
http_server.register_async_uri(self.MIGRATE_DATA_PATH, self.migrate_volumes)
http_server.register_async_uri(self.GET_BLOCK_DEVICES_PATH, self.get_block_devices)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_KVM_HOST_PATH, self.download_from_kvmhost)
http_server.register_async_uri(self.CANCEL_DOWNLOAD_BITS_FROM_KVM_HOST_PATH, self.cancel_download_from_kvmhost)
http_server.register_async_uri(self.GET_BACKING_CHAIN_PATH, self.get_backing_chain)
http_server.register_async_uri(self.CONVERT_VOLUME_PROVISIONING_PATH, self.convert_volume_provisioning)
self.imagestore_client = ImageStoreClient()
def stop(self):
pass
@kvmagent.replyerror
def check_disks(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if cmd.failIfNoPath:
CheckDisk.set_fail_if_no_path()
for diskUuid in cmd.sharedBlockUuids:
disk = CheckDisk(diskUuid)
path = disk.get_path()
if cmd.rescan:
disk.rescan(path.split("/")[-1])
if cmd.vgUuid is not None and lvm.vg_exists(cmd.vgUuid):
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid, False)
return jsonobject.dumps(rsp)
@staticmethod
def create_vg_if_not_found(vgUuid, diskPaths, hostUuid, forceWipe=False):
@linux.retry(times=5, sleep_time=random.uniform(0.1, 3))
def find_vg(vgUuid, raise_exception = True):
cmd = shell.ShellCmd("timeout 5 vgscan --ignorelockingfailure; vgs --nolocking %s -otags | grep %s" % (vgUuid, INIT_TAG))
cmd(is_exception=False)
if cmd.return_code != 0 and raise_exception:
raise RetryException("can not find vg %s with tag %s" % (vgUuid, INIT_TAG))
elif cmd.return_code != 0:
return False
return True
try:
find_vg(vgUuid)
except RetryException as e:
if forceWipe is True:
lvm.wipe_fs(diskPaths, vgUuid)
cmd = shell.ShellCmd("vgcreate -qq --shared --addtag '%s::%s::%s::%s' --metadatasize %s %s %s" %
(INIT_TAG, hostUuid, time.time(), bash.bash_o("hostname").strip(),
DEFAULT_VG_METADATA_SIZE, vgUuid, " ".join(diskPaths)))
cmd(is_exception=False)
logger.debug("created vg %s, ret: %s, stdout: %s, stderr: %s" %
(vgUuid, cmd.return_code, cmd.stdout, cmd.stderr))
if cmd.return_code == 0 and find_vg(vgUuid, False) is True:
return True
try:
if find_vg(vgUuid) is True:
return True
except RetryException as ee:
raise Exception("can not find vg %s with disks: %s and create vg return: %s %s %s " %
(vgUuid, diskPaths, cmd.return_code, cmd.stdout, cmd.stderr))
except Exception as ee:
raise ee
except Exception as e:
raise e
return False
@kvmagent.replyerror
@lock.file_lock(LOCK_FILE)
def connect(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ConnectRsp()
diskPaths = set()
def config_lvm(host_id, enableLvmetad=False):
lvm.backup_lvm_config()
lvm.reset_lvm_conf_default()
lvm.config_lvm_by_sed("use_lvmlockd", "use_lvmlockd=1", ["lvm.conf", "lvmlocal.conf"])
if enableLvmetad:
lvm.config_lvm_by_sed("use_lvmetad", "use_lvmetad=1", ["lvm.conf", "lvmlocal.conf"])
else:
lvm.config_lvm_by_sed("use_lvmetad", "use_lvmetad=0", ["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("host_id", "host_id=%s" % host_id, ["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("sanlock_lv_extend", "sanlock_lv_extend=%s" % DEFAULT_SANLOCK_LV_SIZE, ["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("lvmlockd_lock_retries", "lvmlockd_lock_retries=6", ["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("issue_discards", "issue_discards=1", ["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("reserved_stack", "reserved_stack=256", ["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("reserved_memory", "reserved_memory=131072", ["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_filter(["lvm.conf", "lvmlocal.conf"])
lvm.config_sanlock_by_sed("sh_retries", "sh_retries=20")
lvm.config_sanlock_by_sed("logfile_priority", "logfile_priority=7")
lvm.config_sanlock_by_sed("renewal_read_extend_sec", "renewal_read_extend_sec=24")
lvm.config_sanlock_by_sed("debug_renew", "debug_renew=1")
lvm.config_sanlock_by_sed("use_watchdog", "use_watchdog=0")
sanlock_hostname = "%s-%s-%s" % (cmd.vgUuid[:8], cmd.hostUuid[:8], bash.bash_o("hostname").strip()[:20])
lvm.config_sanlock_by_sed("our_host_name", "our_host_name=%s" % sanlock_hostname)
config_lvm(cmd.hostId, cmd.enableLvmetad)
for diskUuid in cmd.sharedBlockUuids:
disk = CheckDisk(diskUuid)
diskPaths.add(disk.get_path())
lvm.start_lvmlockd()
lvm.check_gl_lock()
logger.debug("find/create vg %s lock..." % cmd.vgUuid)
rsp.isFirst = self.create_vg_if_not_found(cmd.vgUuid, diskPaths, cmd.hostUuid, cmd.forceWipe)
lvm.check_stuck_vglk()
logger.debug("starting vg %s lock..." % cmd.vgUuid)
lvm.start_vg_lock(cmd.vgUuid)
if lvm.lvm_vgck(cmd.vgUuid, 60)[0] is False and lvm.lvm_check_operation(cmd.vgUuid) is False:
lvm.drop_vg_lock(cmd.vgUuid)
logger.debug("restarting vg %s lock..." % cmd.vgUuid)
lvm.check_gl_lock()
lvm.start_vg_lock(cmd.vgUuid)
lvm.clean_vg_exists_host_tags(cmd.vgUuid, cmd.hostUuid, HEARTBEAT_TAG)
lvm.add_vg_tag(cmd.vgUuid, "%s::%s::%s::%s" % (HEARTBEAT_TAG, cmd.hostUuid, time.time(), bash.bash_o('hostname').strip()))
self.clear_stalled_qmp_socket()
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
rsp.hostId = lvm.get_running_host_id(cmd.vgUuid)
rsp.vgLvmUuid = lvm.get_vg_lvm_uuid(cmd.vgUuid)
rsp.hostUuid = cmd.hostUuid
return jsonobject.dumps(rsp)
@staticmethod
@bash.in_bash
def clear_stalled_qmp_socket():
def get_used_qmp_file():
t = bash.bash_o("ps aux | grep -Eo -- '-qmp unix:%s/\w*\.sock'" % QMP_SOCKET_PATH).splitlines()
qmp = []
for i in t:
qmp.append(i.split("/")[-1])
return qmp
exists_qmp_files = set(bash.bash_o("ls %s" % QMP_SOCKET_PATH).splitlines())
if len(exists_qmp_files) == 0:
return
running_qmp_files = set(get_used_qmp_file())
if len(running_qmp_files) == 0:
bash.bash_roe("/bin/rm %s/*" % QMP_SOCKET_PATH)
return
need_delete_qmp_files = exists_qmp_files.difference(running_qmp_files)
if len(need_delete_qmp_files) == 0:
return
for f in need_delete_qmp_files:
bash.bash_roe("/bin/rm %s/%s" % (QMP_SOCKET_PATH, f))
@kvmagent.replyerror
@lock.file_lock(LOCK_FILE)
def disconnect(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
@linux.retry(times=3, sleep_time=random.uniform(0.1, 3))
def find_vg(vgUuid):
cmd = shell.ShellCmd("vgs --nolocking %s -otags | grep %s" % (vgUuid, INIT_TAG))
cmd(is_exception=False)
if cmd.return_code == 0:
return True
logger.debug("can not find vg %s with tag %s" % (vgUuid, INIT_TAG))
cmd = shell.ShellCmd("vgs %s" % vgUuid)
cmd(is_exception=False)
if cmd.return_code == 0:
logger.warn("found vg %s without tag %s" % (vgUuid, INIT_TAG))
return True
raise RetryException("can not find vg %s with or without tag %s" % (vgUuid, INIT_TAG))
try:
find_vg(cmd.vgUuid)
except RetryException:
logger.debug("can not find vg %s; return success" % cmd.vgUuid)
return jsonobject.dumps(rsp)
except Exception as e:
raise e
@linux.retry(times=3, sleep_time=random.uniform(0.1, 3))
def deactive_lvs_on_vg(vgUuid):
active_lvs = lvm.list_local_active_lvs(vgUuid)
if len(active_lvs) == 0:
return
logger.warn("active lvs %s will be deactivate" % active_lvs)
lvm.deactive_lv(vgUuid)
active_lvs = lvm.list_local_active_lvs(vgUuid)
if len(active_lvs) != 0:
raise RetryException("lvs [%s] still active, retry deactive again" % active_lvs)
deactive_lvs_on_vg(cmd.vgUuid)
lvm.clean_vg_exists_host_tags(cmd.vgUuid, cmd.hostUuid, HEARTBEAT_TAG)
lvm.stop_vg_lock(cmd.vgUuid)
if cmd.stopServices:
lvm.quitLockServices()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@lock.file_lock(LOCK_FILE)
def add_disk(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
disk = CheckDisk(cmd.diskUuid)
command = shell.ShellCmd("vgs --nolocking %s -otags | grep %s" % (cmd.vgUuid, INIT_TAG))
command(is_exception=False)
if command.return_code != 0:
self.create_vg_if_not_found(cmd.vgUuid, [disk.get_path()], cmd.hostUuid, cmd.forceWipe)
else:
lvm.check_gl_lock()
if cmd.forceWipe is True:
lvm.wipe_fs([disk.get_path()], cmd.vgUuid)
lvm.add_pv(cmd.vgUuid, disk.get_path(), DEFAULT_VG_METADATA_SIZE)
rsp = AgentRsp
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def resize_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
install_abs_path = translate_absolute_path_from_install_path(cmd.installPath)
with lvm.RecursiveOperateLv(install_abs_path, shared=False):
lvm.resize_lv_from_cmd(install_abs_path, cmd.size, cmd)
if not cmd.live:
shell.call("qemu-img resize %s %s" % (install_abs_path, cmd.size))
ret = linux.qcow2_virtualsize(install_abs_path)
rsp = ResizeVolumeRsp()
rsp.size = ret
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@lock.file_lock(LOCK_FILE)
def create_root_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
template_abs_path_cache = translate_absolute_path_from_install_path(cmd.templatePathInCache)
install_abs_path = translate_absolute_path_from_install_path(cmd.installPath)
qcow2_options = self.calc_qcow2_option(self, cmd.qcow2Options, True, cmd.provisioning)
with lvm.RecursiveOperateLv(template_abs_path_cache, shared=True, skip_deactivate_tags=[IMAGE_TAG]):
virtual_size = linux.qcow2_virtualsize(template_abs_path_cache)
if not lvm.lv_exists(install_abs_path):
lvm.create_lv_from_cmd(install_abs_path, virtual_size, cmd,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(install_abs_path, shared=False, delete_when_exception=True):
linux.qcow2_clone_with_option(template_abs_path_cache, install_abs_path, qcow2_options)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def delete_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if cmd.folder:
raise Exception("not support this operation")
self.do_delete_bits(cmd.path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
def do_delete_bits(self, path):
install_abs_path = translate_absolute_path_from_install_path(path)
if lvm.has_lv_tag(install_abs_path, IMAGE_TAG):
logger.info('deleting lv image: ' + install_abs_path)
lvm.delete_image(install_abs_path, IMAGE_TAG)
else:
logger.info('deleting lv volume: ' + install_abs_path)
lvm.delete_lv(install_abs_path)
@kvmagent.replyerror
def create_template_from_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
volume_abs_path = translate_absolute_path_from_install_path(cmd.volumePath)
install_abs_path = translate_absolute_path_from_install_path(cmd.installPath)
if cmd.sharedVolume:
lvm.do_active_lv(volume_abs_path, lvm.LvmlockdLockType.SHARE, True)
with lvm.RecursiveOperateLv(volume_abs_path, shared=cmd.sharedVolume, skip_deactivate_tags=[IMAGE_TAG]):
virtual_size = linux.qcow2_virtualsize(volume_abs_path)
total_size = 0
for qcow2 in linux.qcow2_get_file_chain(volume_abs_path):
total_size += int(lvm.get_lv_size(qcow2))
if total_size > virtual_size:
total_size = virtual_size
if not lvm.lv_exists(install_abs_path):
lvm.create_lv_from_absolute_path(install_abs_path, total_size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(install_abs_path, shared=False, delete_when_exception=True):
linux.create_template(volume_abs_path, install_abs_path)
logger.debug('successfully created template[%s] from volume[%s]' % (cmd.installPath, cmd.volumePath))
if cmd.compareQcow2 is True:
logger.debug("comparing qcow2 between %s and %s")
bash.bash_errorout("time qemu-img compare %s %s" % (volume_abs_path, install_abs_path))
logger.debug("confirmed qcow2 %s and %s are identical" % (volume_abs_path, install_abs_path))
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@staticmethod
@bash.in_bash
def compare(src, dst):
return bash.bash_r("cmp %s %s" % (src, dst)) == 0
@kvmagent.replyerror
def upload_to_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
install_abs_path = translate_absolute_path_from_install_path(cmd.primaryStorageInstallPath)
def upload():
if not os.path.exists(cmd.primaryStorageInstallPath):
raise kvmagent.KvmError('cannot find %s' % cmd.primaryStorageInstallPath)
linux.scp_upload(cmd.hostname, cmd.sshKey, cmd.primaryStorageInstallPath, cmd.backupStorageInstallPath, cmd.username, cmd.sshPort)
with lvm.OperateLv(install_abs_path, shared=True):
upload()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
install_abs_path = translate_absolute_path_from_install_path(cmd.primaryStorageInstallPath)
self.do_download_from_sftp(cmd, install_abs_path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
def do_download_from_sftp(self, cmd, install_abs_path):
if not lvm.lv_exists(install_abs_path):
size = linux.sftp_get(cmd.hostname, cmd.sshKey, cmd.backupStorageInstallPath, install_abs_path, cmd.username, cmd.sshPort, True)
lvm.create_lv_from_absolute_path(install_abs_path, size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(install_abs_path, shared=False, delete_when_exception=True):
linux.scp_download(cmd.hostname, cmd.sshKey, cmd.backupStorageInstallPath, install_abs_path, cmd.username, cmd.sshPort, cmd.bandWidth)
logger.debug('successfully download %s/%s to %s' % (cmd.hostname, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath))
self.do_active_lv(cmd.primaryStorageInstallPath, cmd.lockType, False)
def cancel_download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
install_abs_path = translate_absolute_path_from_install_path(cmd.primaryStorageInstallPath)
shell.run("pkill -9 -f '%s'" % install_abs_path)
self.do_delete_bits(cmd.primaryStorageInstallPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@completetask
def download_from_kvmhost(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
install_abs_path = translate_absolute_path_from_install_path(cmd.primaryStorageInstallPath)
# todo: assume agent will not restart, maybe need clean
last_task = self.load_and_save_task(req, rsp, os.path.exists, install_abs_path)
if last_task and last_task.agent_pid == os.getpid():
rsp = self.wait_task_complete(last_task)
return jsonobject.dumps(rsp)
self.do_download_from_sftp(cmd, install_abs_path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def cancel_download_from_kvmhost(self, req):
return self.cancel_download_from_sftp(req)
@kvmagent.replyerror
def upload_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(cmd, req)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
@kvmagent.replyerror
def download_from_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.imagestore_client.download_from_imagestore(None, cmd.hostname, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath)
self.do_active_lv(cmd.primaryStorageInstallPath, cmd.lockType, True)
rsp = AgentRsp()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def revert_volume_from_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotRsp()
snapshot_abs_path = translate_absolute_path_from_install_path(cmd.snapshotInstallPath)
qcow2_options = self.calc_qcow2_option(self, cmd.qcow2Options, True, cmd.provisioning)
new_volume_path = cmd.installPath
if new_volume_path is None or new_volume_path == "":
new_volume_path = "/dev/%s/%s" % (cmd.vgUuid, uuidhelper.uuid())
else:
new_volume_path = translate_absolute_path_from_install_path(new_volume_path)
with lvm.RecursiveOperateLv(snapshot_abs_path, shared=True):
size = linux.qcow2_virtualsize(snapshot_abs_path)
lvm.create_lv_from_cmd(new_volume_path, size, cmd,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(new_volume_path, shared=False, delete_when_exception=True):
linux.qcow2_clone_with_option(snapshot_abs_path, new_volume_path, qcow2_options)
size = linux.qcow2_virtualsize(new_volume_path)
rsp.newVolumeInstallPath = new_volume_path
rsp.size = size
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotRsp()
snapshot_abs_path = translate_absolute_path_from_install_path(cmd.snapshotInstallPath)
workspace_abs_path = translate_absolute_path_from_install_path(cmd.workspaceInstallPath)
with lvm.RecursiveOperateLv(snapshot_abs_path, shared=True):
virtual_size = linux.qcow2_virtualsize(snapshot_abs_path)
if not lvm.lv_exists(workspace_abs_path):
lvm.create_lv_from_absolute_path(workspace_abs_path, virtual_size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(workspace_abs_path, shared=False, delete_when_exception=True):
linux.create_template(snapshot_abs_path, workspace_abs_path)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(workspace_abs_path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
rsp.actualSize = rsp.size
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def offline_merge_snapshots(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = OfflineMergeSnapshotRsp()
src_abs_path = translate_absolute_path_from_install_path(cmd.srcPath)
dst_abs_path = translate_absolute_path_from_install_path(cmd.destPath)
with lvm.RecursiveOperateLv(src_abs_path, shared=True):
virtual_size = linux.qcow2_virtualsize(src_abs_path)
if not lvm.lv_exists(dst_abs_path):
lvm.create_lv_from_absolute_path(dst_abs_path, virtual_size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.RecursiveOperateLv(dst_abs_path, shared=False):
if not cmd.fullRebase:
linux.qcow2_rebase(src_abs_path, dst_abs_path)
else:
tmp_lv = 'tmp_%s' % uuidhelper.uuid()
tmp_abs_path = os.path.join(os.path.dirname(dst_abs_path), tmp_lv)
tmp_abs_path = os.path.join(os.path.dirname(dst_abs_path), tmp_lv)
logger.debug("creating temp lv %s" % tmp_abs_path)
lvm.create_lv_from_absolute_path(tmp_abs_path, virtual_size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(tmp_abs_path, shared=False, delete_when_exception=True):
linux.create_template(dst_abs_path, tmp_abs_path)
lvm.lv_rename(tmp_abs_path, dst_abs_path, overwrite=True)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@lock.file_lock(LOCK_FILE)
def create_empty_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
install_abs_path = translate_absolute_path_from_install_path(cmd.installPath)
if cmd.backingFile:
qcow2_options = self.calc_qcow2_option(self, cmd.qcow2Options, True, cmd.provisioning)
backing_abs_path = translate_absolute_path_from_install_path(cmd.backingFile)
with lvm.RecursiveOperateLv(backing_abs_path, shared=True):
virtual_size = linux.qcow2_virtualsize(backing_abs_path)
if not lvm.lv_exists(install_abs_path):
lvm.create_lv_from_cmd(install_abs_path, virtual_size, cmd,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(install_abs_path, shared=False, delete_when_exception=True):
linux.qcow2_create_with_backing_file_and_option(backing_abs_path, install_abs_path, qcow2_options)
elif not lvm.lv_exists(install_abs_path):
lvm.create_lv_from_cmd(install_abs_path, cmd.size, cmd,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
if cmd.volumeFormat != 'raw':
qcow2_options = self.calc_qcow2_option(self, cmd.qcow2Options, False, cmd.provisioning)
with lvm.OperateLv(install_abs_path, shared=False, delete_when_exception=True):
linux.qcow2_create_with_option(install_abs_path, cmd.size, qcow2_options)
linux.qcow2_fill(0, 1048576, install_abs_path)
logger.debug('successfully create empty volume[uuid:%s, size:%s] at %s' % (cmd.volumeUuid, cmd.size, cmd.installPath))
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def convert_image_to_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
install_abs_path = translate_absolute_path_from_install_path(cmd.primaryStorageInstallPath)
with lvm.OperateLv(install_abs_path, shared=False):
lvm.clean_lv_tag(install_abs_path, IMAGE_TAG)
lvm.add_lv_tag(install_abs_path, "%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckBitsRsp()
install_abs_path = translate_absolute_path_from_install_path(cmd.path)
rsp.existing = lvm.lv_exists(install_abs_path)
if cmd.vgUuid is not None and lvm.vg_exists(cmd.vgUuid):
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid, False)
return jsonobject.dumps(rsp)
def do_active_lv(self, installPath, lockType, recursive):
def handle_lv(lockType, fpath):
if lockType > lvm.LvmlockdLockType.NULL:
lvm.active_lv(fpath, lockType == lvm.LvmlockdLockType.SHARE)
else:
lvm.deactive_lv(fpath)
install_abs_path = translate_absolute_path_from_install_path(installPath)
handle_lv(lockType, install_abs_path)
if recursive is False or lockType is lvm.LvmlockdLockType.NULL:
return
while linux.qcow2_get_backing_file(install_abs_path) != "":
install_abs_path = linux.qcow2_get_backing_file(install_abs_path)
if lockType == lvm.LvmlockdLockType.NULL:
handle_lv(lvm.LvmlockdLockType.NULL, install_abs_path)
else:
# activate backing files only in shared mode
handle_lv(lvm.LvmlockdLockType.SHARE, install_abs_path)
@kvmagent.replyerror
def active_lv(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid, raise_exception=False)
self.do_active_lv(cmd.installPath, cmd.lockType, cmd.recursive)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
install_abs_path = translate_absolute_path_from_install_path(cmd.installPath)
with lvm.OperateLv(install_abs_path, shared=True):
rsp.size = linux.qcow2_virtualsize(install_abs_path)
rsp.actualSize = lvm.get_lv_size(install_abs_path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@bash.in_bash
def migrate_volumes(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
for struct in cmd.migrateVolumeStructs:
target_abs_path = translate_absolute_path_from_install_path(struct.targetInstallPath)
current_abs_path = translate_absolute_path_from_install_path(struct.currentInstallPath)
with lvm.OperateLv(current_abs_path, shared=True):
lv_size = lvm.get_lv_size(current_abs_path)
if lvm.lv_exists(target_abs_path):
target_ps_uuid = get_primary_storage_uuid_from_install_path(struct.targetInstallPath)
raise Exception("found %s already exists on ps %s" %
(target_abs_path, target_ps_uuid))
lvm.create_lv_from_absolute_path(target_abs_path, lvm.getOriginalSize(lv_size),
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
lvm.active_lv(target_abs_path, lvm.LvmlockdLockType.SHARE)
try:
for struct in cmd.migrateVolumeStructs:
target_abs_path = translate_absolute_path_from_install_path(struct.targetInstallPath)
current_abs_path = translate_absolute_path_from_install_path(struct.currentInstallPath)
with lvm.OperateLv(current_abs_path, shared=True):
bash.bash_errorout("cp %s %s" % (current_abs_path, target_abs_path))
for struct in cmd.migrateVolumeStructs:
target_abs_path = translate_absolute_path_from_install_path(struct.targetInstallPath)
current_abs_path = translate_absolute_path_from_install_path(struct.currentInstallPath)
with lvm.RecursiveOperateLv(current_abs_path, shared=True):
previous_ps_uuid = get_primary_storage_uuid_from_install_path(struct.currentInstallPath)
target_ps_uuid = get_primary_storage_uuid_from_install_path(struct.targetInstallPath)
current_backing_file = linux.qcow2_get_backing_file(current_abs_path) # type: str
target_backing_file = current_backing_file.replace(previous_ps_uuid, target_ps_uuid)
if struct.compareQcow2:
logger.debug("comparing qcow2 between %s and %s" % (current_abs_path, target_abs_path))
if not self.compare(current_abs_path, target_abs_path):
raise Exception("qcow2 %s and %s are not identical" % (current_abs_path, target_abs_path))
logger.debug("confirmed qcow2 %s and %s are identical" % (current_abs_path, target_abs_path))
if current_backing_file is not None and current_backing_file != "":
lvm.do_active_lv(target_backing_file, lvm.LvmlockdLockType.SHARE, False)
logger.debug("rebase %s to %s" % (target_abs_path, target_backing_file))
linux.qcow2_rebase_no_check(target_backing_file, target_abs_path)
except Exception as e:
for struct in cmd.migrateVolumeStructs:
target_abs_path = translate_absolute_path_from_install_path(struct.targetInstallPath)
if struct.currentInstallPath == struct.targetInstallPath:
logger.debug("current install path %s equals target %s, skip to delete" %
(struct.currentInstallPath, struct.targetInstallPath))
else:
logger.debug("error happened, delete lv %s" % target_abs_path)
lvm.delete_lv(target_abs_path, False)
raise e
finally:
for struct in cmd.migrateVolumeStructs:
target_abs_path = translate_absolute_path_from_install_path(struct.targetInstallPath)
lvm.deactive_lv(target_abs_path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@staticmethod
def calc_qcow2_option(self, options, has_backing_file, provisioning=None):
if options is None or options == "":
return " "
if has_backing_file or provisioning == lvm.VolumeProvisioningStrategy.ThinProvisioning:
return re.sub("-o preallocation=\w* ", " ", options)
return options
@kvmagent.replyerror
def get_block_devices(self, req):
rsp = GetBlockDevicesRsp()
rsp.blockDevices = lvm.get_block_devices()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_backing_chain(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetBackingChainRsp()
abs_path = translate_absolute_path_from_install_path(cmd.installPath)
with lvm.RecursiveOperateLv(abs_path, shared=True, skip_deactivate_tags=[IMAGE_TAG], delete_when_exception=False):
rsp.backingChain = linux.qcow2_get_file_chain(abs_path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@bash.in_bash
def convert_volume_provisioning(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ConvertVolumeProvisioningRsp()
if cmd.provisioningStrategy != "ThinProvisioning":
raise NotImplementedError
abs_path = translate_absolute_path_from_install_path(cmd.installPath)
with lvm.RecursiveOperateLv(abs_path, shared=False):
image_offest = long(
bash.bash_o("qemu-img check %s | grep 'Image end offset' | awk -F ': ' '{print $2}'" % abs_path).strip())
current_size = long(lvm.get_lv_size(abs_path))
virtual_size = linux.qcow2_virtualsize(abs_path)
size = image_offest + cmd.addons[lvm.thinProvisioningInitializeSize]
if size > current_size:
size = current_size
if size > virtual_size:
size = virtual_size
lvm.resize_lv(abs_path, size, True)
rsp.actualSize = size
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
class NfsPrimaryStoragePlugin(kvmagent.KvmAgent):
'''
classdocs
'''
MOUNT_PATH = '/nfsprimarystorage/mount'
UNMOUNT_PATH = '/nfsprimarystorage/unmount'
CREATE_VOLUME_FROM_TEMPLATE_PATH = "/nfsprimarystorage/sftp/createvolumefromtemplate"
CREATE_EMPTY_VOLUME_PATH = "/nfsprimarystorage/createemptyvolume"
GET_CAPACITY_PATH = "/nfsprimarystorage/getcapacity"
CREATE_TEMPLATE_FROM_VOLUME_PATH = "/nfsprimarystorage/sftp/createtemplatefromvolume"
REVERT_VOLUME_FROM_SNAPSHOT_PATH = "/nfsprimarystorage/revertvolumefromsnapshot"
REINIT_IMAGE_PATH = "/nfsprimarystorage/reinitimage"
DELETE_PATH = "/nfsprimarystorage/delete"
CHECK_BITS_PATH = "/nfsprimarystorage/checkbits"
UPLOAD_TO_SFTP_PATH = "/nfsprimarystorage/uploadtosftpbackupstorage"
DOWNLOAD_FROM_SFTP_PATH = "/nfsprimarystorage/downloadfromsftpbackupstorage"
UPLOAD_TO_IMAGESTORE_PATH = "/nfsprimarystorage/imagestore/upload"
COMMIT_TO_IMAGESTORE_PATH = "/nfsprimarystorage/imagestore/commit"
DOWNLOAD_FROM_IMAGESTORE_PATH = "/nfsprimarystorage/imagestore/download"
MERGE_SNAPSHOT_PATH = "/nfsprimarystorage/mergesnapshot"
REBASE_MERGE_SNAPSHOT_PATH = "/nfsprimarystorage/rebaseandmergesnapshot"
MOVE_BITS_PATH = "/nfsprimarystorage/movebits"
OFFLINE_SNAPSHOT_MERGE = "/nfsprimarystorage/offlinesnapshotmerge"
REMOUNT_PATH = "/nfsprimarystorage/remount"
GET_VOLUME_SIZE_PATH = "/nfsprimarystorage/getvolumesize"
PING_PATH = "/nfsprimarystorage/ping"
GET_VOLUME_BASE_IMAGE_PATH = "/nfsprimarystorage/getvolumebaseimage"
UPDATE_MOUNT_POINT_PATH = "/nfsprimarystorage/updatemountpoint"
RESIZE_VOLUME_PATH = "/nfsprimarystorage/volume/resize"
NFS_TO_NFS_MIGRATE_BITS_PATH = "/nfsprimarystorage/migratebits"
NFS_REBASE_VOLUME_BACKING_FILE_PATH = "/nfsprimarystorage/rebasevolumebackingfile"
ERR_UNABLE_TO_FIND_IMAGE_IN_CACHE = "unable to find image in cache"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_sync_uri(self.MOUNT_PATH, self.mount)
http_server.register_sync_uri(self.UNMOUNT_PATH, self.umount)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_TEMPLATE_PATH, self.create_root_volume_from_template)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.create_empty_volume)
http_server.register_async_uri(self.DOWNLOAD_FROM_SFTP_PATH, self.download_from_sftp)
http_server.register_async_uri(self.GET_CAPACITY_PATH, self.get_capacity)
http_server.register_async_uri(self.DELETE_PATH, self.delete)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH, self.create_template_from_root_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH, self.revert_volume_from_snapshot)
http_server.register_async_uri(self.REINIT_IMAGE_PATH, self.reinit_image)
http_server.register_async_uri(self.UPLOAD_TO_SFTP_PATH, self.upload_to_sftp)
http_server.register_async_uri(self.UPLOAD_TO_IMAGESTORE_PATH, self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_TO_IMAGESTORE_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_FROM_IMAGESTORE_PATH, self.download_from_imagestore)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.merge_snapshot)
http_server.register_async_uri(self.REBASE_MERGE_SNAPSHOT_PATH, self.rebase_and_merge_snapshot)
http_server.register_async_uri(self.MOVE_BITS_PATH, self.move_bits)
http_server.register_async_uri(self.OFFLINE_SNAPSHOT_MERGE, self.merge_snapshot_to_volume)
http_server.register_async_uri(self.REMOUNT_PATH, self.remount)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH, self.get_volume_size)
http_server.register_async_uri(self.PING_PATH, self.ping)
http_server.register_async_uri(self.GET_VOLUME_BASE_IMAGE_PATH, self.get_volume_base_image_path)
http_server.register_async_uri(self.UPDATE_MOUNT_POINT_PATH, self.update_mount_point)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH, self.resize_volume)
http_server.register_async_uri(self.NFS_TO_NFS_MIGRATE_BITS_PATH, self.migrate_bits)
http_server.register_async_uri(self.NFS_REBASE_VOLUME_BACKING_FILE_PATH, self.rebase_volume_backing_file)
self.mount_path = {}
self.image_cache = None
self.imagestore_client = ImageStoreClient()
def stop(self):
pass
@kvmagent.replyerror
def migrate_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = NfsToNfsMigrateBitsRsp()
mount_path = cmd.mountPath
dst_folder_path = cmd.dstFolderPath
temp_dir = None
try:
if not cmd.isMounted:
linux.is_valid_nfs_url(cmd.url)
temp_dir = tempfile.mkdtemp()
# dst folder is absolute path
mount_path = temp_dir + mount_path
dst_folder_path = temp_dir + dst_folder_path
if not linux.is_mounted(mount_path, cmd.url):
linux.mount(cmd.url, mount_path, cmd.options, "nfs4")
# Report task progress based on flow chain for now
# To get more accurate progress, we need to report from here someday
# begin migration, then check md5 sums
shell.call("mkdir -p %s; cp -r %s/* %s; sync" % (dst_folder_path, cmd.srcFolderPath, dst_folder_path))
src_md5 = shell.call(
"find %s -type f -exec md5sum {} \; | awk '{ print $1 }' | sort | md5sum" % cmd.srcFolderPath)
dst_md5 = shell.call("find %s -type f -exec md5sum {} \; | awk '{ print $1 }' | sort | md5sum" % dst_folder_path)
if src_md5 != dst_md5:
rsp.error = "failed to copy files from %s to %s, md5sum not match" % (cmd.srcFolderPath, dst_folder_path)
rsp.success = False
if not cmd.isMounted:
linux.umount(mount_path)
finally:
if temp_dir is not None:
return_code = shell.run("mount | grep '%s'" % temp_dir)
if return_code != 0:
# in case dir is not empty
try:
os.rmdir(temp_dir)
except OSError as e:
logger.warn("delete temp_dir %s failed: %s", (temp_dir, str(e)))
else:
logger.warn("temp_dir %s still had mounted destination primary storage, skip cleanup operation" % temp_dir)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def rebase_volume_backing_file(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = NfsRebaseVolumeBackingFileRsp()
if not cmd.dstImageCacheTemplateFolderPath:
qcow2s = shell.call("find %s -type f -regex '.*\.qcow2$'" % cmd.dstVolumeFolderPath)
else:
qcow2s = shell.call("find %s %s -type f -regex '.*\.qcow2$'" % (cmd.dstVolumeFolderPath, cmd.dstImageCacheTemplateFolderPath))
for qcow2 in qcow2s.split():
fmt = shell.call("qemu-img info %s | grep '^file format' | awk -F ': ' '{ print $2 }'" % qcow2)
if fmt.strip() != "qcow2":
continue
backing_file = linux.qcow2_get_backing_file(qcow2)
if backing_file == "":
continue
# actions like `create snapshot -> recover snapshot -> delete snapshot` may produce garbage qcow2, whose backing file doesn't exist
new_backing_file = backing_file.replace(cmd.srcPsMountPath, cmd.dstPsMountPath)
if not os.path.exists(new_backing_file):
logger.debug("the backing file[%s] of volume[%s] doesn't exist, skip rebasing" % (new_backing_file, qcow2))
continue
linux.qcow2_rebase_no_check(new_backing_file, qcow2)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def resize_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
install_path = cmd.installPath
rsp = ResizeVolumeRsp()
shell.call("qemu-img resize %s %s" % (install_path, cmd.size))
ret = linux.qcow2_virtualsize(install_path)
rsp.size = ret
return jsonobject.dumps(rsp)
def _get_disk_capacity(self, uuid):
path = self.mount_path.get(uuid)
if not path:
raise Exception('cannot find mount path of primary storage[uuid: %s]' % uuid)
return linux.get_disk_capacity_by_df(path)
def _json_meta_file_name(self, path):
return path + '.json'
def _set_capacity_to_response(self, uuid, rsp):
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(uuid)
@kvmagent.replyerror
def update_mount_point(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = UpdateMountPointResponse()
linux.is_valid_nfs_url(cmd.newMountPoint)
if not linux.is_mounted(cmd.mountPath, cmd.newMountPoint):
# umount old one
if linux.is_mounted(cmd.mountPath, cmd.oldMountPoint):
linux.umount(cmd.mountPath)
# mount new
linux.mount(cmd.newMountPoint, cmd.mountPath, cmd.options, "nfs4")
self.mount_path[cmd.uuid] = cmd.mountPath
logger.debug('updated the mount path[%s] mounting point from %s to %s' % (cmd.mountPath, cmd.oldMountPoint, cmd.newMountPoint))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_volume_base_image_path(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeBaseImagePathRsp()
if not os.path.basename(cmd.volumeInstallDir).endswith(cmd.volumeUuid):
raise Exception('maybe you pass a wrong install dir')
path = linux.get_qcow2_base_image_recusively(cmd.volumeInstallDir, cmd.imageCacheDir)
if not path:
return jsonobject.dumps(rsp)
rsp.path = path
rsp.size = linux.get_qcow2_file_chain_size(path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@in_bash
def ping(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
mount_path = self.mount_path[cmd.uuid]
# if nfs service stop, os.path.isdir will hung
if not linux.timeout_isdir(mount_path) or not linux.is_mounted(path=mount_path):
raise Exception('the mount path[%s] of the nfs primary storage[uuid:%s] is not existing' % (mount_path, cmd.uuid))
test_file = os.path.join(mount_path, '%s-ping-test-file' % uuidhelper.uuid())
touch = shell.ShellCmd('timeout 60 touch %s' % test_file)
touch(False)
if touch.return_code == 124:
raise Exception('unable to access the mount path[%s] of the nfs primary storage[uuid:%s] in 60s, timeout' %
(mount_path, cmd.uuid))
elif touch.return_code != 0:
touch.raise_error()
linux.rm_file_force(test_file)
return jsonobject.dumps(NfsResponse())
@kvmagent.replyerror
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot_to_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = OfflineMergeSnapshotRsp()
if not cmd.fullRebase:
linux.qcow2_rebase(cmd.srcPath, cmd.destPath)
else:
tmp = os.path.join(os.path.dirname(cmd.destPath), '%s.qcow2' % uuidhelper.uuid())
linux.create_template(cmd.destPath, tmp)
shell.call("mv %s %s" % (tmp, cmd.destPath))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def move_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MoveBitsRsp()
if not os.path.exists(cmd.srcPath):
rsp.error = "%s is not existing" % cmd.srcPath
rsp.success = False
else:
dirname = os.path.dirname(cmd.destPath)
if not os.path.exists(dirname):
os.makedirs(dirname)
shell.call("mv %s %s" % (cmd.srcPath, cmd.destPath))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def rebase_and_merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
snapshots = cmd.snapshotInstallPaths
count = len(snapshots)
for i in range(count):
if i+1 < count:
target = snapshots[i]
backing_file = snapshots[i+1]
linux.qcow2_rebase_no_check(backing_file, target)
latest = snapshots[0]
rsp = RebaseAndMergeSnapshotsResponse()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
try:
linux.create_template(latest, cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = cmd.workspaceInstallPath
self._set_capacity_to_response(cmd.uuid, rsp)
except linux.LinuxError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotResponse()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
try:
linux.create_template(cmd.snapshotInstallPath, cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.workspaceInstallPath)
self._set_capacity_to_response(cmd.uuid, rsp)
except linux.LinuxError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CopyToSftpBackupStorageResponse()
def upload():
if not os.path.exists(cmd.primaryStorageInstallPath):
raise kvmagent.KvmError('cannot find %s' % cmd.primaryStorageInstallPath)
linux.scp_upload(cmd.backupStorageHostName, cmd.backupStorageSshKey, cmd.primaryStorageInstallPath, cmd.backupStorageInstallPath, cmd.backupStorageUserName, cmd.backupStorageSshPort)
try:
upload()
except kvmagent.KvmError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(cmd, req)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
@kvmagent.replyerror
def download_from_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
mount_path = self.mount_path.get(cmd.uuid)
self.check_nfs_mounted(mount_path)
cachedir = None if cmd.isData else mount_path
self.imagestore_client.download_from_imagestore(cachedir, cmd.hostname, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath)
rsp = kvmagent.AgentResponse()
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def reinit_image(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ReInitImageResponse()
install_path = cmd.imagePath
dirname = os.path.dirname(cmd.volumePath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
new_volume_path = os.path.join(dirname, '{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone_with_cmd(install_path, new_volume_path, cmd)
rsp.newVolumeInstallPath = new_volume_path
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def revert_volume_from_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotResponse()
install_path = cmd.snapshotInstallPath
new_volume_path = os.path.join(os.path.dirname(install_path), '{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone_with_cmd(install_path, new_volume_path, cmd)
rsp.newVolumeInstallPath = new_volume_path
size = linux.qcow2_virtualsize(new_volume_path)
rsp.size = size
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckIsBitsExistingRsp()
rsp.existing = os.path.exists(cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def delete(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = DeleteResponse()
if cmd.folder:
linux.rm_dir_checked(cmd.installPath)
else:
kvmagent.deleteImage(cmd.installPath)
logger.debug('successfully delete %s' % cmd.installPath)
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def remount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MountResponse()
linux.is_valid_nfs_url(cmd.url)
linux.remount(cmd.url, cmd.mountPath, cmd.options)
self.mount_path[cmd.uuid] = cmd.mountPath
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@lock.lock('mount')
def mount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MountResponse()
linux.is_valid_nfs_url(cmd.url)
if not linux.is_mounted(cmd.mountPath, cmd.url):
linux.mount(cmd.url, cmd.mountPath, cmd.options, "nfs4")
self.mount_path[cmd.uuid] = cmd.mountPath
logger.debug(http.path_msg(self.MOUNT_PATH, 'mounted %s on %s' % (cmd.url, cmd.mountPath)))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def umount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = UnmountResponse()
if linux.is_mounted(path=cmd.mountPath):
ret = linux.umount(cmd.mountPath)
if not ret: logger.warn(http.path_msg(self.UNMOUNT_PATH, 'unmount %s from %s failed' % (cmd.mountPath, cmd.url)))
logger.debug(http.path_msg(self.UNMOUNT_PATH, 'umounted %s from %s' % (cmd.mountPath, cmd.url)))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_capacity(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetCapacityResponse()
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_empty_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CreateEmptyVolumeResponse()
try:
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname)
linux.qcow2_create_with_cmd(cmd.installUrl, cmd.size, cmd)
except Exception as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = 'unable to create empty volume[uuid:%s, name:%s], %s' % (cmd.uuid, cmd.name, str(e))
rsp.success = False
return jsonobject.dumps(rsp)
meta = VolumeMeta()
meta.account_uuid = cmd.accountUuid
meta.hypervisor_type = cmd.hypervisorType
meta.name = cmd.name
meta.uuid = cmd.volumeUuid
meta.size = cmd.size
meta_path = self._json_meta_file_name(cmd.installUrl)
with open(meta_path, 'w') as fd:
fd.write(jsonobject.dumps(meta, pretty=True))
self._set_capacity_to_response(cmd.uuid, rsp)
logger.debug('successfully create empty volume[uuid:%s, name:%s, size:%s] at %s' % (cmd.uuid, cmd.name, cmd.size, cmd.installUrl))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_template_from_root_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CreateTemplateFromRootVolumeRsp()
try:
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
linux.create_template(cmd.rootVolumePath, cmd.installPath)
except linux.LinuxError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = 'unable to create image to root@%s:%s from root volume[%s], %s' % (cmd.sftpBackupStorageHostName,
cmd.installPath, cmd.rootVolumePath, str(e))
rsp.success = False
self._set_capacity_to_response(cmd.uuid, rsp)
logger.debug('successfully created template[%s] from root volume[%s]' % (cmd.installPath, cmd.rootVolumePath))
return jsonobject.dumps(rsp)
def check_nfs_mounted(self, mount_path):
if not linux.is_mounted(mount_path):
raise Exception('NFS not mounted on: %s' % mount_path)
@kvmagent.replyerror
def download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.check_nfs_mounted(self.mount_path.get(cmd.uuid))
rsp = DownloadBitsFromSftpBackupStorageResponse()
try:
linux.scp_download(cmd.hostname, cmd.sshKey, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath, cmd.username, cmd.sshPort)
logger.debug('successfully download %s/%s to %s' % (cmd.hostname, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath))
self._set_capacity_to_response(cmd.uuid, rsp)
except Exception as e:
content = traceback.format_exc()
logger.warn(content)
err = "unable to download %s/%s, because %s" % (cmd.hostname, cmd.backupStorageInstallPath, str(e))
rsp.error = err
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_root_volume_from_template(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CreateRootVolumeFromTemplateResponse()
if not os.path.exists(cmd.templatePathInCache):
rsp.error = self.ERR_UNABLE_TO_FIND_IMAGE_IN_CACHE
rsp.success = False
return jsonobject.dumps(rsp)
try:
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
linux.qcow2_clone_with_cmd(cmd.templatePathInCache, cmd.installUrl, cmd)
logger.debug('successfully create root volume[%s] from template in cache[%s]' % (cmd.installUrl, cmd.templatePathInCache))
meta = VolumeMeta()
meta.account_uuid = cmd.accountUuid
meta.hypervisor_type = cmd.hypervisorType
meta.name = cmd.name
meta.uuid = cmd.volumeUuid
meta.size = os.path.getsize(cmd.templatePathInCache)
meta_path = self._json_meta_file_name(cmd.installUrl)
with open(meta_path, 'w') as fd:
fd.write(jsonobject.dumps(meta, pretty=True))
self._set_capacity_to_response(cmd.uuid, rsp)
logger.debug('successfully create root volume[%s] from template in cache[%s]' % (cmd.installUrl, cmd.templatePathInCache))
except Exception as e:
content = traceback.format_exc()
logger.warn(content)
err = 'unable to clone qcow2 template[%s] to %s' % (cmd.templatePathInCache, cmd.installUrl)
rsp.error = err
rsp.success = False
return jsonobject.dumps(rsp)
class SharedMountPointPrimaryStoragePlugin(kvmagent.KvmAgent):
CONNECT_PATH = "/sharedmountpointprimarystorage/connect"
CREATE_VOLUME_FROM_CACHE_PATH = "/sharedmountpointprimarystorage/createrootvolume"
DELETE_BITS_PATH = "/sharedmountpointprimarystorage/bits/delete"
CREATE_TEMPLATE_FROM_VOLUME_PATH = "/sharedmountpointprimarystorage/createtemplatefromvolume"
UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH = "/sharedmountpointprimarystorage/sftp/upload"
DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH = "/sharedmountpointprimarystorage/sftp/download"
UPLOAD_BITS_TO_IMAGESTORE_PATH = "/sharedmountpointprimarystorage/imagestore/upload"
COMMIT_BITS_TO_IMAGESTORE_PATH = "/sharedmountpointprimarystorage/imagestore/commit"
DOWNLOAD_BITS_FROM_IMAGESTORE_PATH = "/sharedmountpointprimarystorage/imagestore/download"
REVERT_VOLUME_FROM_SNAPSHOT_PATH = "/sharedmountpointprimarystorage/volume/revertfromsnapshot"
MERGE_SNAPSHOT_PATH = "/sharedmountpointprimarystorage/snapshot/merge"
OFFLINE_MERGE_SNAPSHOT_PATH = "/sharedmountpointprimarystorage/snapshot/offlinemerge"
CREATE_EMPTY_VOLUME_PATH = "/sharedmountpointprimarystorage/volume/createempty"
CHECK_BITS_PATH = "/sharedmountpointprimarystorage/bits/check"
GET_VOLUME_SIZE_PATH = "/sharedmountpointprimarystorage/volume/getsize"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.CONNECT_PATH, self.connect)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH, self.create_root_volume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete_bits)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH, self.create_template_from_volume)
http_server.register_async_uri(self.UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH, self.upload_to_sftp)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH, self.download_from_sftp)
http_server.register_async_uri(self.UPLOAD_BITS_TO_IMAGESTORE_PATH, self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_BITS_TO_IMAGESTORE_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_IMAGESTORE_PATH, self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH, self.revert_volume_from_snapshot)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.merge_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_SNAPSHOT_PATH, self.offline_merge_snapshots)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.create_empty_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH, self.get_volume_size)
self.mount_point = None
self.imagestore_client = ImageStoreClient()
def stop(self):
pass
def _get_disk_capacity(self):
return linux.get_disk_capacity_by_df(self.mount_point)
@kvmagent.replyerror
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def connect(self, req):
none_shared_mount_fs_type = ['xfs', 'ext2', 'ext3', 'ext4', 'vfat', 'tmpfs', 'btrfs']
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.mount_point = cmd.mountPoint
if not os.path.isdir(self.mount_point):
raise kvmagent.KvmError('%s is not a directory, the mount point seems not setup' % self.mount_point)
folder_fs_type = shell.call("df -T %s|tail -1|awk '{print $2}'" % self.mount_point).strip()
if folder_fs_type in none_shared_mount_fs_type:
raise kvmagent.KvmError('%s filesystem is %s, which is not a shared mount point type.' % (self.mount_point, folder_fs_type))
rsp = AgentRsp()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_root_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if not os.path.exists(cmd.templatePathInCache):
rsp.error = "UNABLE_TO_FIND_IMAGE_IN_CACHE"
rsp.success = False
return jsonobject.dumps(rsp)
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
linux.qcow2_clone(cmd.templatePathInCache, cmd.installPath)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def delete_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
shell.call('rm -f %s' % cmd.path)
pdir = os.path.dirname(cmd.path)
linux.rmdir_if_empty(pdir)
logger.debug('successfully delete %s' % cmd.path)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_template_from_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
linux.qcow2_create_template(cmd.volumePath, cmd.installPath)
logger.debug('successfully created template[%s] from volume[%s]' % (cmd.installPath, cmd.volumePath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
def upload():
if not os.path.exists(cmd.primaryStorageInstallPath):
raise kvmagent.KvmError('cannot find %s' % cmd.primaryStorageInstallPath)
linux.scp_upload(cmd.hostname, cmd.sshKey, cmd.primaryStorageInstallPath, cmd.backupStorageInstallPath, cmd.username, cmd.sshPort)
upload()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
linux.scp_download(cmd.hostname, cmd.sshKey, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath, cmd.username, cmd.sshPort)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
logger.debug('successfully download %s/%s to %s' % (cmd.hostname, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(cmd, req)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
@kvmagent.replyerror
def download_from_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.imagestore_client.download_from_imagestore(self.mount_point, cmd.hostname, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath)
rsp = AgentRsp()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def revert_volume_from_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotRsp()
install_path = cmd.snapshotInstallPath
new_volume_path = os.path.join(os.path.dirname(install_path), '{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone(install_path, new_volume_path)
rsp.newVolumeInstallPath = new_volume_path
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotRsp()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
linux.qcow2_create_template(cmd.snapshotInstallPath, cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.workspaceInstallPath)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def offline_merge_snapshots(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if not cmd.fullRebase:
linux.qcow2_rebase(cmd.srcPath, cmd.destPath)
else:
tmp = os.path.join(os.path.dirname(cmd.destPath), '%s.qcow2' % uuidhelper.uuid())
linux.qcow2_create_template(cmd.destPath, tmp)
shell.call("mv %s %s" % (tmp, cmd.destPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_empty_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname)
if cmd.backingFile:
linux.qcow2_create_with_backing_file(cmd.backingFile, cmd.installPath)
else:
linux.qcow2_create(cmd.installPath, cmd.size)
logger.debug('successfully create empty volume[uuid:%s, size:%s] at %s' % (cmd.volumeUuid, cmd.size, cmd.installPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckBitsRsp()
rsp.existing = os.path.exists(cmd.path)
return jsonobject.dumps(rsp)
class MiniStoragePlugin(kvmagent.KvmAgent):
CONNECT_PATH = "/ministorage/connect"
DISCONNECT_PATH = "/ministorage/disconnect"
CREATE_VOLUME_FROM_CACHE_PATH = "/ministorage/createrootvolume"
DELETE_BITS_PATH = "/ministorage/bits/delete"
CREATE_TEMPLATE_FROM_VOLUME_PATH = "/ministorage/createtemplatefromvolume"
UPLOAD_BITS_TO_IMAGESTORE_PATH = "/ministorage/imagestore/upload"
COMMIT_BITS_TO_IMAGESTORE_PATH = "/ministorage/imagestore/commit"
DOWNLOAD_BITS_FROM_IMAGESTORE_PATH = "/ministorage/imagestore/download"
CREATE_EMPTY_VOLUME_PATH = "/ministorage/volume/createempty"
CHECK_BITS_PATH = "/ministorage/bits/check"
RESIZE_VOLUME_PATH = "/ministorage/volume/resize"
CONVERT_IMAGE_TO_VOLUME = "/ministorage/image/tovolume"
CHANGE_VOLUME_ACTIVE_PATH = "/ministorage/volume/active"
GET_VOLUME_SIZE_PATH = "/ministorage/volume/getsize"
CHECK_DISKS_PATH = "/ministorage/disks/check"
MIGRATE_DATA_PATH = "/ministorage/volume/migrate"
REVERT_VOLUME_FROM_SNAPSHOT_PATH = "/ministorage/volume/revertfromsnapshot"
GET_QCOW2_REFERENCE = "/ministorage/getqcow2reference"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.CONNECT_PATH, self.connect)
http_server.register_async_uri(self.DISCONNECT_PATH, self.disconnect)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH,
self.create_root_volume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete_bits)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH,
self.create_template_from_volume)
http_server.register_async_uri(self.UPLOAD_BITS_TO_IMAGESTORE_PATH,
self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_BITS_TO_IMAGESTORE_PATH,
self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_IMAGESTORE_PATH,
self.download_from_imagestore)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH,
self.create_empty_volume)
http_server.register_async_uri(self.CONVERT_IMAGE_TO_VOLUME,
self.convert_image_to_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH,
self.resize_volume)
http_server.register_async_uri(self.CHANGE_VOLUME_ACTIVE_PATH,
self.active_lv)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH,
self.get_volume_size)
http_server.register_async_uri(self.CHECK_DISKS_PATH, self.check_disks)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH,
self.revert_volume_from_snapshot)
http_server.register_async_uri(self.GET_QCOW2_REFERENCE,
self.get_qcow2_reference)
self.imagestore_client = ImageStoreClient()
def stop(self):
pass
@kvmagent.replyerror
def check_disks(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
for diskId in cmd.diskIdentifiers:
disk = CheckDisk(diskId)
path = disk.get_path()
if cmd.rescan:
disk.rescan(path.split("/")[-1])
if cmd.failIfNoPath:
linux.set_fail_if_no_path()
if cmd.vgUuid is not None and lvm.vg_exists(cmd.vgUuid):
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(
cmd.vgUuid, False)
return jsonobject.dumps(rsp)
@staticmethod
@bash.in_bash
def create_thin_pool_if_not_found(vgUuid, init_pool_ratio):
def round_sector(size, sector):
return round(float(size) / float(sector)) * sector
if lvm.lv_exists("/dev/%s/%s_thinpool" % (vgUuid, vgUuid)):
return
tot, avil = lvm.get_vg_size(vgUuid)
init_pool_size = float(tot) * float(init_pool_ratio)
# meta_size = "%s" % ((tot / DEFAULT_CHUNK_SIZE) * 48 * 2) # ref: https://www.kernel.org/doc/Documentation/device-mapper/thin-provisioning.txt
meta_size = 1024**3 # ref: https://www.systutorials.com/docs/linux/man/7-lvmthin/#lbBD
bash.bash_errorout(
"lvcreate --type thin-pool -L %sB -c %sB --poolmetadatasize %sB -n %s_thinpool %s"
% (int(round_sector(init_pool_size, 4096)), DEFAULT_CHUNK_SIZE,
meta_size, vgUuid, vgUuid))
@staticmethod
def create_vg_if_not_found(vgUuid, diskPaths, hostUuid, forceWipe=False):
@linux.retry(times=5, sleep_time=random.uniform(0.1, 3))
def find_vg(vgUuid, raise_exception=True):
cmd = shell.ShellCmd(
"timeout 5 vgscan --ignorelockingfailure; vgs --nolocking %s -otags | grep %s"
% (vgUuid, INIT_TAG))
cmd(is_exception=False)
if cmd.return_code != 0 and raise_exception:
raise RetryException("can not find vg %s with tag %s" %
(vgUuid, INIT_TAG))
elif cmd.return_code != 0:
return False
return True
try:
find_vg(vgUuid)
except RetryException as e:
if forceWipe is True:
running_vm = bash.bash_o(
"virsh list | grep running | awk '{print $2}'").strip(
).split()
if running_vm != [] and running_vm[0] != "":
for vm in running_vm:
bash.bash_r("virsh destroy %s" % vm)
r = bash.bash_r("drbdadm down all")
if r == 0:
bash.bash_r("mkdir -p %s" % BACKUP_DIR)
bash.bash_r("mv /etc/drbd.d/*.res %s" % BACKUP_DIR)
lvm.wipe_fs(diskPaths, vgUuid)
cmd = shell.ShellCmd(
"vgcreate -qq --addtag '%s::%s::%s::%s' --metadatasize %s %s %s"
% (INIT_TAG, hostUuid, time.time(),
bash.bash_o("hostname").strip(), DEFAULT_VG_METADATA_SIZE,
vgUuid, " ".join(diskPaths)))
cmd(is_exception=False)
logger.debug("created vg %s, ret: %s, stdout: %s, stderr: %s" %
(vgUuid, cmd.return_code, cmd.stdout, cmd.stderr))
if cmd.return_code == 0 and find_vg(vgUuid, False) is True:
return True
try:
if find_vg(vgUuid) is True:
return True
except RetryException as ee:
raise Exception(
"can not find vg %s with disks: %s and create vg return: %s %s %s "
% (vgUuid, diskPaths, cmd.return_code, cmd.stdout,
cmd.stderr))
except Exception as ee:
raise ee
except Exception as e:
raise e
return False
@kvmagent.replyerror
@lock.file_lock(LOCK_FILE)
def connect(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ConnectRsp()
diskPaths = set()
def config_lvm(enableLvmetad=False):
lvm.backup_lvm_config()
lvm.reset_lvm_conf_default()
if enableLvmetad:
lvm.config_lvm_by_sed("use_lvmetad", "use_lvmetad=1",
["lvm.conf", "lvmlocal.conf"])
else:
lvm.config_lvm_by_sed("use_lvmetad", "use_lvmetad=0",
["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("issue_discards", "issue_discards=1",
["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("reserved_stack", "reserved_stack=256",
["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("reserved_memory", "reserved_memory=131072",
["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("thin_pool_autoextend_threshold",
"thin_pool_autoextend_threshold=80",
["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("snapshot_autoextend_threshold",
"snapshot_autoextend_threshold=80",
["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_filter(["lvm.conf", "lvmlocal.conf"], True)
def config_drbd():
bash.bash_r(
"sed -i 's/usage-count yes/usage-count no/g' /etc/drbd.d/global_common.conf"
)
bash.bash_r(
"iptables -I INPUT -p tcp -m tcp --dport 20000:30000 -j ACCEPT"
)
drbd.install_drbd()
config_lvm()
config_drbd()
for diskId in cmd.diskIdentifiers:
disk = CheckDisk(diskId)
diskPaths.add(disk.get_path())
logger.debug("find/create vg %s ..." % cmd.vgUuid)
self.create_vg_if_not_found(cmd.vgUuid, diskPaths, cmd.hostUuid,
cmd.forceWipe)
self.create_thin_pool_if_not_found(cmd.vgUuid, INIT_POOL_RATIO)
drbd.up_all_resouces()
if lvm.lvm_check_operation(cmd.vgUuid) is False:
logger.warn("lvm operation test failed!")
lvm.clean_vg_exists_host_tags(cmd.vgUuid, cmd.hostUuid, HEARTBEAT_TAG)
lvm.add_vg_tag(
cmd.vgUuid,
"%s::%s::%s::%s" % (HEARTBEAT_TAG, cmd.hostUuid, time.time(),
bash.bash_o('hostname').strip()))
if cmd.fencerAddress:
lvm.clean_vg_exists_host_tags(cmd.vgUuid, '\'\'', FENCER_TAG)
lvm.add_vg_tag(cmd.vgUuid,
"%s::%s" % (FENCER_TAG, cmd.fencerAddress))
lvm.clean_vg_exists_host_tags(cmd.vgUuid, '\'\'', MANAGEMENT_TAG)
lvm.add_vg_tag(cmd.vgUuid,
"%s::%s" % (MANAGEMENT_TAG, cmd.magementAddress))
self.generate_fencer(cmd.peerManagementAddress, cmd.peerSshUsername,
cmd.peerSshPassword)
if cmd.storageNetworkCidr is not None:
nics = linux.get_nics_by_cidr(cmd.storageNetworkCidr)
if len(nics) != 0:
rsp.storageNetworkAddress = nics[0].values()[0]
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
rsp.vgLvmUuid = lvm.get_vg_lvm_uuid(cmd.vgUuid)
rsp.hostUuid = cmd.hostUuid
return jsonobject.dumps(rsp)
@staticmethod
@bash.in_bash
def generate_fencer(peer_addr, peer_username, peer_password):
def configure_ssh_key():
bash.bash_roe("/bin/rm %s*" % mini_fencer.MINI_FENCER_KEY)
bash.bash_roe("ssh-keygen -P \"\" -f %s" %
mini_fencer.MINI_FENCER_KEY)
r, o, e = bash.bash_roe(
"sshpass -p '%s' ssh-copy-id -i %s %s@%s" %
(peer_password, mini_fencer.MINI_FENCER_KEY, peer_username,
peer_addr))
if r == 0:
return
configure_ssh_key()
current_dir = os.path.split(os.path.realpath(__file__))[0]
fencer_path = "%s/mini_fencer.py" % current_dir
bash.bash_roe(
"sed -i 's/^PEER_USERNAME = .*$/PEER_USERNAME = \"%s\"/g' %s" %
(peer_username, fencer_path))
bash.bash_roe(
"sed -i 's/^PEER_MGMT_ADDR = .*$/PEER_MGMT_ADDR = \"%s\"/g' %s" %
(peer_addr, fencer_path))
bash.bash_roe("cp %s /usr/lib/drbd/mini_fencer.py" % fencer_path)
bash.bash_roe("sudo chmod 777 /usr/lib/drbd/mini_fencer.py")
@kvmagent.replyerror
@lock.file_lock(LOCK_FILE)
def disconnect(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
@linux.retry(times=3, sleep_time=random.uniform(0.1, 3))
def find_vg(vgUuid):
cmd = shell.ShellCmd("vgs --nolocking %s -otags | grep %s" %
(vgUuid, INIT_TAG))
cmd(is_exception=False)
if cmd.return_code == 0:
return True
logger.debug("can not find vg %s with tag %s" % (vgUuid, INIT_TAG))
cmd = shell.ShellCmd("vgs %s" % vgUuid)
cmd(is_exception=False)
if cmd.return_code == 0:
logger.warn("found vg %s without tag %s" % (vgUuid, INIT_TAG))
return True
raise RetryException("can not find vg %s with or without tag %s" %
(vgUuid, INIT_TAG))
@linux.retry(times=3, sleep_time=random.uniform(0.1, 3))
def deactive_drbd_resouces_on_vg(vgUuid):
active_lvs = lvm.list_local_active_lvs(vgUuid)
if len(active_lvs) == 0:
return
drbd_resources = [
drbd.DrbdResource(lv.split("/")[-1]) for lv in active_lvs
]
for r in drbd_resources:
r.destroy()
logger.warn("active lvs %s will be deactivate" % active_lvs)
lvm.deactive_lv(vgUuid)
active_lvs = lvm.list_local_active_lvs(vgUuid)
if len(active_lvs) != 0:
raise RetryException(
"lvs [%s] still active, retry deactive again" % active_lvs)
try:
find_vg(cmd.vgUuid)
except RetryException:
logger.debug("can not find vg %s; return success" % cmd.vgUuid)
return jsonobject.dumps(rsp)
except Exception as e:
raise e
deactive_drbd_resouces_on_vg(cmd.vgUuid)
lvm.clean_vg_exists_host_tags(cmd.vgUuid, cmd.hostUuid, HEARTBEAT_TAG)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@lock.file_lock(LOCK_FILE)
def add_disk(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
disk = CheckDisk(cmd.diskUuid)
command = shell.ShellCmd("vgs --nolocking %s -otags | grep %s" %
(cmd.vgUuid, INIT_TAG))
command(is_exception=False)
if command.return_code != 0:
self.create_vg_if_not_found(cmd.vgUuid, [disk.get_path()],
cmd.hostUuid, cmd.forceWipe)
else:
lvm.check_gl_lock()
if cmd.forceWipe is True:
lvm.wipe_fs([disk.get_path()], cmd.vgUuid)
lvm.add_pv(cmd.vgUuid, disk.get_path(), DEFAULT_VG_METADATA_SIZE)
rsp = AgentRsp
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def resize_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
install_abs_path = get_absolute_path_from_install_path(cmd.installPath)
rsp = ResizeVolumeRsp()
if not cmd.drbd:
lvm.resize_lv_from_cmd(install_abs_path, cmd.size, cmd)
return jsonobject.dumps(rsp)
r = drbd.DrbdResource(cmd.installPath.split("/")[-1])
r._init_from_disk(install_abs_path)
with drbd.OperateDrbd(r):
r.resize()
with drbd.OperateDrbd(r):
if not cmd.live:
shell.call("qemu-img resize %s %s" %
(r.get_dev_path(), cmd.size))
ret = linux.qcow2_virtualsize(r.get_dev_path())
rsp.size = ret
rsp._init_from_drbd(r)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_root_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = VolumeRsp()
template_abs_path_cache = get_absolute_path_from_install_path(
cmd.templatePathInCache)
install_abs_path = get_absolute_path_from_install_path(cmd.installPath)
drbdResource = drbd.DrbdResource(
self.get_name_from_installPath(cmd.installPath), False)
drbdResource.config.local_host.hostname = cmd.local_host_name
drbdResource.config.local_host.disk = install_abs_path
drbdResource.config.local_host.minor = cmd.local_host_port - DRBD_START_PORT
drbdResource.config.local_host.address = "%s:%s" % (
cmd.local_address, cmd.local_host_port)
drbdResource.config.remote_host.hostname = cmd.remote_host_name
drbdResource.config.remote_host.disk = install_abs_path
drbdResource.config.remote_host.minor = cmd.remote_host_port - DRBD_START_PORT
drbdResource.config.remote_host.address = "%s:%s" % (
cmd.remote_address, cmd.remote_host_port)
drbdResource.config.write_config()
virtual_size = linux.qcow2_virtualsize(template_abs_path_cache)
try:
lvm.qcow2_lv_recursive_active(template_abs_path_cache,
lvm.LvmlockdLockType.SHARE)
if not lvm.lv_exists(install_abs_path):
lvm.create_lv_from_cmd(
install_abs_path, virtual_size, cmd,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()),
False)
lvm.active_lv(install_abs_path)
drbdResource.initialize(cmd.init, cmd, template_abs_path_cache)
except Exception as e:
drbdResource.destroy()
lvm.delete_lv(install_abs_path)
raise e
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
rsp._init_from_drbd(drbdResource)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def delete_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if cmd.folder:
raise Exception("not support this operation")
self.do_delete_bits(cmd.path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
def do_delete_bits(self, path):
install_abs_path = get_absolute_path_from_install_path(path)
if lvm.has_lv_tag(install_abs_path, IMAGE_TAG):
logger.info('deleting lv image: ' + install_abs_path)
if lvm.lv_exists(install_abs_path):
lvm.delete_image(install_abs_path, IMAGE_TAG)
else:
logger.info('deleting lv volume: ' + install_abs_path)
r = drbd.DrbdResource(self.get_name_from_installPath(path))
r.destroy()
lvm.delete_lv(install_abs_path)
lvm.delete_snapshots(install_abs_path)
@kvmagent.replyerror
def get_qcow2_reference(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetQCOW2ReferenceRsp()
rsp.referencePaths = []
real_path = get_absolute_path_from_install_path(cmd.path)
for f in lvm.list_local_active_lvs(cmd.vgUuid):
backing_file = linux.qcow2_direct_get_backing_file(f)
if backing_file == real_path:
rsp.referencePaths.append(f)
for f in bash.bash_o(
"ls -l /dev/drbd* | grep -E '^b' | awk '{print $NF}'"
).splitlines():
f = f.strip()
if f == "":
continue
try:
if linux.qcow2_get_backing_file(f) == real_path:
rsp.referencePaths.append(f)
except IOError:
continue
logger.debug("find qcow2 %s referencess: %s" %
(real_path, rsp.referencePaths))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_template_from_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = VolumeRsp()
volume_abs_path = get_absolute_path_from_install_path(cmd.volumePath)
snap_name = cmd.installPath.split("/")[-1]
lvm.create_lvm_snapshot(volume_abs_path, snapName=snap_name)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(cmd, req)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
@kvmagent.replyerror
def download_from_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.imagestore_client.download_from_imagestore(
cmd.mountPoint, cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath)
rsp = AgentRsp()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_empty_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = VolumeRsp()
install_abs_path = get_absolute_path_from_install_path(cmd.installPath)
drbdResource = drbd.DrbdResource(
self.get_name_from_installPath(cmd.installPath), False)
drbdResource.config.local_host.hostname = cmd.local_host_name
drbdResource.config.local_host.disk = install_abs_path
drbdResource.config.local_host.minor = cmd.local_host_port - DRBD_START_PORT
drbdResource.config.local_host.address = "%s:%s" % (
cmd.local_address, cmd.local_host_port)
drbdResource.config.remote_host.hostname = cmd.remote_host_name
drbdResource.config.remote_host.disk = install_abs_path
drbdResource.config.remote_host.minor = cmd.remote_host_port - DRBD_START_PORT
drbdResource.config.remote_host.address = "%s:%s" % (
cmd.remote_address, cmd.remote_host_port)
drbdResource.config.write_config()
try:
if cmd.backingFile:
backing_abs_path = get_absolute_path_from_install_path(
cmd.backingFile)
virtual_size = linux.qcow2_virtualsize(backing_abs_path)
lvm.create_lv_from_cmd(
install_abs_path, virtual_size, cmd,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()),
False)
lvm.active_lv(install_abs_path)
drbdResource.initialize(cmd.init, cmd, backing_abs_path)
elif not lvm.lv_exists(install_abs_path):
lvm.create_lv_from_cmd(
install_abs_path, cmd.size, cmd,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()),
False)
lvm.active_lv(install_abs_path)
drbdResource.initialize(cmd.init, cmd)
except Exception as e:
drbdResource.destroy()
lvm.delete_lv(install_abs_path)
logger.debug(
'failed to create empty volume[uuid:%s, size:%s] at %s' %
(cmd.volumeUuid, cmd.size, cmd.installPath))
raise e
logger.debug(
'successfully create empty volume[uuid:%s, size:%s] at %s' %
(cmd.volumeUuid, cmd.size, cmd.installPath))
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
rsp._init_from_drbd(drbdResource)
return jsonobject.dumps(rsp)
@staticmethod
def get_name_from_installPath(path):
return path.split("/")[3]
@kvmagent.replyerror
def convert_image_to_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = VolumeRsp()
install_abs_path = get_absolute_path_from_install_path(
cmd.primaryStorageInstallPath)
lvm.active_lv(install_abs_path)
lvm.clean_lv_tag(install_abs_path, IMAGE_TAG)
lvm.add_lv_tag(install_abs_path,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
drbdResource = drbd.DrbdResource(
install_abs_path.split("/")[-1], False)
drbdResource.config.local_host.hostname = cmd.local_host_name
drbdResource.config.local_host.disk = install_abs_path
drbdResource.config.local_host.minor = cmd.local_host_port - DRBD_START_PORT
drbdResource.config.local_host.address = "%s:%s" % (
cmd.local_address, cmd.local_host_port)
drbdResource.config.remote_host.hostname = cmd.remote_host_name
drbdResource.config.remote_host.disk = install_abs_path
drbdResource.config.remote_host.minor = cmd.remote_host_port - DRBD_START_PORT
drbdResource.config.remote_host.address = "%s:%s" % (
cmd.remote_address, cmd.remote_host_port)
drbdResource.config.write_config()
drbdResource.initialize(False, None, skip_clear_bits=cmd.init)
rsp._init_from_drbd(drbdResource)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckBitsRsp()
install_abs_path = get_absolute_path_from_install_path(cmd.path)
rsp.existing = lvm.lv_exists(install_abs_path)
if cmd.vgUuid is not None and lvm.vg_exists(cmd.vgUuid):
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(
cmd.vgUuid, False)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def active_lv(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ActiveRsp()
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(
cmd.vgUuid, raise_exception=False)
install_abs_path = get_absolute_path_from_install_path(cmd.installPath)
if lvm.has_lv_tag(install_abs_path, IMAGE_TAG):
lvm.qcow2_lv_recursive_active(install_abs_path,
lvm.LvmlockdLockType.SHARE)
return jsonobject.dumps(rsp)
drbdResource = drbd.DrbdResource(
self.get_name_from_installPath(cmd.installPath))
if cmd.role == drbd.DrbdRole.Secondary:
drbdResource.demote()
rsp._init_from_drbd(drbdResource)
return jsonobject.dumps(rsp)
if self.test_network_ok_to_peer(drbdResource.config.remote_host.address.split(":")[0]) is False \
and mini_fencer.test_fencer(cmd.vgUuid, drbdResource.name) is False:
raise Exception("can not connect storage network or fencer")
if cmd.checkPeer and drbdResource.get_remote_role(
) == drbd.DrbdRole.Primary:
raise Exception("remote is also in primary role, can not promote")
if drbdResource.get_dstate() != "UpToDate":
raise Exception("local data is not uptodate, can not promote")
lvm.qcow2_lv_recursive_active(install_abs_path,
lvm.LvmlockdLockType.EXCLUSIVE)
try:
drbdResource.promote()
except Exception as e:
if not cmd.force:
raise e
if self.test_network_ok_to_peer(
drbdResource.config.remote_host.address.split(":")[0]):
raise Exception(
"storage network address %s still connected, wont force promote"
% drbdResource.config.remote_host.address.split(":")[0])
if cmd.vmNics:
for vmNic in cmd.vmNics:
if self.test_network_ok_to_peer(vmNic.ipAddress,
vmNic.bridgeName):
raise Exception(
"could arping %s via %s, it may split brain, wont proceed force promote"
% (vmNic.ipAddress, vmNic.bridgeName))
snap_path = None
try:
snap_path = lvm.create_lvm_snapshot(install_abs_path)
drbdResource.promote(True, 2, 2)
rsp.snapPath = snap_path
except Exception as ee:
if snap_path is not None:
lvm.delete_lv(snap_path)
raise ee
rsp._init_from_drbd(drbdResource)
return jsonobject.dumps(rsp)
@staticmethod
@bash.in_bash
def test_network_ok_to_peer(peer_address, via_dev=None):
if not via_dev:
via_dev = bash.bash_o("ip -o r get %s | awk '{print $3}'" %
peer_address).strip()
for i in range(5):
recv = bash.bash_r("timeout 2 arping -w 1 -b %s -I %s -c 1" %
(peer_address, via_dev))
if recv == 0:
return True
return False
@kvmagent.replyerror
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
install_abs_path = get_absolute_path_from_install_path(cmd.installPath)
r = drbd.DrbdResource(cmd.installPath.split("/")[-1])
with drbd.OperateDrbd(r):
rsp.size = linux.qcow2_virtualsize(r.get_dev_path())
rsp.actualSize = lvm.get_lv_size(install_abs_path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
rsp._init_from_drbd(r)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def revert_volume_from_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotRsp()
snapshot_abs_path = get_absolute_path_from_install_path(
cmd.snapshotInstallPath)
install_abs_path = get_absolute_path_from_install_path(cmd.installPath)
rsp.size = False
rsp.error = "not supported yet!"
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return rsp
class SharedMountPointPrimaryStoragePlugin(kvmagent.KvmAgent):
CONNECT_PATH = "/sharedmountpointprimarystorage/connect"
CREATE_VOLUME_FROM_CACHE_PATH = "/sharedmountpointprimarystorage/createrootvolume"
DELETE_BITS_PATH = "/sharedmountpointprimarystorage/bits/delete"
CREATE_TEMPLATE_FROM_VOLUME_PATH = "/sharedmountpointprimarystorage/createtemplatefromvolume"
UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH = "/sharedmountpointprimarystorage/sftp/upload"
DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH = "/sharedmountpointprimarystorage/sftp/download"
UPLOAD_BITS_TO_IMAGESTORE_PATH = "/sharedmountpointprimarystorage/imagestore/upload"
COMMIT_BITS_TO_IMAGESTORE_PATH = "/sharedmountpointprimarystorage/imagestore/commit"
DOWNLOAD_BITS_FROM_IMAGESTORE_PATH = "/sharedmountpointprimarystorage/imagestore/download"
REVERT_VOLUME_FROM_SNAPSHOT_PATH = "/sharedmountpointprimarystorage/volume/revertfromsnapshot"
MERGE_SNAPSHOT_PATH = "/sharedmountpointprimarystorage/snapshot/merge"
OFFLINE_MERGE_SNAPSHOT_PATH = "/sharedmountpointprimarystorage/snapshot/offlinemerge"
CREATE_EMPTY_VOLUME_PATH = "/sharedmountpointprimarystorage/volume/createempty"
CHECK_BITS_PATH = "/sharedmountpointprimarystorage/bits/check"
GET_VOLUME_SIZE_PATH = "/sharedmountpointprimarystorage/volume/getsize"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.CONNECT_PATH, self.connect)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH, self.create_root_volume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete_bits)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH, self.create_template_from_volume)
http_server.register_async_uri(self.UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH, self.upload_to_sftp)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH, self.download_from_sftp)
http_server.register_async_uri(self.UPLOAD_BITS_TO_IMAGESTORE_PATH, self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_BITS_TO_IMAGESTORE_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_IMAGESTORE_PATH, self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH, self.revert_volume_from_snapshot)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.merge_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_SNAPSHOT_PATH, self.offline_merge_snapshots)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.create_empty_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH, self.get_volume_size)
self.imagestore_client = ImageStoreClient()
self.id_file = None
def stop(self):
pass
@staticmethod
def _get_disk_capacity(mount_point):
if not mount_point:
raise Exception('storage mount point cannot be None')
return linux.get_disk_capacity_by_df(mount_point)
@kvmagent.replyerror
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def connect(self, req):
none_shared_mount_fs_type = ['xfs', 'ext2', 'ext3', 'ext4', 'vfat', 'tmpfs', 'btrfs']
cmd = jsonobject.loads(req[http.REQUEST_BODY])
if not linux.timeout_isdir(cmd.mountPoint):
raise kvmagent.KvmError('%s is not a directory, the mount point seems not setup' % cmd.mountPoint)
folder_fs_type = shell.call("df -T %s|tail -1|awk '{print $2}'" % cmd.mountPoint).strip()
if folder_fs_type in none_shared_mount_fs_type:
raise kvmagent.KvmError(
'%s filesystem is %s, which is not a shared mount point type.' % (cmd.mountPoint, folder_fs_type))
id_dir = os.path.join(cmd.mountPoint, "zstack_smp_id_file")
shell.call("mkdir -p %s" % id_dir)
lock_file = os.path.join(id_dir, "uuid.lock")
@lock.file_lock(lock_file)
def check_other_smp_and_set_id_file(uuid, existUuids):
o = shell.ShellCmd('''
ls %s | grep -v %s | grep -o "[0-9a-f]\{8\}[0-9a-f]\{4\}[1-5][0-9a-f]\{3\}[89ab][0-9a-f]\{3\}[0-9a-f]\{12\}"
''' % (id_dir, uuid))
o(False)
if o.return_code != 0:
file_uuids = []
else:
file_uuids = o.stdout.split("\n")
for file_uuid in file_uuids:
if file_uuid in existUuids:
raise Exception(
"the mount point [%s] has been occupied by other SMP[uuid:%s], Please attach this directly"
% (cmd.mountPoint, file_uuid))
self.id_file = os.path.join(id_dir, uuid)
if not os.path.exists(self.id_file):
# check if hosts in the same cluster mount the same path but different storages.
rsp.isFirst = True
need_clean_file = os.path.join(id_dir, "*")
shell.call("rm -rf %s" % need_clean_file)
shell.call("touch %s" % self.id_file)
rsp = ConnectRsp()
check_other_smp_and_set_id_file(cmd.uuid, cmd.existUuids)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_root_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if not os.path.exists(cmd.templatePathInCache):
rsp.error = "UNABLE_TO_FIND_IMAGE_IN_CACHE"
rsp.success = False
return jsonobject.dumps(rsp)
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
linux.qcow2_clone(cmd.templatePathInCache, cmd.installPath)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def delete_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
kvmagent.deleteImage(cmd.path)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_template_from_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
linux.qcow2_create_template(cmd.volumePath, cmd.installPath)
logger.debug('successfully created template[%s] from volume[%s]' % (cmd.installPath, cmd.volumePath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
def upload():
if not os.path.exists(cmd.primaryStorageInstallPath):
raise kvmagent.KvmError('cannot find %s' % cmd.primaryStorageInstallPath)
linux.scp_upload(cmd.hostname, cmd.sshKey, cmd.primaryStorageInstallPath, cmd.backupStorageInstallPath, cmd.username, cmd.sshPort)
upload()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
linux.scp_download(cmd.hostname, cmd.sshKey, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath, cmd.username, cmd.sshPort)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
logger.debug('successfully download %s/%s to %s' % (cmd.hostname, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(cmd, req)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
@kvmagent.replyerror
def download_from_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.imagestore_client.download_from_imagestore(cmd.mountPoint, cmd.hostname, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath)
rsp = AgentRsp()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def revert_volume_from_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotRsp()
install_path = cmd.snapshotInstallPath
new_volume_path = os.path.join(os.path.dirname(install_path), '{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone(install_path, new_volume_path)
rsp.newVolumeInstallPath = new_volume_path
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotRsp()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
linux.qcow2_create_template(cmd.snapshotInstallPath, cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.workspaceInstallPath)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def offline_merge_snapshots(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if not cmd.fullRebase:
linux.qcow2_rebase(cmd.srcPath, cmd.destPath)
else:
tmp = os.path.join(os.path.dirname(cmd.destPath), '%s.qcow2' % uuidhelper.uuid())
linux.qcow2_create_template(cmd.destPath, tmp)
shell.call("mv %s %s" % (tmp, cmd.destPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_empty_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname)
if cmd.backingFile:
linux.qcow2_create_with_backing_file(cmd.backingFile, cmd.installPath)
else:
linux.qcow2_create(cmd.installPath, cmd.size)
logger.debug('successfully create empty volume[uuid:%s, size:%s] at %s' % (cmd.volumeUuid, cmd.size, cmd.installPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckBitsRsp()
rsp.existing = os.path.exists(cmd.path)
return jsonobject.dumps(rsp)
class AliyunNasStoragePlugin(kvmagent.KvmAgent):
MOUNT_PATH = "/aliyun/nas/primarystorage/firstmount"
IS_MOUNT_PATH = "/aliyun/nas/primarystorage/ismount"
MOUNT_DATA_PATH = "/aliyun/nas/primarystorage/mountdata"
INIT_PATH = "/aliyun/nas/primarystorage/init"
PING_PATH = "/aliyun/nas/primarystorage/ping"
GET_CAPACITY_PATH = "/aliyun/nas/primarystorage/getcapacity"
LIST_PATH = "/aliyun/nas/primarystorage/list"
UPDATE_MOUNT_POINT_PATH = "/aliyun/nas/primarystorage/updatemountpoint"
REMOUNT_PATH = "/aliyun/nas/primarystorage/remount"
UNMOUNT_PATH = "/aliyun/nas/primarystorage/unmount"
CHECK_BITS_PATH = "/aliyun/nas/primarystorage/checkbits"
CREATE_EMPTY_VOLUME_PATH = "/aliyun/nas/primarystorage/createempty"
CREATE_VOLUME_FROM_CACHE_PATH = "/aliyun/nas/primarystorage/createvolume"
DELETE_BITS_PATH = "/aliyun/nas/primarystorage/deletebits"
GET_VOLUME_SIZE_PATH = "/aliyun/nas/primarystorage/getvolumesize"
REVERT_VOLUME_FROM_SNAPSHOT_PATH = "/aliyun/nas/primarystorage/revertvolume"
DOWNLOAD_BIT_TO_IMAGESTORE_PATH = "/aliyun/nas/primarystorage/imagestore/download"
UPLOAD_BIT_TO_IMAGESTORE__PATH = "/aliyun/nas/primarystorage/imagestore/upload"
REINIT_VOLUME_PATH = "/aliyun/nas/primarystorage/reinit"
RESIZE_VOLUME_PATH = "/aliyun/nas/primarystorage/resize"
COMMIT_PATH = "/aliyun/nas/primarystorage/commit"
CREATE_TEMPLATE_FROM_VOLUME_PATH = "/aliyun/nas/primarystorage/createtemplatefromvolume"
MERGE_SNAPSHOT_PATH = "/aliyun/nas/primarystorage/mergesnapshot"
OFFLINE_MERGE_SNAPSHOT_PATH = "/aliyun/nas/primarystorage/snapshot/offlinemerge"
CHECK_MOUNT_PATH = "/aliyun/nas/primarystorage/checkmount"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.MOUNT_PATH, self.mount)
http_server.register_async_uri(self.IS_MOUNT_PATH, self.ismount)
http_server.register_async_uri(self.MOUNT_DATA_PATH, self.mountdata)
http_server.register_async_uri(self.INIT_PATH, self.init)
http_server.register_async_uri(self.PING_PATH, self.ping)
http_server.register_async_uri(self.LIST_PATH, self.list)
http_server.register_async_uri(self.UPDATE_MOUNT_POINT_PATH,
self.updateMount)
http_server.register_async_uri(self.REMOUNT_PATH, self.remount)
http_server.register_async_uri(self.UNMOUNT_PATH, self.umount)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.checkbits)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH,
self.createempty)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH,
self.createvolume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.deletebits)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH,
self.getvolumesize)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH,
self.revertvolume)
http_server.register_async_uri(self.REINIT_VOLUME_PATH, self.reinit)
http_server.register_async_uri(self.UPLOAD_BIT_TO_IMAGESTORE__PATH,
self.uploadtoimagestore)
http_server.register_async_uri(self.DOWNLOAD_BIT_TO_IMAGESTORE_PATH,
self.downloadfromimagestore)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH, self.resize)
http_server.register_async_uri(self.COMMIT_PATH,
self.commit_to_imagestore)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH,
self.createtemplate)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH,
self.mergesnapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_SNAPSHOT_PATH,
self.offlinemerge)
http_server.register_async_uri(self.GET_CAPACITY_PATH,
self.getcapacity)
http_server.register_async_uri(self.CHECK_MOUNT_PATH,
self.checkmountpath)
self.mount_path = {}
self.uuid = None
self.imagestore_client = ImageStoreClient()
def stop(self):
pass
def _set_capacity_to_response(self, uuid, rsp):
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
uuid)
def _get_disk_capacity(self, uuid):
path = self.mount_path.get(uuid)
if not path:
raise Exception(
'cannot find mount path of primary storage[uuid: %s]' % uuid)
return linux.get_disk_capacity_by_df(path)
@kvmagent.replyerror
def mount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
linux.is_valid_nfs_url(cmd.url)
if not naslinux.is_mounted(cmd.mountPath, cmd.url):
linux.mount(cmd.url, cmd.mountPath, cmd.options)
logger.debug(
http.path_msg(self.MOUNT_PATH,
'mounted %s on %s' % (cmd.url, cmd.mountPath)))
rsp.mounted = True
self.mount_path[cmd.uuid] = cmd.mountPath
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def ismount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = IsMountResponse()
linux.is_valid_nfs_url(cmd.url)
if naslinux.is_mounted(cmd.mountPath, cmd.url):
rsp.ismounted = True
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def mountdata(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
naslinux.createCommonPath(cmd.mountPath, cmd.basePath)
if not naslinux.is_mounted(cmd.dataPath, cmd.url):
linux.mount(cmd.url, cmd.dataPath, cmd.options)
logger.debug(
http.path_msg(self.MOUNT_DATA_PATH,
'mounted %s on %s' % (cmd.url, cmd.dataPath)))
rsp.mounted = True
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def init(self, req):
'''
cmd.url --> domain:/ps-[uuid]
cmd.mountPath --> /opt/ps
cmd.common --> /opt/ps/commons
cmd.data --> /opt/ps/datas
cmd.dirs --> []
'''
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = InitResponse()
linux.is_valid_nfs_url(cmd.url)
'''
example:
1. mount url: /opt/ps
2. mkdir /opt/ps/ps-[uuid]
3. mkdir /opt/ps/ps-[uuid]/commons/xxx.. (such as heartbeat, cache, ..)
at last we get url:/ps-[uuid] for hosts mount
'''
domain = cmd.url.split(':')[0] + ":/"
psDir = cmd.url.split(':')[1].lstrip('/')
basedir = os.path.join(cmd.mountPath, psDir)
'''
check if mounted {cmd.mountPath}
'''
if linux.is_mounted(path=cmd.mountPath) and not naslinux.is_mounted(
cmd.mountPath, cmd.url):
raise Exception('mountPath[%s] already mount to another url' %
cmd.mountPath)
linux.mount(domain, cmd.mountPath, cmd.options)
shell.call('mkdir -p %s' % basedir)
for dir in cmd.dirs:
shell.call('mkdir -p %s' % os.path.join(basedir, dir))
linux.umount(cmd.mountPath)
common_dir = os.path.join(cmd.mountPath, cmd.common)
data_dir = os.path.join(cmd.mountPath, cmd.data)
shell.call('mkdir -p %s' % common_dir)
shell.call('mkdir -p %s' % data_dir)
linux.mount(cmd.url, common_dir, cmd.options)
rsp.mounted = True
self.mount_path[cmd.uuid] = common_dir
self._set_capacity_to_response(cmd.uuid, rsp)
self.uuid = cmd.uuid
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@in_bash
def ping(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
mount_path = cmd.mountPath
# if nfs service stop, os.path.isdir will hung
if not linux.timeout_isdir(mount_path) or not linux.is_mounted(
path=mount_path):
raise Exception(
'the mount path[%s] of the nas primary storage[uuid:%s] is not existing'
% (mount_path, cmd.uuid))
test_file = os.path.join(mount_path, cmd.heartbeat,
'%s-ping-test-file' % cmd.uuid)
touch = shell.ShellCmd('timeout 60 touch %s' % test_file)
touch(False)
if touch.return_code == 124:
raise Exception(
'unable to access the mount path[%s] of the nas primary storage[uuid:%s] in 60s, timeout'
% (mount_path, cmd.uuid))
elif touch.return_code != 0:
touch.raise_error()
shell.call('rm -f %s' % test_file)
return jsonobject.dumps(AliyunNasResponse())
@kvmagent.replyerror
def getvolumesize(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
self._set_capacity_to_response(cmd.uuid, rsp)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def list(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ListResponse()
rsp.paths = kvmagent.listPath(cmd.path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def updateMount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
linux.is_valid_nfs_url(cmd.newUrl)
if not naslinux.is_mounted(cmd.mountPath, cmd.newUrl):
# umount old one
if naslinux.is_mounted(cmd.mountPath, cmd.url):
linux.umount(cmd.mountPath)
# mount new
linux.mount(cmd.newUrl, cmd.mountPath, cmd.options)
self.mount_path[cmd.uuid] = cmd.mountPath
logger.debug(
'updated the mount path[%s] mounting point from %s to %s' %
(cmd.mountPath, cmd.url, cmd.newUrl))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def remount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
linux.is_valid_nfs_url(cmd.url)
naslinux.remount(cmd.url, cmd.mountPath, cmd.options)
self.mount_path[cmd.uuid] = cmd.mountPath
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def umount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
if naslinux.is_mounted(path=cmd.mountPath):
ret = linux.umount(cmd.mountPath)
if not ret:
logger.warn(
http.path_msg(self.UNMOUNT_PATH,
'unmount %s failed' % cmd.mountPath))
logger.debug(
http.path_msg(self.UNMOUNT_PATH, 'umounted %s' % cmd.mountPath))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def checkbits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckIsBitsExistingResponse()
rsp.existing = os.path.exists(cmd.path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def createempty(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname)
if cmd.backingFile:
linux.qcow2_create_with_backing_file_and_cmd(
cmd.backingFile, cmd.installPath, cmd)
else:
linux.qcow2_create_with_cmd(cmd.installPath, cmd.size, cmd)
logger.debug(
'successfully create empty volume[uuid:%s, size:%s] at %s' %
(cmd.volumeUuid, cmd.size, cmd.installPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.uuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def createvolume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
if not os.path.exists(cmd.templatePathInCache):
rsp.error = "unable to find image in cache"
rsp.success = False
return jsonobject.dumps(rsp)
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
linux.qcow2_clone_with_cmd(cmd.templatePathInCache, cmd.installPath,
cmd)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.uuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def deletebits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
self.delNasBits(cmd.folder, cmd.path)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.uuid)
return jsonobject.dumps(rsp)
def delNasBits(self, folder, path):
if folder:
shell.call('rm -rf %s' % path)
else:
kvmagent.deleteImage(path)
# pdir = os.path.dirname(path)
# if os.path.exists(pdir) or not os.listdir(pdir):
# linux.umount(pdir, False)
@kvmagent.replyerror
def revertvolume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotRsp()
install_path = cmd.snapshotInstallPath
new_volume_path = os.path.join(os.path.dirname(install_path),
'{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone_with_cmd(install_path, new_volume_path, cmd)
size = linux.qcow2_virtualsize(new_volume_path)
rsp.newVolumeInstallPath = new_volume_path
rsp.size = size
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def reinit(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ReInitImageRsp()
install_path = cmd.imagePath
new_volume_path = os.path.join(os.path.dirname(cmd.volumePath),
'{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone_with_cmd(install_path, new_volume_path, cmd)
rsp.newVolumeInstallPath = new_volume_path
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.uuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def uploadtoimagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(cmd, req)
@kvmagent.replyerror
def downloadfromimagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.imagestore_client.download_from_imagestore(
cmd.cacheDir, cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath)
rsp = AliyunNasResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.uuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def resize(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
install_path = cmd.installPath
rsp = ResizeVolumeRsp()
shell.call("qemu-img resize %s %s" % (install_path, cmd.size))
ret = linux.qcow2_virtualsize(install_path)
rsp.size = ret
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
@kvmagent.replyerror
def createtemplate(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
linux.create_template(cmd.volumePath, cmd.installPath)
logger.debug('successfully created template[%s] from volume[%s]' %
(cmd.installPath, cmd.volumePath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.uuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def mergesnapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotRsp()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
linux.create_template(cmd.snapshotInstallPath,
cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
cmd.workspaceInstallPath)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.uuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def offlinemerge(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
if not cmd.fullRebase:
linux.qcow2_rebase(cmd.srcPath, cmd.destPath)
else:
tmp = os.path.join(os.path.dirname(cmd.destPath),
'%s.qcow2' % uuidhelper.uuid())
linux.create_template(cmd.destPath, tmp)
shell.call("mv %s %s" % (tmp, cmd.destPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.uuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def getcapacity(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
def findMountUrl(self, mount):
if ' on ' not in mount:
raise Exception("' on ' not in mount: %s" % mount)
return mount.split(' on ')[0].strip()
def findMountPath(self, mount):
if ' on ' not in mount:
raise Exception("' on ' not in mount: %s" % mount)
tmp = mount.split(' on ')[1]
return tmp.split(' ')[0].strip()
def findMountInfo(self, mount):
if ' on ' not in mount:
raise Exception("' on ' not in mount: %s" % mount)
tmp = mount.split(' on ')[1]
if ' (' in tmp:
return '(' + tmp.split(' (')[1].strip()
else:
return None
@kvmagent.replyerror
def checkmountpath(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckMountPathRsp()
mounts = naslinux.getMountInfo(cmd.psUrl)
for mount in mounts:
data = MountData()
data.mountUrl = self.findMountUrl(mount)
data.mountPath = self.findMountPath(mount)
try:
info = self.findMountInfo(mount)
naslinux.checkMountStatus(data.mountUrl, data.mountPath, info)
data.status = "Normal"
except naslinux.InvalidMountDomainException as e:
data.status = "RemoteFault"
data.info = str(e)
except naslinux.InvalidMountPathException as e:
data.status = "LocalFault"
data.info = str(e)
rsp.datas.append(data)
return jsonobject.dumps(rsp)
class NfsPrimaryStoragePlugin(kvmagent.KvmAgent):
'''
classdocs
'''
MOUNT_PATH = '/nfsprimarystorage/mount'
UNMOUNT_PATH = '/nfsprimarystorage/unmount'
CREATE_VOLUME_FROM_TEMPLATE_PATH = "/nfsprimarystorage/sftp/createvolumefromtemplate"
CREATE_EMPTY_VOLUME_PATH = "/nfsprimarystorage/createemptyvolume"
GET_CAPACITY_PATH = "/nfsprimarystorage/getcapacity"
CREATE_TEMPLATE_FROM_VOLUME_PATH = "/nfsprimarystorage/sftp/createtemplatefromvolume"
REVERT_VOLUME_FROM_SNAPSHOT_PATH = "/nfsprimarystorage/revertvolumefromsnapshot"
DELETE_PATH = "/nfsprimarystorage/delete"
CHECK_BITS_PATH = "/nfsprimarystorage/checkbits"
UPLOAD_TO_SFTP_PATH = "/nfsprimarystorage/uploadtosftpbackupstorage"
DOWNLOAD_FROM_SFTP_PATH = "/nfsprimarystorage/downloadfromsftpbackupstorage"
UPLOAD_TO_IMAGESTORE_PATH = "/nfsprimarystorage/imagestore/upload"
COMMIT_TO_IMAGESTORE_PATH = "/nfsprimarystorage/imagestore/commit"
DOWNLOAD_FROM_IMAGESTORE_PATH = "/nfsprimarystorage/imagestore/download"
MERGE_SNAPSHOT_PATH = "/nfsprimarystorage/mergesnapshot"
REBASE_MERGE_SNAPSHOT_PATH = "/nfsprimarystorage/rebaseandmergesnapshot"
MOVE_BITS_PATH = "/nfsprimarystorage/movebits"
OFFLINE_SNAPSHOT_MERGE = "/nfsprimarystorage/offlinesnapshotmerge"
REMOUNT_PATH = "/nfsprimarystorage/remount"
GET_VOLUME_SIZE_PATH = "/nfsprimarystorage/getvolumesize"
PING_PATH = "/nfsprimarystorage/ping"
GET_VOLUME_BASE_IMAGE_PATH = "/nfsprimarystorage/getvolumebaseimage"
UPDATE_MOUNT_POINT_PATH = "/nfsprimarystorage/updatemountpoint"
ERR_UNABLE_TO_FIND_IMAGE_IN_CACHE = "UNABLE_TO_FIND_IMAGE_IN_CACHE"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_sync_uri(self.MOUNT_PATH, self.mount)
http_server.register_sync_uri(self.UNMOUNT_PATH, self.umount)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_TEMPLATE_PATH,
self.create_root_volume_from_template)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH,
self.create_empty_volume)
http_server.register_async_uri(self.DOWNLOAD_FROM_SFTP_PATH,
self.download_from_sftp)
http_server.register_async_uri(self.GET_CAPACITY_PATH,
self.get_capacity)
http_server.register_async_uri(self.DELETE_PATH, self.delete)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH,
self.create_template_from_root_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH,
self.revert_volume_from_snapshot)
http_server.register_async_uri(self.UPLOAD_TO_SFTP_PATH,
self.upload_to_sftp)
http_server.register_async_uri(self.UPLOAD_TO_IMAGESTORE_PATH,
self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_TO_IMAGESTORE_PATH,
self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_FROM_IMAGESTORE_PATH,
self.download_from_imagestore)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH,
self.merge_snapshot)
http_server.register_async_uri(self.REBASE_MERGE_SNAPSHOT_PATH,
self.rebase_and_merge_snapshot)
http_server.register_async_uri(self.MOVE_BITS_PATH, self.move_bits)
http_server.register_async_uri(self.OFFLINE_SNAPSHOT_MERGE,
self.merge_snapshot_to_volume)
http_server.register_async_uri(self.REMOUNT_PATH, self.remount)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH,
self.get_volume_size)
http_server.register_async_uri(self.PING_PATH, self.ping)
http_server.register_async_uri(self.GET_VOLUME_BASE_IMAGE_PATH,
self.get_volume_base_image_path)
http_server.register_async_uri(self.UPDATE_MOUNT_POINT_PATH,
self.update_mount_point)
self.mount_path = {}
self.image_cache = None
self.imagestore_client = ImageStoreClient()
def stop(self):
pass
def _get_disk_capacity(self, uuid):
path = self.mount_path.get(uuid)
if not path:
raise Exception(
'cannot find mount path of primary storage[uuid: %s]' % uuid)
return linux.get_disk_capacity_by_df(path)
def _json_meta_file_name(self, path):
return path + '.json'
def _set_capacity_to_response(self, uuid, rsp):
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
uuid)
@kvmagent.replyerror
def update_mount_point(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = UpdateMountPointResponse()
linux.is_valid_nfs_url(cmd.newMountPoint)
if not linux.is_mounted(cmd.mountPath, cmd.newMountPoint):
# umount old one
if linux.is_mounted(cmd.mountPath, cmd.oldMountPoint):
linux.umount(cmd.mountPath)
# mount new
linux.mount(cmd.newMountPoint, cmd.mountPath, cmd.options)
self.mount_path[cmd.uuid] = cmd.mountPath
logger.debug(
'updated the mount path[%s] mounting point from %s to %s' %
(cmd.mountPath, cmd.oldMountPoint, cmd.newMountPoint))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_volume_base_image_path(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeBaseImagePathRsp()
rsp.path = linux.get_qcow2_base_image_path_recusively(cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def ping(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
mount_path = self.mount_path[cmd.uuid]
if not mount_path or not os.path.isdir(mount_path):
raise Exception(
'the mount path[%s] of the nfs primary storage[uuid:%s] is not existing'
% (mount_path, cmd.uuid))
test_file = os.path.join(mount_path,
'%s-ping-test-file' % uuidhelper.uuid())
touch = shell.ShellCmd('timeout 60 touch %s' % test_file)
touch(False)
if touch.return_code == 124:
raise Exception(
'unable to access the mount path[%s] of the nfs primary storage[uuid:%s] in 60s, timeout'
% (mount_path, cmd.uuid))
elif touch.return_code != 0:
touch.raise_error()
shell.call('rm -f %s' % test_file)
return jsonobject.dumps(NfsResponse())
@kvmagent.replyerror
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot_to_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = OfflineMergeSnapshotRsp()
if not cmd.fullRebase:
linux.qcow2_rebase(cmd.srcPath, cmd.destPath)
else:
tmp = os.path.join(os.path.dirname(cmd.destPath),
'%s.qcow2' % uuidhelper.uuid())
linux.qcow2_create_template(cmd.destPath, tmp)
shell.call("mv %s %s" % (tmp, cmd.destPath))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def move_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MoveBitsRsp()
if not os.path.exists(cmd.srcPath):
rsp.error = "%s is not existing" % cmd.srcPath
rsp.success = False
else:
dirname = os.path.dirname(cmd.destPath)
if not os.path.exists(dirname):
os.makedirs(dirname)
shell.call("mv %s %s" % (cmd.srcPath, cmd.destPath))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def rebase_and_merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
snapshots = cmd.snapshotInstallPaths
count = len(snapshots)
for i in range(count):
if i + 1 < count:
target = snapshots[i]
backing_file = snapshots[i + 1]
linux.qcow2_rebase_no_check(backing_file, target)
latest = snapshots[0]
rsp = RebaseAndMergeSnapshotsResponse()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
try:
linux.qcow2_create_template(latest, cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = cmd.workspaceInstallPath
self._set_capacity_to_response(cmd.uuid, rsp)
except linux.LinuxError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotResponse()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
try:
linux.qcow2_create_template(cmd.snapshotInstallPath,
cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
cmd.workspaceInstallPath)
self._set_capacity_to_response(cmd.uuid, rsp)
except linux.LinuxError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CopyToSftpBackupStorageResponse()
def upload():
if not os.path.exists(cmd.primaryStorageInstallPath):
raise kvmagent.KvmError('cannot find %s' %
cmd.primaryStorageInstallPath)
linux.scp_upload(cmd.backupStorageHostName,
cmd.backupStorageSshKey,
cmd.primaryStorageInstallPath,
cmd.backupStorageInstallPath,
cmd.backupStorageUserName,
cmd.backupStorageSshPort)
try:
upload()
except kvmagent.KvmError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(
cmd.hostname, cmd.primaryStorageInstallPath)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(
cmd.primaryStorageInstallPath)
@kvmagent.replyerror
def download_from_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.imagestore_client.download_from_imagestore(
self.mount_path.get(cmd.uuid), cmd.hostname,
cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath)
rsp = kvmagent.AgentResponse()
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def revert_volume_from_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotResponse()
install_path = cmd.snapshotInstallPath
new_volume_path = os.path.join(os.path.dirname(install_path),
'{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone(install_path, new_volume_path)
rsp.newVolumeInstallPath = new_volume_path
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckIsBitsExistingRsp()
rsp.existing = os.path.exists(cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def delete(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = DeleteResponse()
if cmd.isFolder:
shell.call('rm -rf %s' % cmd.installPath)
else:
shell.call('rm -f %s' % cmd.installPath)
pdir = os.path.dirname(cmd.installPath)
linux.rmdir_if_empty(pdir)
logger.debug('successfully delete %s' % cmd.installPath)
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def remount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MountResponse()
linux.is_valid_nfs_url(cmd.url)
if not linux.is_mounted(cmd.mountPath, cmd.url):
linux.mount(cmd.url, cmd.mountPath, cmd.options)
shell.call('mount -o remount %s' % cmd.mountPath)
self.mount_path[cmd.uuid] = cmd.mountPath
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def mount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MountResponse()
linux.is_valid_nfs_url(cmd.url)
if not linux.is_mounted(cmd.mountPath, cmd.url):
linux.mount(cmd.url, cmd.mountPath, cmd.options)
self.mount_path[cmd.uuid] = cmd.mountPath
logger.debug(
http.path_msg(self.MOUNT_PATH,
'mounted %s on %s' % (cmd.url, cmd.mountPath)))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def umount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = UnmountResponse()
if linux.is_mounted(path=cmd.mountPath):
ret = linux.umount(cmd.mountPath)
if not ret:
logger.warn(
http.path_msg(
self.UNMOUNT_PATH, 'unmount %s from %s failed' %
(cmd.mountPath, cmd.url)))
logger.debug(
http.path_msg(self.UNMOUNT_PATH,
'umounted %s from %s' % (cmd.mountPath, cmd.url)))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_capacity(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetCapacityResponse()
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_empty_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CreateEmptyVolumeResponse()
try:
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname)
linux.qcow2_create(cmd.installUrl, cmd.size)
except Exception as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = 'unable to create empty volume[uuid:%s, name:%s], %s' % (
cmd.uuid, cmd.name, str(e))
rsp.success = False
return jsonobject.dumps(rsp)
meta = VolumeMeta()
meta.account_uuid = cmd.accountUuid
meta.hypervisor_type = cmd.hypervisorType
meta.name = cmd.name
meta.uuid = cmd.volumeUuid
meta.size = cmd.size
meta_path = self._json_meta_file_name(cmd.installUrl)
with open(meta_path, 'w') as fd:
fd.write(jsonobject.dumps(meta, pretty=True))
self._set_capacity_to_response(cmd.uuid, rsp)
logger.debug(
'successfully create empty volume[uuid:%s, name:%s, size:%s] at %s'
% (cmd.uuid, cmd.name, cmd.size, cmd.installUrl))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_template_from_root_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CreateTemplateFromRootVolumeRsp()
try:
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
linux.qcow2_create_template(cmd.rootVolumePath, cmd.installPath)
except linux.LinuxError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = 'unable to create image to root@%s:%s from root volume[%s], %s' % (
cmd.sftpBackupStorageHostName, cmd.installPath,
cmd.rootVolumePath, str(e))
rsp.success = False
self._set_capacity_to_response(cmd.uuid, rsp)
logger.debug('successfully created template[%s] from root volume[%s]' %
(cmd.installPath, cmd.rootVolumePath))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = DownloadBitsFromSftpBackupStorageResponse()
try:
linux.scp_download(cmd.hostname, cmd.sshKey,
cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath, cmd.username,
cmd.sshPort)
logger.debug('successfully download %s/%s to %s' %
(cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath))
self._set_capacity_to_response(cmd.uuid, rsp)
except Exception as e:
content = traceback.format_exc()
logger.warn(content)
err = "unable to download %s/%s, because %s" % (
cmd.hostname, cmd.backupStorageInstallPath, str(e))
rsp.error = err
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_root_volume_from_template(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CreateRootVolumeFromTemplateResponse()
if not os.path.exists(cmd.templatePathInCache):
rsp.error = self.ERR_UNABLE_TO_FIND_IMAGE_IN_CACHE
rsp.success = False
return jsonobject.dumps(rsp)
try:
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
linux.qcow2_clone(cmd.templatePathInCache, cmd.installUrl)
logger.debug(
'successfully create root volume[%s] from template in cache[%s]'
% (cmd.installUrl, cmd.templatePathInCache))
meta = VolumeMeta()
meta.account_uuid = cmd.accountUuid
meta.hypervisor_type = cmd.hypervisorType
meta.name = cmd.name
meta.uuid = cmd.volumeUuid
meta.size = os.path.getsize(cmd.templatePathInCache)
meta_path = self._json_meta_file_name(cmd.installUrl)
with open(meta_path, 'w') as fd:
fd.write(jsonobject.dumps(meta, pretty=True))
self._set_capacity_to_response(cmd.uuid, rsp)
logger.debug(
'successfully create root volume[%s] from template in cache[%s]'
% (cmd.installUrl, cmd.templatePathInCache))
except Exception as e:
content = traceback.format_exc()
logger.warn(content)
err = 'unable to clone qcow2 template[%s] to %s' % (
cmd.templatePathInCache, cmd.installUrl)
rsp.error = err
rsp.success = False
return jsonobject.dumps(rsp)
class SharedBlockPlugin(kvmagent.KvmAgent):
CONNECT_PATH = "/sharedblock/connect"
DISCONNECT_PATH = "/sharedblock/disconnect"
CREATE_VOLUME_FROM_CACHE_PATH = "/sharedblock/createrootvolume"
DELETE_BITS_PATH = "/sharedblock/bits/delete"
CREATE_TEMPLATE_FROM_VOLUME_PATH = "/sharedblock/createtemplatefromvolume"
UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH = "/sharedblock/sftp/upload"
DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH = "/sharedblock/sftp/download"
UPLOAD_BITS_TO_IMAGESTORE_PATH = "/sharedblock/imagestore/upload"
COMMIT_BITS_TO_IMAGESTORE_PATH = "/sharedblock/imagestore/commit"
DOWNLOAD_BITS_FROM_IMAGESTORE_PATH = "/sharedblock/imagestore/download"
REVERT_VOLUME_FROM_SNAPSHOT_PATH = "/sharedblock/volume/revertfromsnapshot"
MERGE_SNAPSHOT_PATH = "/sharedblock/snapshot/merge"
OFFLINE_MERGE_SNAPSHOT_PATH = "/sharedblock/snapshot/offlinemerge"
CREATE_EMPTY_VOLUME_PATH = "/sharedblock/volume/createempty"
CHECK_BITS_PATH = "/sharedblock/bits/check"
RESIZE_VOLUME_PATH = "/sharedblock/volume/resize"
CONVERT_IMAGE_TO_VOLUME = "/sharedblock/image/tovolume"
CHANGE_VOLUME_ACTIVE_PATH = "/sharedblock/volume/active"
GET_VOLUME_SIZE_PATH = "/sharedblock/volume/getsize"
CHECK_DISKS_PATH = "/sharedblock/disks/check"
ADD_SHARED_BLOCK = "/sharedblock/disks/add"
MIGRATE_DATA_PATH = "/sharedblock/volume/migrate"
GET_BLOCK_DEVICES_PATH = "/sharedblock/blockdevices"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.CONNECT_PATH, self.connect)
http_server.register_async_uri(self.DISCONNECT_PATH, self.disconnect)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH,
self.create_root_volume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete_bits)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH,
self.create_template_from_volume)
http_server.register_async_uri(
self.UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH, self.upload_to_sftp)
http_server.register_async_uri(
self.DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH,
self.download_from_sftp)
http_server.register_async_uri(self.UPLOAD_BITS_TO_IMAGESTORE_PATH,
self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_BITS_TO_IMAGESTORE_PATH,
self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_IMAGESTORE_PATH,
self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH,
self.revert_volume_from_snapshot)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH,
self.merge_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_SNAPSHOT_PATH,
self.offline_merge_snapshots)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH,
self.create_empty_volume)
http_server.register_async_uri(self.CONVERT_IMAGE_TO_VOLUME,
self.convert_image_to_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH,
self.resize_volume)
http_server.register_async_uri(self.CHANGE_VOLUME_ACTIVE_PATH,
self.active_lv)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH,
self.get_volume_size)
http_server.register_async_uri(self.CHECK_DISKS_PATH, self.check_disks)
http_server.register_async_uri(self.ADD_SHARED_BLOCK, self.add_disk)
http_server.register_async_uri(self.MIGRATE_DATA_PATH,
self.migrate_volumes)
http_server.register_async_uri(self.GET_BLOCK_DEVICES_PATH,
self.get_block_devices)
self.imagestore_client = ImageStoreClient()
def stop(self):
pass
@kvmagent.replyerror
def check_disks(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
for diskUuid in cmd.sharedBlockUuids:
disk = CheckDisk(diskUuid)
path = disk.get_path()
if cmd.rescan:
disk.rescan(path.split("/")[-1])
if cmd.vgUuid is not None and lvm.vg_exists(cmd.vgUuid):
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(
cmd.vgUuid, False)
return jsonobject.dumps(rsp)
@staticmethod
def create_vg_if_not_found(vgUuid, diskPaths, hostUuid, forceWipe=False):
@linux.retry(times=5, sleep_time=random.uniform(0.1, 3))
def find_vg(vgUuid):
cmd = shell.ShellCmd("vgs %s -otags | grep %s" %
(vgUuid, INIT_TAG))
cmd(is_exception=False)
if cmd.return_code != 0:
raise RetryException("can not find vg %s with tag %s" %
(vgUuid, INIT_TAG))
return True
try:
find_vg(vgUuid)
except RetryException as e:
if forceWipe is True:
lvm.wipe_fs(diskPaths)
cmd = shell.ShellCmd(
"vgcreate -qq --shared --addtag '%s::%s::%s' --metadatasize %s %s %s"
% (INIT_TAG, hostUuid, time.time(), DEFAULT_VG_METADATA_SIZE,
vgUuid, " ".join(diskPaths)))
cmd(is_exception=False)
logger.debug("created vg %s, ret: %s, stdout: %s, stderr: %s" %
(vgUuid, cmd.return_code, cmd.stdout, cmd.stderr))
if cmd.return_code == 0 and find_vg(vgUuid) is True:
return True
if find_vg(vgUuid) is False:
raise Exception(
"can not find vg %s with disks: %s and create vg return: %s %s %s "
% (vgUuid, diskPaths, cmd.return_code, cmd.stdout,
cmd.stderr))
except Exception as e:
raise e
return False
@kvmagent.replyerror
@lock.file_lock(LOCK_FILE)
def connect(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ConnectRsp()
diskPaths = set()
def config_lvm(host_id):
lvm.backup_lvm_config()
lvm.reset_lvm_conf_default()
lvm.config_lvm_by_sed("use_lvmlockd", "use_lvmlockd=1",
["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("use_lvmetad", "use_lvmetad=0",
["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("host_id", "host_id=%s" % host_id,
["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed(
"sanlock_lv_extend",
"sanlock_lv_extend=%s" % DEFAULT_SANLOCK_LV_SIZE,
["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("lvmlockd_lock_retries",
"lvmlockd_lock_retries=6",
["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("issue_discards", "issue_discards=1",
["lvm.conf", "lvmlocal.conf"])
lvm.config_sanlock_by_sed("sh_retries", "sh_retries=20")
lvm.config_sanlock_by_sed("logfile_priority", "logfile_priority=7")
lvm.config_sanlock_by_sed("renewal_read_extend_sec",
"renewal_read_extend_sec=24")
lvm.config_sanlock_by_sed("debug_renew", "debug_renew=1")
lvm.config_sanlock_by_sed("use_watchdog", "use_watchdog=0")
config_lvm(cmd.hostId)
for diskUuid in cmd.sharedBlockUuids:
disk = CheckDisk(diskUuid)
diskPaths.add(disk.get_path())
lvm.start_lvmlockd()
lvm.check_gl_lock()
rsp.isFirst = self.create_vg_if_not_found(cmd.vgUuid, diskPaths,
cmd.hostUuid, cmd.forceWipe)
lvm.start_vg_lock(cmd.vgUuid)
lvm.clean_vg_exists_host_tags(cmd.vgUuid, cmd.hostUuid, HEARTBEAT_TAG)
lvm.add_vg_tag(
cmd.vgUuid,
"%s::%s::%s" % (HEARTBEAT_TAG, cmd.hostUuid, time.time()))
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
rsp.hostId = lvm.get_running_host_id(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@lock.file_lock(LOCK_FILE)
def disconnect(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
@linux.retry(times=3, sleep_time=random.uniform(0.1, 3))
def find_vg(vgUuid):
cmd = shell.ShellCmd("vgs %s -otags | grep %s" %
(vgUuid, INIT_TAG))
cmd(is_exception=False)
if cmd.return_code == 0:
return True
logger.debug("can not find vg %s with tag %s" % (vgUuid, INIT_TAG))
cmd = shell.ShellCmd("vgs %s" % vgUuid)
cmd(is_exception=False)
if cmd.return_code == 0:
logger.warn("found vg %s without tag %s" % (vgUuid, INIT_TAG))
return True
raise RetryException("can not find vg %s with or without tag %s" %
(vgUuid, INIT_TAG))
try:
find_vg(cmd.vgUuid)
except RetryException:
logger.debug("can not find vg %s; return success" % cmd.vgUuid)
return jsonobject.dumps(rsp)
except Exception as e:
raise e
@linux.retry(times=3, sleep_time=random.uniform(0.1, 3))
def deactive_lvs_on_vg(vgUuid):
active_lvs = lvm.list_local_active_lvs(vgUuid)
if len(active_lvs) == 0:
return
logger.warn("active lvs %s will be deactivate" % active_lvs)
lvm.deactive_lv(vgUuid)
active_lvs = lvm.list_local_active_lvs(vgUuid)
if len(active_lvs) != 0:
raise RetryException(
"lvs [%s] still active, retry deactive again" % active_lvs)
deactive_lvs_on_vg(cmd.vgUuid)
lvm.clean_vg_exists_host_tags(cmd.vgUuid, cmd.hostUuid, HEARTBEAT_TAG)
lvm.stop_vg_lock(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@lock.file_lock(LOCK_FILE)
def add_disk(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
disk = CheckDisk(cmd.diskUuid)
command = shell.ShellCmd("vgs %s -otags | grep %s" %
(cmd.vgUuid, INIT_TAG))
command(is_exception=False)
if command.return_code != 0:
self.create_vg_if_not_found(cmd.vgUuid, [disk.get_path()],
cmd.hostUuid, cmd.forceWipe)
else:
lvm.check_gl_lock()
if cmd.forceWipe is True:
lvm.wipe_fs([disk.get_path()])
lvm.add_pv(cmd.vgUuid, disk.get_path(), DEFAULT_VG_METADATA_SIZE)
rsp = AgentRsp
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def resize_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
install_abs_path = translate_absolute_path_from_install_path(
cmd.installPath)
with lvm.RecursiveOperateLv(install_abs_path, shared=False):
lvm.resize_lv(install_abs_path, cmd.size)
if not cmd.live:
shell.call("qemu-img resize %s %s" %
(install_abs_path, cmd.size))
ret = linux.qcow2_virtualsize(install_abs_path)
rsp = ResizeVolumeRsp()
rsp.size = ret
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@lock.file_lock(LOCK_FILE)
def create_root_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
template_abs_path_cache = translate_absolute_path_from_install_path(
cmd.templatePathInCache)
install_abs_path = translate_absolute_path_from_install_path(
cmd.installPath)
qcow2_options = self.calc_qcow2_option(self, cmd.qcow2Options, True)
with lvm.RecursiveOperateLv(template_abs_path_cache,
shared=True,
skip_deactivate_tag=IMAGE_TAG):
virtual_size = linux.qcow2_virtualsize(template_abs_path_cache)
if not lvm.lv_exists(install_abs_path):
lvm.create_lv_from_absolute_path(
install_abs_path, virtual_size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(install_abs_path,
shared=False,
delete_when_exception=True):
linux.qcow2_clone_with_option(template_abs_path_cache,
install_abs_path, qcow2_options)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def delete_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if cmd.folder:
raise Exception("not support this operation")
install_abs_path = translate_absolute_path_from_install_path(cmd.path)
if lvm.has_lv_tag(install_abs_path, IMAGE_TAG):
logger.info('deleting lv image: ' + install_abs_path)
lvm.delete_image(install_abs_path, IMAGE_TAG)
else:
logger.info('deleting lv volume: ' + install_abs_path)
lvm.delete_lv(install_abs_path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_template_from_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
volume_abs_path = translate_absolute_path_from_install_path(
cmd.volumePath)
install_abs_path = translate_absolute_path_from_install_path(
cmd.installPath)
if cmd.sharedVolume:
lvm.do_active_lv(volume_abs_path, lvm.LvmlockdLockType.SHARE, True)
with lvm.RecursiveOperateLv(volume_abs_path,
shared=cmd.sharedVolume,
skip_deactivate_tag=IMAGE_TAG):
virtual_size = linux.qcow2_virtualsize(volume_abs_path)
if not lvm.lv_exists(install_abs_path):
lvm.create_lv_from_absolute_path(
install_abs_path, virtual_size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(install_abs_path,
shared=False,
delete_when_exception=True):
linux.create_template(volume_abs_path, install_abs_path)
logger.debug(
'successfully created template[%s] from volume[%s]' %
(cmd.installPath, cmd.volumePath))
if cmd.compareQcow2 is True:
logger.debug("comparing qcow2 between %s and %s")
bash.bash_errorout("time qemu-img compare %s %s" %
(volume_abs_path, install_abs_path))
logger.debug("confirmed qcow2 %s and %s are identical" %
(volume_abs_path, install_abs_path))
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
install_abs_path = translate_absolute_path_from_install_path(
cmd.primaryStorageInstallPath)
def upload():
if not os.path.exists(cmd.primaryStorageInstallPath):
raise kvmagent.KvmError('cannot find %s' %
cmd.primaryStorageInstallPath)
linux.scp_upload(cmd.hostname, cmd.sshKey,
cmd.primaryStorageInstallPath,
cmd.backupStorageInstallPath, cmd.username,
cmd.sshPort)
with lvm.OperateLv(install_abs_path, shared=True):
upload()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
install_abs_path = translate_absolute_path_from_install_path(
cmd.primaryStorageInstallPath)
size = linux.sftp_get(cmd.hostname, cmd.sshKey,
cmd.backupStorageInstallPath, install_abs_path,
cmd.username, cmd.sshPort, True)
if not lvm.lv_exists(install_abs_path):
lvm.create_lv_from_absolute_path(
install_abs_path, size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(install_abs_path,
shared=False,
delete_when_exception=True):
linux.scp_download(cmd.hostname, cmd.sshKey,
cmd.backupStorageInstallPath, install_abs_path,
cmd.username, cmd.sshPort)
logger.debug('successfully download %s/%s to %s' %
(cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath))
self.do_active_lv(cmd.primaryStorageInstallPath, cmd.lockType, False)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(cmd, req)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
@kvmagent.replyerror
def download_from_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.imagestore_client.download_from_imagestore(
cmd.mountPoint, cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath)
self.do_active_lv(cmd.primaryStorageInstallPath, cmd.lockType, True)
rsp = AgentRsp()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def revert_volume_from_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotRsp()
snapshot_abs_path = translate_absolute_path_from_install_path(
cmd.snapshotInstallPath)
qcow2_options = self.calc_qcow2_option(self, cmd.qcow2Options, True)
with lvm.RecursiveOperateLv(snapshot_abs_path, shared=True):
size = linux.qcow2_virtualsize(snapshot_abs_path)
new_volume_path = "/dev/%s/%s" % (cmd.vgUuid, uuidhelper.uuid())
lvm.create_lv_from_absolute_path(
new_volume_path, size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(new_volume_path,
shared=False,
delete_when_exception=True):
linux.qcow2_clone_with_option(snapshot_abs_path,
new_volume_path, qcow2_options)
size = linux.qcow2_virtualsize(new_volume_path)
rsp.newVolumeInstallPath = new_volume_path
rsp.size = size
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotRsp()
snapshot_abs_path = translate_absolute_path_from_install_path(
cmd.snapshotInstallPath)
workspace_abs_path = translate_absolute_path_from_install_path(
cmd.workspaceInstallPath)
with lvm.RecursiveOperateLv(snapshot_abs_path, shared=True):
virtual_size = linux.qcow2_virtualsize(snapshot_abs_path)
if not lvm.lv_exists(workspace_abs_path):
lvm.create_lv_from_absolute_path(
workspace_abs_path, virtual_size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(workspace_abs_path,
shared=False,
delete_when_exception=True):
linux.create_template(snapshot_abs_path, workspace_abs_path)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
workspace_abs_path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
rsp.actualSize = rsp.size
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def offline_merge_snapshots(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = OfflineMergeSnapshotRsp()
src_abs_path = translate_absolute_path_from_install_path(cmd.srcPath)
dst_abs_path = translate_absolute_path_from_install_path(cmd.destPath)
with lvm.RecursiveOperateLv(src_abs_path, shared=True):
virtual_size = linux.qcow2_virtualsize(src_abs_path)
if not lvm.lv_exists(dst_abs_path):
lvm.create_lv_from_absolute_path(
dst_abs_path, virtual_size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.RecursiveOperateLv(dst_abs_path, shared=False):
if not cmd.fullRebase:
linux.qcow2_rebase(src_abs_path, dst_abs_path)
else:
tmp_lv = 'tmp_%s' % uuidhelper.uuid()
tmp_abs_path = os.path.join(os.path.dirname(dst_abs_path),
tmp_lv)
logger.debug("creating temp lv %s" % tmp_abs_path)
lvm.create_lv_from_absolute_path(
tmp_abs_path, virtual_size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(tmp_abs_path,
shared=False,
delete_when_exception=True):
linux.create_template(dst_abs_path, tmp_abs_path)
lvm.lv_rename(tmp_abs_path,
dst_abs_path,
overwrite=True)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@lock.file_lock(LOCK_FILE)
def create_empty_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
install_abs_path = translate_absolute_path_from_install_path(
cmd.installPath)
if cmd.backingFile:
qcow2_options = self.calc_qcow2_option(self, cmd.qcow2Options,
True)
backing_abs_path = translate_absolute_path_from_install_path(
cmd.backingFile)
with lvm.RecursiveOperateLv(backing_abs_path, shared=True):
virtual_size = linux.qcow2_virtualsize(backing_abs_path)
if not lvm.lv_exists(install_abs_path):
lvm.create_lv_from_absolute_path(
install_abs_path, virtual_size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(install_abs_path,
shared=False,
delete_when_exception=True):
linux.qcow2_create_with_backing_file_and_option(
backing_abs_path, install_abs_path, qcow2_options)
elif not lvm.lv_exists(install_abs_path):
qcow2_options = self.calc_qcow2_option(self, cmd.qcow2Options,
False)
lvm.create_lv_from_absolute_path(
install_abs_path, cmd.size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(install_abs_path,
shared=False,
delete_when_exception=True):
linux.qcow2_create_with_option(install_abs_path, cmd.size,
qcow2_options)
linux.qcow2_fill(0, 1048576, install_abs_path)
logger.debug(
'successfully create empty volume[uuid:%s, size:%s] at %s' %
(cmd.volumeUuid, cmd.size, cmd.installPath))
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def convert_image_to_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
install_abs_path = translate_absolute_path_from_install_path(
cmd.primaryStorageInstallPath)
with lvm.OperateLv(install_abs_path, shared=False):
lvm.clean_lv_tag(install_abs_path, IMAGE_TAG)
lvm.add_lv_tag(
install_abs_path,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckBitsRsp()
install_abs_path = translate_absolute_path_from_install_path(cmd.path)
rsp.existing = lvm.lv_exists(install_abs_path)
return jsonobject.dumps(rsp)
def do_active_lv(self, installPath, lockType, recursive):
def handle_lv(lockType, fpath):
if lockType > lvm.LvmlockdLockType.NULL:
lvm.active_lv(fpath, lockType == lvm.LvmlockdLockType.SHARE)
else:
lvm.deactive_lv(fpath)
install_abs_path = translate_absolute_path_from_install_path(
installPath)
handle_lv(lockType, install_abs_path)
if recursive is False or lockType is lvm.LvmlockdLockType.NULL:
return
while linux.qcow2_get_backing_file(install_abs_path) != "":
install_abs_path = linux.qcow2_get_backing_file(install_abs_path)
if lockType == lvm.LvmlockdLockType.NULL:
handle_lv(lvm.LvmlockdLockType.NULL, install_abs_path)
else:
# activate backing files only in shared mode
handle_lv(lvm.LvmlockdLockType.SHARE, install_abs_path)
@kvmagent.replyerror
def active_lv(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(
cmd.vgUuid, raise_exception=False)
self.do_active_lv(cmd.installPath, cmd.lockType, cmd.recursive)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
install_abs_path = translate_absolute_path_from_install_path(
cmd.installPath)
rsp.size = lvm.get_lv_size(install_abs_path)
rsp.actualSize = rsp.size
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@bash.in_bash
def migrate_volumes(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
for struct in cmd.migrateVolumeStructs:
target_abs_path = translate_absolute_path_from_install_path(
struct.targetInstallPath)
current_abs_path = translate_absolute_path_from_install_path(
struct.currentInstallPath)
with lvm.OperateLv(current_abs_path, shared=True):
virtual_size = lvm.get_lv_size(current_abs_path)
if lvm.lv_exists(target_abs_path):
target_ps_uuid = get_primary_storage_uuid_from_install_path(
struct.targetInstallPath)
raise Exception("found %s already exists on ps %s" %
(target_abs_path, target_ps_uuid))
lvm.create_lv_from_absolute_path(
target_abs_path, virtual_size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
lvm.active_lv(target_abs_path, lvm.LvmlockdLockType.SHARE)
try:
for struct in cmd.migrateVolumeStructs:
target_abs_path = translate_absolute_path_from_install_path(
struct.targetInstallPath)
current_abs_path = translate_absolute_path_from_install_path(
struct.currentInstallPath)
with lvm.OperateLv(current_abs_path, shared=True):
bash.bash_errorout("cp %s %s" %
(current_abs_path, target_abs_path))
for struct in cmd.migrateVolumeStructs:
target_abs_path = translate_absolute_path_from_install_path(
struct.targetInstallPath)
current_abs_path = translate_absolute_path_from_install_path(
struct.currentInstallPath)
with lvm.RecursiveOperateLv(current_abs_path, shared=True):
previous_ps_uuid = get_primary_storage_uuid_from_install_path(
struct.currentInstallPath)
target_ps_uuid = get_primary_storage_uuid_from_install_path(
struct.targetInstallPath)
current_backing_file = linux.qcow2_get_backing_file(
current_abs_path) # type: str
target_backing_file = current_backing_file.replace(
previous_ps_uuid, target_ps_uuid)
if current_backing_file is not None and current_backing_file != "":
lvm.do_active_lv(target_backing_file,
lvm.LvmlockdLockType.SHARE, False)
logger.debug("rebase %s to %s" %
(target_abs_path, target_backing_file))
linux.qcow2_rebase_no_check(target_backing_file,
target_abs_path)
if struct.compareQcow2:
bash.bash_errorout("time qemu-img compare %s %s" %
(current_abs_path, target_abs_path))
except Exception as e:
for struct in cmd.migrateVolumeStructs:
target_abs_path = translate_absolute_path_from_install_path(
struct.targetInstallPath)
if struct.currentInstallPath == struct.targetInstallPath:
logger.debug(
"current install path %s equals target %s, skip to delete"
%
(struct.currentInstallPath, struct.targetInstallPath))
else:
logger.debug("error happened, delete lv %s" %
target_abs_path)
lvm.delete_lv(target_abs_path, False)
raise e
finally:
for struct in cmd.migrateVolumeStructs:
target_abs_path = translate_absolute_path_from_install_path(
struct.targetInstallPath)
lvm.deactive_lv(target_abs_path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@staticmethod
def calc_qcow2_option(self, options, has_backing_file):
if options is None or options == "":
return " "
if has_backing_file:
return re.sub("-o preallocation=\w* ", " ", options)
return options
@kvmagent.replyerror
def get_block_devices(self, req):
rsp = GetBlockDevicesRsp()
rsp.blockDevices = lvm.get_block_devices()
return jsonobject.dumps(rsp)
class NfsPrimaryStoragePlugin(kvmagent.KvmAgent):
'''
classdocs
'''
MOUNT_PATH = '/nfsprimarystorage/mount'
UNMOUNT_PATH = '/nfsprimarystorage/unmount'
CREATE_VOLUME_FROM_TEMPLATE_PATH = "/nfsprimarystorage/sftp/createvolumefromtemplate"
CREATE_EMPTY_VOLUME_PATH = "/nfsprimarystorage/createemptyvolume"
CREATE_FOLDER_PATH = "/nfsprimarystorage/createfolder"
GET_CAPACITY_PATH = "/nfsprimarystorage/getcapacity"
CREATE_TEMPLATE_FROM_VOLUME_PATH = "/nfsprimarystorage/sftp/createtemplatefromvolume"
REVERT_VOLUME_FROM_SNAPSHOT_PATH = "/nfsprimarystorage/revertvolumefromsnapshot"
REINIT_IMAGE_PATH = "/nfsprimarystorage/reinitimage"
DELETE_PATH = "/nfsprimarystorage/delete"
CHECK_BITS_PATH = "/nfsprimarystorage/checkbits"
UPLOAD_TO_SFTP_PATH = "/nfsprimarystorage/uploadtosftpbackupstorage"
DOWNLOAD_FROM_SFTP_PATH = "/nfsprimarystorage/downloadfromsftpbackupstorage"
UPLOAD_TO_IMAGESTORE_PATH = "/nfsprimarystorage/imagestore/upload"
COMMIT_TO_IMAGESTORE_PATH = "/nfsprimarystorage/imagestore/commit"
DOWNLOAD_FROM_IMAGESTORE_PATH = "/nfsprimarystorage/imagestore/download"
MERGE_SNAPSHOT_PATH = "/nfsprimarystorage/mergesnapshot"
REBASE_MERGE_SNAPSHOT_PATH = "/nfsprimarystorage/rebaseandmergesnapshot"
MOVE_BITS_PATH = "/nfsprimarystorage/movebits"
OFFLINE_SNAPSHOT_MERGE = "/nfsprimarystorage/offlinesnapshotmerge"
REMOUNT_PATH = "/nfsprimarystorage/remount"
GET_VOLUME_SIZE_PATH = "/nfsprimarystorage/getvolumesize"
PING_PATH = "/nfsprimarystorage/ping"
GET_VOLUME_BASE_IMAGE_PATH = "/nfsprimarystorage/getvolumebaseimage"
UPDATE_MOUNT_POINT_PATH = "/nfsprimarystorage/updatemountpoint"
RESIZE_VOLUME_PATH = "/nfsprimarystorage/volume/resize"
NFS_TO_NFS_MIGRATE_BITS_PATH = "/nfsprimarystorage/migratebits"
NFS_REBASE_VOLUME_BACKING_FILE_PATH = "/nfsprimarystorage/rebasevolumebackingfile"
DOWNLOAD_BITS_FROM_KVM_HOST_PATH = "/nfsprimarystorage/kvmhost/download"
CANCEL_DOWNLOAD_BITS_FROM_KVM_HOST_PATH = "/nfsprimarystorage/kvmhost/download/cancel"
GET_DOWNLOAD_BITS_FROM_KVM_HOST_PROGRESS_PATH = "/nfsprimarystorage/kvmhost/download/progress"
ERR_UNABLE_TO_FIND_IMAGE_IN_CACHE = "unable to find image in cache"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_sync_uri(self.MOUNT_PATH, self.mount)
http_server.register_sync_uri(self.UNMOUNT_PATH, self.umount)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_TEMPLATE_PATH,
self.create_root_volume_from_template)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH,
self.create_empty_volume)
http_server.register_async_uri(self.CREATE_FOLDER_PATH,
self.create_folder)
http_server.register_async_uri(self.DOWNLOAD_FROM_SFTP_PATH,
self.download_from_sftp)
http_server.register_async_uri(self.GET_CAPACITY_PATH,
self.get_capacity)
http_server.register_async_uri(self.DELETE_PATH, self.delete)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH,
self.create_template_from_root_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH,
self.revert_volume_from_snapshot)
http_server.register_async_uri(self.REINIT_IMAGE_PATH,
self.reinit_image)
http_server.register_async_uri(self.UPLOAD_TO_SFTP_PATH,
self.upload_to_sftp)
http_server.register_async_uri(self.UPLOAD_TO_IMAGESTORE_PATH,
self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_TO_IMAGESTORE_PATH,
self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_FROM_IMAGESTORE_PATH,
self.download_from_imagestore)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH,
self.merge_snapshot)
http_server.register_async_uri(self.REBASE_MERGE_SNAPSHOT_PATH,
self.rebase_and_merge_snapshot)
http_server.register_async_uri(self.MOVE_BITS_PATH, self.move_bits)
http_server.register_async_uri(self.OFFLINE_SNAPSHOT_MERGE,
self.merge_snapshot_to_volume)
http_server.register_async_uri(self.REMOUNT_PATH, self.remount)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH,
self.get_volume_size)
http_server.register_async_uri(self.PING_PATH, self.ping)
http_server.register_async_uri(self.GET_VOLUME_BASE_IMAGE_PATH,
self.get_volume_base_image_path)
http_server.register_async_uri(self.UPDATE_MOUNT_POINT_PATH,
self.update_mount_point)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH,
self.resize_volume)
http_server.register_async_uri(self.NFS_TO_NFS_MIGRATE_BITS_PATH,
self.migrate_bits)
http_server.register_async_uri(
self.NFS_REBASE_VOLUME_BACKING_FILE_PATH,
self.rebase_volume_backing_file)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_KVM_HOST_PATH,
self.download_from_kvmhost)
http_server.register_async_uri(
self.CANCEL_DOWNLOAD_BITS_FROM_KVM_HOST_PATH,
self.cancel_download_from_kvmhost)
http_server.register_async_uri(
self.GET_DOWNLOAD_BITS_FROM_KVM_HOST_PROGRESS_PATH,
self.get_download_bits_from_kvmhost_progress)
self.mount_path = {}
self.image_cache = None
self.imagestore_client = ImageStoreClient()
def stop(self):
pass
@kvmagent.replyerror
def migrate_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = NfsToNfsMigrateBitsRsp()
mount_path = cmd.mountPath
dst_folder_path = cmd.dstFolderPath
temp_dir = None
fd, PFILE = tempfile.mkstemp()
os.close(fd)
f = open(PFILE, 'r')
try:
if not cmd.isMounted:
linux.is_valid_nfs_url(cmd.url)
temp_dir = tempfile.mkdtemp()
# dst folder is absolute path
mount_path = temp_dir + mount_path
dst_folder_path = temp_dir + dst_folder_path
if not linux.is_mounted(mount_path, cmd.url):
linux.mount(cmd.url, mount_path, cmd.options, "nfs4")
# begin migration, then check md5 sums
linux.mkdir(dst_folder_path)
t_shell = traceable_shell.get_shell(cmd)
rsync_excludes = ""
md5_excludes = ""
if cmd.filtPaths:
for filtPath in cmd.filtPaths:
# filtPath cannot start with '/', because it must be a relative path
if filtPath.startswith('/'):
filtPath = filtPath[1:]
if filtPath != '':
rsync_excludes = rsync_excludes + " --exclude=%s" % filtPath
md5_excludes = md5_excludes + " ! -path %s/%s" % (
cmd.srcFolderPath, filtPath)
total_size = int(
shell.call(
"rsync -aznv %s/ %s %s | grep -o -P 'total size is \K\d*'"
% (cmd.srcFolderPath, dst_folder_path, rsync_excludes)))
stage = get_task_stage(cmd)
reporter = Report.from_spec(cmd, "MigrateVolume")
def _get_progress(synced):
def get_written(regex):
matcher = re.match(regex, line)
return int(matcher.group(1)) if matcher else 0
lines = f.readlines()
writing = 0
for line in lines:
if line[1] == ' ' and line[-1] == '\n':
synced += get_written(r'\s.*?(\d+)\s+100%')
elif line[-1] == '\r' and line[1] == ' ':
writing = get_written(r'.*?(\d+)\s+\d+%[^\r]*\r$')
reporter.progress_report(
get_exact_percent(
float(synced + writing) / total_size * 100, stage))
return synced
t_shell.bash_progress_1(
"rsync -az --progress %s/ %s %s > %s" %
(cmd.srcFolderPath, dst_folder_path, rsync_excludes, PFILE),
_get_progress)
src_md5 = t_shell.call(
"find %s -type f %s -exec md5sum {} \; | awk '{ print $1 }' | sort | md5sum"
% (cmd.srcFolderPath, md5_excludes))
dst_md5 = t_shell.call(
"find %s -type f -exec md5sum {} \; | awk '{ print $1 }' | sort | md5sum"
% dst_folder_path)
if src_md5 != dst_md5:
rsp.error = "failed to copy files from %s to %s, md5sum not match" % (
cmd.srcFolderPath, dst_folder_path)
rsp.success = False
if not cmd.isMounted:
linux.umount(mount_path)
finally:
if temp_dir is not None:
return_code = shell.run("mount | grep '%s'" % temp_dir)
if return_code != 0:
# in case dir is not empty
try:
os.rmdir(temp_dir)
except OSError as e:
logger.warn("delete temp_dir %s failed: %s",
(temp_dir, str(e)))
else:
logger.warn(
"temp_dir %s still had mounted destination primary storage, skip cleanup operation"
% temp_dir)
f.close()
linux.rm_file_force(PFILE)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def rebase_volume_backing_file(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = NfsRebaseVolumeBackingFileRsp()
if not cmd.dstImageCacheTemplateFolderPath:
qcow2s = shell.call("find %s -type f -regex '.*\.qcow2$'" %
cmd.dstVolumeFolderPath)
else:
qcow2s = shell.call(
"find %s %s -type f -regex '.*\.qcow2$'" %
(cmd.dstVolumeFolderPath, cmd.dstImageCacheTemplateFolderPath))
for qcow2 in qcow2s.split():
fmt = shell.call(
"%s %s | grep '^file format' | awk -F ': ' '{ print $2 }'" %
(qemu_img.subcmd('info'), qcow2))
if fmt.strip() != "qcow2":
continue
backing_file = linux.qcow2_get_backing_file(qcow2)
if backing_file == "":
continue
# actions like `create snapshot -> recover snapshot -> delete snapshot` may produce garbage qcow2, whose backing file doesn't exist
new_backing_file = backing_file.replace(cmd.srcPsMountPath,
cmd.dstPsMountPath)
if not os.path.exists(new_backing_file):
logger.debug(
"the backing file[%s] of volume[%s] doesn't exist, skip rebasing"
% (new_backing_file, qcow2))
continue
linux.qcow2_rebase_no_check(new_backing_file, qcow2)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def resize_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
install_path = cmd.installPath
rsp = ResizeVolumeRsp()
shell.call("qemu-img resize %s %s" % (install_path, cmd.size))
ret = linux.qcow2_virtualsize(install_path)
rsp.size = ret
return jsonobject.dumps(rsp)
def _get_disk_capacity(self, uuid):
path = self.mount_path.get(uuid)
if not path:
raise Exception(
'cannot find mount path of primary storage[uuid: %s]' % uuid)
return linux.get_disk_capacity_by_df(path)
def _json_meta_file_name(self, path):
return path + '.json'
def _set_capacity_to_response(self, uuid, rsp):
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
uuid)
@kvmagent.replyerror
def update_mount_point(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = UpdateMountPointResponse()
linux.is_valid_nfs_url(cmd.newMountPoint)
if not linux.is_mounted(cmd.mountPath, cmd.newMountPoint):
# umount old one
if linux.is_mounted(cmd.mountPath, cmd.oldMountPoint):
linux.umount(cmd.mountPath)
# mount new
linux.mount(cmd.newMountPoint, cmd.mountPath, cmd.options, "nfs4")
self.mount_path[cmd.uuid] = cmd.mountPath
logger.debug(
'updated the mount path[%s] mounting point from %s to %s' %
(cmd.mountPath, cmd.oldMountPoint, cmd.newMountPoint))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_volume_base_image_path(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeBaseImagePathRsp()
if not os.path.basename(cmd.volumeInstallDir).endswith(cmd.volumeUuid):
raise Exception('maybe you pass a wrong install dir')
path = linux.get_qcow2_base_image_recusively(cmd.volumeInstallDir,
cmd.imageCacheDir)
if not path:
return jsonobject.dumps(rsp)
rsp.path = path
rsp.size = linux.get_qcow2_file_chain_size(path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@in_bash
def ping(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
if cmd.uuid not in self.mount_path.keys():
self.mount_path[cmd.uuid] = cmd.mountPath
mount_path = self.mount_path[cmd.uuid]
# if nfs service stop, os.path.isdir will hung
if not linux.timeout_isdir(mount_path) or not linux.is_mounted(
path=mount_path):
raise Exception(
'the mount path[%s] of the nfs primary storage[uuid:%s] is not existing'
% (mount_path, cmd.uuid))
test_file = os.path.join(mount_path,
'%s-ping-test-file' % uuidhelper.uuid())
touch = shell.ShellCmd('timeout 60 touch %s' % test_file)
touch(False)
if touch.return_code == 124:
raise Exception(
'unable to access the mount path[%s] of the nfs primary storage[uuid:%s] in 60s, timeout'
% (mount_path, cmd.uuid))
elif touch.return_code != 0:
touch.raise_error()
linux.rm_file_force(test_file)
return jsonobject.dumps(NfsResponse())
@kvmagent.replyerror
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot_to_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = OfflineMergeSnapshotRsp()
if not cmd.fullRebase:
linux.qcow2_rebase(cmd.srcPath, cmd.destPath)
else:
tmp = os.path.join(os.path.dirname(cmd.destPath),
'%s.qcow2' % uuidhelper.uuid())
linux.create_template(cmd.destPath, tmp)
shell.call("mv %s %s" % (tmp, cmd.destPath))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def move_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MoveBitsRsp()
if not os.path.exists(cmd.srcPath):
rsp.error = "%s is not existing" % cmd.srcPath
rsp.success = False
else:
dirname = os.path.dirname(cmd.destPath)
if not os.path.exists(dirname):
os.makedirs(dirname)
shell.call("mv %s %s" % (cmd.srcPath, cmd.destPath))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def rebase_and_merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
snapshots = cmd.snapshotInstallPaths
count = len(snapshots)
for i in range(count):
if i + 1 < count:
target = snapshots[i]
backing_file = snapshots[i + 1]
linux.qcow2_rebase_no_check(backing_file, target)
latest = snapshots[0]
rsp = RebaseAndMergeSnapshotsResponse()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
try:
linux.create_template(latest, cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = cmd.workspaceInstallPath
self._set_capacity_to_response(cmd.uuid, rsp)
except linux.LinuxError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotResponse()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
try:
linux.create_template(cmd.snapshotInstallPath,
cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
cmd.workspaceInstallPath)
self._set_capacity_to_response(cmd.uuid, rsp)
except linux.LinuxError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CopyToSftpBackupStorageResponse()
def upload():
if not os.path.exists(cmd.primaryStorageInstallPath):
raise kvmagent.KvmError('cannot find %s' %
cmd.primaryStorageInstallPath)
linux.scp_upload(cmd.backupStorageHostName,
cmd.backupStorageSshKey,
cmd.primaryStorageInstallPath,
cmd.backupStorageInstallPath,
cmd.backupStorageUserName,
cmd.backupStorageSshPort)
try:
upload()
except kvmagent.KvmError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(cmd, req)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
@kvmagent.replyerror
def download_from_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
mount_path = self.mount_path.get(cmd.uuid)
self.check_nfs_mounted(mount_path)
cachedir = None if cmd.isData else mount_path
self.imagestore_client.download_from_imagestore(
cachedir, cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath)
rsp = kvmagent.AgentResponse()
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def reinit_image(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ReInitImageResponse()
install_path = cmd.imagePath
dirname = os.path.dirname(cmd.volumePath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
new_volume_path = os.path.join(dirname,
'{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone_with_cmd(install_path, new_volume_path, cmd)
rsp.newVolumeInstallPath = new_volume_path
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def revert_volume_from_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotResponse()
install_path = cmd.snapshotInstallPath
new_volume_path = os.path.join(os.path.dirname(install_path),
'{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone_with_cmd(install_path, new_volume_path, cmd)
rsp.newVolumeInstallPath = new_volume_path
size = linux.qcow2_virtualsize(new_volume_path)
rsp.size = size
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckIsBitsExistingRsp()
rsp.existing = os.path.exists(cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def delete(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = DeleteResponse()
if cmd.folder:
linux.rm_dir_checked(cmd.installPath)
else:
kvmagent.deleteImage(cmd.installPath)
logger.debug('successfully delete %s' % cmd.installPath)
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def remount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MountResponse()
linux.is_valid_nfs_url(cmd.url)
linux.remount(cmd.url, cmd.mountPath, cmd.options)
self.mount_path[cmd.uuid] = cmd.mountPath
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@lock.lock('mount')
def mount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MountResponse()
linux.is_valid_nfs_url(cmd.url)
if not linux.is_mounted(cmd.mountPath, cmd.url):
linux.mount(cmd.url, cmd.mountPath, cmd.options, "nfs4")
with tempfile.TemporaryFile(dir=cmd.mountPath) as f:
try:
fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
linux.umount(cmd.mountPath)
raise Exception(
'File lock unavailable on NFS: {}, mount options: {}'.
format(cmd.url, cmd.options))
self.mount_path[cmd.uuid] = cmd.mountPath
logger.debug(
http.path_msg(self.MOUNT_PATH,
'mounted %s on %s' % (cmd.url, cmd.mountPath)))
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def umount(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = UnmountResponse()
if linux.is_mounted(path=cmd.mountPath):
ret = linux.umount(cmd.mountPath)
if not ret:
logger.warn(
http.path_msg(
self.UNMOUNT_PATH, 'unmount %s from %s failed' %
(cmd.mountPath, cmd.url)))
logger.debug(
http.path_msg(self.UNMOUNT_PATH,
'umounted %s from %s' % (cmd.mountPath, cmd.url)))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_capacity(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetCapacityResponse()
self._set_capacity_to_response(cmd.uuid, rsp)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_folder(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CreateEmptyVolumeResponse()
try:
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname)
except Exception as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = 'unable to create folder[installUrl: %s], %s' % (
cmd.installUrl, str(e))
rsp.success = False
return jsonobject.dumps(rsp)
self._set_capacity_to_response(cmd.uuid, rsp)
logger.debug('successfully create folder at %s' % cmd.installUrl)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_empty_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CreateEmptyVolumeResponse()
try:
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname)
linux.qcow2_create_with_cmd(cmd.installUrl, cmd.size, cmd)
except Exception as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = 'unable to create empty volume[uuid:%s, name:%s], %s' % (
cmd.uuid, cmd.name, str(e))
rsp.success = False
return jsonobject.dumps(rsp)
meta = VolumeMeta()
meta.account_uuid = cmd.accountUuid
meta.hypervisor_type = cmd.hypervisorType
meta.name = cmd.name
meta.uuid = cmd.volumeUuid
meta.size = cmd.size
meta_path = self._json_meta_file_name(cmd.installUrl)
with open(meta_path, 'w') as fd:
fd.write(jsonobject.dumps(meta, pretty=True))
self._set_capacity_to_response(cmd.uuid, rsp)
logger.debug(
'successfully create empty volume[uuid:%s, name:%s, size:%s] at %s'
% (cmd.uuid, cmd.name, cmd.size, cmd.installUrl))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_template_from_root_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CreateTemplateFromRootVolumeRsp()
try:
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
t_shell = traceable_shell.get_shell(cmd)
linux.create_template(cmd.rootVolumePath,
cmd.installPath,
shell=t_shell)
except linux.LinuxError as e:
linux.rm_file_force(cmd.installPath)
logger.warn(linux.get_exception_stacktrace())
rsp.error = 'unable to create image to root@%s:%s from root volume[%s], %s' % (
cmd.sftpBackupStorageHostName, cmd.installPath,
cmd.rootVolumePath, str(e))
rsp.success = False
self._set_capacity_to_response(cmd.uuid, rsp)
logger.debug('successfully created template[%s] from root volume[%s]' %
(cmd.installPath, cmd.rootVolumePath))
return jsonobject.dumps(rsp)
def check_nfs_mounted(self, mount_path):
if not linux.is_mounted(mount_path):
raise Exception('NFS not mounted on: %s' % mount_path)
@kvmagent.replyerror
def download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.check_nfs_mounted(self.mount_path.get(cmd.uuid))
rsp = DownloadBitsFromSftpBackupStorageResponse()
try:
linux.scp_download(cmd.hostname, cmd.sshKey,
cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath, cmd.username,
cmd.sshPort)
logger.debug('successfully download %s/%s to %s' %
(cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath))
self._set_capacity_to_response(cmd.uuid, rsp)
except Exception as e:
content = traceback.format_exc()
logger.warn(content)
err = "unable to download %s/%s, because %s" % (
cmd.hostname, cmd.backupStorageInstallPath, str(e))
rsp.error = err
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_root_volume_from_template(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CreateRootVolumeFromTemplateResponse()
if not os.path.exists(cmd.templatePathInCache):
rsp.error = self.ERR_UNABLE_TO_FIND_IMAGE_IN_CACHE
rsp.success = False
return jsonobject.dumps(rsp)
try:
dirname = os.path.dirname(cmd.installUrl)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
linux.qcow2_clone_with_cmd(cmd.templatePathInCache, cmd.installUrl,
cmd)
logger.debug(
'successfully create root volume[%s] from template in cache[%s]'
% (cmd.installUrl, cmd.templatePathInCache))
meta = VolumeMeta()
meta.account_uuid = cmd.accountUuid
meta.hypervisor_type = cmd.hypervisorType
meta.name = cmd.name
meta.uuid = cmd.volumeUuid
meta.size = os.path.getsize(cmd.templatePathInCache)
meta_path = self._json_meta_file_name(cmd.installUrl)
with open(meta_path, 'w') as fd:
fd.write(jsonobject.dumps(meta, pretty=True))
self._set_capacity_to_response(cmd.uuid, rsp)
logger.debug(
'successfully create root volume[%s] from template in cache[%s]'
% (cmd.installUrl, cmd.templatePathInCache))
except Exception as e:
content = traceback.format_exc()
logger.warn(content)
err = 'unable to clone qcow2 template[%s] to %s' % (
cmd.templatePathInCache, cmd.installUrl)
rsp.error = err
rsp.success = False
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@completetask
def download_from_kvmhost(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = DownloadBitsFromKvmHostRsp()
install_abs_path = cmd.primaryStorageInstallPath
last_task = self.load_and_save_task(req, rsp, os.path.exists,
install_abs_path)
if last_task and last_task.agent_pid == os.getpid():
rsp = self.wait_task_complete(last_task)
return jsonobject.dumps(rsp)
linux.scp_download(cmd.hostname, cmd.sshKey,
cmd.backupStorageInstallPath, install_abs_path,
cmd.username, cmd.sshPort, cmd.bandWidth)
rsp.format = linux.get_img_fmt(install_abs_path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def cancel_download_from_kvmhost(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = kvmagent.AgentResponse()
install_abs_path = cmd.primaryStorageInstallPath
shell.run("pkill -9 -f '%s'" % install_abs_path)
linux.rm_file_force(cmd.primaryStorageInstallPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_download_bits_from_kvmhost_progress(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetDownloadBitsFromKvmHostProgressRsp()
rsp.totalSize = linux.get_total_file_size(cmd.volumePaths)
return jsonobject.dumps(rsp)
class SharedMountPointPrimaryStoragePlugin(kvmagent.KvmAgent):
CONNECT_PATH = "/sharedmountpointprimarystorage/connect"
CREATE_VOLUME_FROM_CACHE_PATH = "/sharedmountpointprimarystorage/createrootvolume"
DELETE_BITS_PATH = "/sharedmountpointprimarystorage/bits/delete"
CREATE_TEMPLATE_FROM_VOLUME_PATH = "/sharedmountpointprimarystorage/createtemplatefromvolume"
UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH = "/sharedmountpointprimarystorage/sftp/upload"
DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH = "/sharedmountpointprimarystorage/sftp/download"
UPLOAD_BITS_TO_IMAGESTORE_PATH = "/sharedmountpointprimarystorage/imagestore/upload"
COMMIT_BITS_TO_IMAGESTORE_PATH = "/sharedmountpointprimarystorage/imagestore/commit"
DOWNLOAD_BITS_FROM_IMAGESTORE_PATH = "/sharedmountpointprimarystorage/imagestore/download"
REVERT_VOLUME_FROM_SNAPSHOT_PATH = "/sharedmountpointprimarystorage/volume/revertfromsnapshot"
MERGE_SNAPSHOT_PATH = "/sharedmountpointprimarystorage/snapshot/merge"
OFFLINE_MERGE_SNAPSHOT_PATH = "/sharedmountpointprimarystorage/snapshot/offlinemerge"
CREATE_EMPTY_VOLUME_PATH = "/sharedmountpointprimarystorage/volume/createempty"
CREATE_FOLDER_PATH = "/sharedmountpointprimarystorage/volume/createfolder"
CHECK_BITS_PATH = "/sharedmountpointprimarystorage/bits/check"
GET_VOLUME_SIZE_PATH = "/sharedmountpointprimarystorage/volume/getsize"
RESIZE_VOLUME_PATH = "/sharedmountpointprimarystorage/volume/resize"
REINIT_IMAGE_PATH = "/sharedmountpointprimarystorage/volume/reinitimage"
DOWNLOAD_BITS_FROM_KVM_HOST_PATH = "/sharedmountpointprimarystorage/kvmhost/download"
CANCEL_DOWNLOAD_BITS_FROM_KVM_HOST_PATH = "/sharedmountpointprimarystorage/kvmhost/download/cancel"
GET_DOWNLOAD_BITS_FROM_KVM_HOST_PROGRESS_PATH = "/sharedmountpointprimarystorage/kvmhost/download/progress"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.CONNECT_PATH, self.connect)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH, self.create_root_volume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete_bits)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH, self.create_template_from_volume)
http_server.register_async_uri(self.UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH, self.upload_to_sftp)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH, self.download_from_sftp)
http_server.register_async_uri(self.UPLOAD_BITS_TO_IMAGESTORE_PATH, self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_BITS_TO_IMAGESTORE_PATH, self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_IMAGESTORE_PATH, self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH, self.revert_volume_from_snapshot)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.merge_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_SNAPSHOT_PATH, self.offline_merge_snapshots)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.create_empty_volume)
http_server.register_async_uri(self.CREATE_FOLDER_PATH, self.create_folder)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH, self.get_volume_size)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH, self.resize_volume)
http_server.register_async_uri(self.REINIT_IMAGE_PATH, self.reinit_image)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_KVM_HOST_PATH, self.download_from_kvmhost)
http_server.register_async_uri(self.CANCEL_DOWNLOAD_BITS_FROM_KVM_HOST_PATH, self.cancel_download_from_kvmhost)
http_server.register_async_uri(self.GET_DOWNLOAD_BITS_FROM_KVM_HOST_PROGRESS_PATH, self.get_download_bits_from_kvmhost_progress)
self.imagestore_client = ImageStoreClient()
self.id_files = {}
def stop(self):
pass
@kvmagent.replyerror
def resize_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
install_path = cmd.installPath
rsp = ResizeVolumeRsp()
shell.call("qemu-img resize %s %s" % (install_path, cmd.size))
ret = linux.qcow2_virtualsize(install_path)
rsp.size = ret
return jsonobject.dumps(rsp)
@staticmethod
def _get_disk_capacity(mount_point):
if not mount_point:
raise Exception('storage mount point cannot be None')
return linux.get_disk_capacity_by_df(mount_point)
@kvmagent.replyerror
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.installPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def connect(self, req):
none_shared_mount_fs_type = ['xfs', 'ext2', 'ext3', 'ext4', 'vfat', 'tmpfs', 'btrfs']
cmd = jsonobject.loads(req[http.REQUEST_BODY])
if not linux.timeout_isdir(cmd.mountPoint):
raise kvmagent.KvmError('%s is not a directory, the mount point seems not setup' % cmd.mountPoint)
folder_fs_type = shell.call("df -T %s|tail -1|awk '{print $2}'" % cmd.mountPoint).strip()
if folder_fs_type in none_shared_mount_fs_type:
raise kvmagent.KvmError(
'%s filesystem is %s, which is not a shared mount point type.' % (cmd.mountPoint, folder_fs_type))
id_dir = os.path.join(cmd.mountPoint, "zstack_smp_id_file")
shell.call("mkdir -p %s" % id_dir)
lock_file = os.path.join(id_dir, "uuid.lock")
@lock.file_lock(lock_file, locker=lock.Flock())
def check_other_smp_and_set_id_file(uuid, existUuids):
o = shell.ShellCmd('''\
ls %s | grep -v %s | grep -o "[0-9a-f]\{8\}[0-9a-f]\{4\}[1-5][0-9a-f]\{3\}[89ab][0-9a-f]\{3\}[0-9a-f]\{12\}"\
''' % (id_dir, uuid))
o(False)
if o.return_code != 0:
file_uuids = []
else:
file_uuids = o.stdout.splitlines()
for file_uuid in file_uuids:
if file_uuid in existUuids:
raise Exception(
"the mount point [%s] has been occupied by other SMP[uuid:%s], Please attach this directly"
% (cmd.mountPoint, file_uuid))
logger.debug("existing id files: %s" % file_uuids)
self.id_files[uuid] = os.path.join(id_dir, uuid)
if not os.path.exists(self.id_files[uuid]):
# check if hosts in the same cluster mount the same path but different storages.
rsp.isFirst = True
for file_uuid in file_uuids:
linux.rm_file_force(os.path.join(id_dir, file_uuid))
linux.touch_file(self.id_files[uuid])
linux.sync_file(self.id_files[uuid])
rsp = ConnectRsp()
check_other_smp_and_set_id_file(cmd.uuid, cmd.existUuids)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_root_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if not os.path.exists(cmd.templatePathInCache):
rsp.error = "unable to find image in cache"
rsp.success = False
return jsonobject.dumps(rsp)
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
linux.qcow2_clone_with_cmd(cmd.templatePathInCache, cmd.installPath, cmd)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def delete_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if cmd.folder:
linux.rm_dir_checked(cmd.path)
else:
kvmagent.deleteImage(cmd.path)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@rollback.rollback
def create_template_from_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
@rollback.rollbackable
def _0():
linux.rm_file_force(cmd.installPath)
_0()
t_shell = traceable_shell.get_shell(cmd)
linux.create_template(cmd.volumePath, cmd.installPath, shell=t_shell)
logger.debug('successfully created template[%s] from volume[%s]' % (cmd.installPath, cmd.volumePath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
def upload():
if not os.path.exists(cmd.primaryStorageInstallPath):
raise kvmagent.KvmError('cannot find %s' % cmd.primaryStorageInstallPath)
linux.scp_upload(cmd.hostname, cmd.sshKey, cmd.primaryStorageInstallPath, cmd.backupStorageInstallPath, cmd.username, cmd.sshPort)
upload()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
linux.scp_download(cmd.hostname, cmd.sshKey, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath, cmd.username, cmd.sshPort)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
logger.debug('successfully download %s/%s to %s' % (cmd.hostname, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def upload_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(cmd, req)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
@kvmagent.replyerror
def download_from_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
cachedir = None if cmd.isData else cmd.mountPoint
self.imagestore_client.download_from_imagestore(cachedir, cmd.hostname, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath)
rsp = AgentRsp()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def reinit_image(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ReinitImageRsp()
install_path = cmd.imageInstallPath
dirname = os.path.dirname(cmd.volumeInstallPath)
if not os.path.exists(dirname):
os.makedirs(dirname, 0775)
new_volume_path = os.path.join(dirname, '{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone_with_cmd(install_path, new_volume_path, cmd)
rsp.newVolumeInstallPath = new_volume_path
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def revert_volume_from_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotRsp()
install_path = cmd.snapshotInstallPath
new_volume_path = os.path.join(os.path.dirname(install_path), '{0}.qcow2'.format(uuidhelper.uuid()))
linux.qcow2_clone_with_cmd(install_path, new_volume_path, cmd)
size = linux.qcow2_virtualsize(new_volume_path)
rsp.newVolumeInstallPath = new_volume_path
rsp.size = size
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotRsp()
workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
if not os.path.exists(workspace_dir):
os.makedirs(workspace_dir)
linux.create_template(cmd.snapshotInstallPath, cmd.workspaceInstallPath)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.workspaceInstallPath)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def offline_merge_snapshots(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if not cmd.fullRebase:
linux.qcow2_rebase(cmd.srcPath, cmd.destPath)
else:
tmp = os.path.join(os.path.dirname(cmd.destPath), '%s.qcow2' % uuidhelper.uuid())
linux.create_template(cmd.destPath, tmp)
shell.call("mv %s %s" % (tmp, cmd.destPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_folder(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname)
logger.debug('successfully create empty volume[uuid:%s, size:%s] at %s' % (cmd.volumeUuid, cmd.size, cmd.installPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def create_empty_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
dirname = os.path.dirname(cmd.installPath)
if not os.path.exists(dirname):
os.makedirs(dirname)
if cmd.backingFile:
linux.qcow2_create_with_backing_file_and_cmd(cmd.backingFile, cmd.installPath, cmd)
else:
linux.qcow2_create_with_cmd(cmd.installPath, cmd.size, cmd)
logger.debug('successfully create empty volume[uuid:%s, size:%s] at %s' % (cmd.volumeUuid, cmd.size, cmd.installPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckBitsRsp()
rsp.existing = os.path.exists(cmd.path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@completetask
def download_from_kvmhost(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = DownloadBitsFromKvmHostRsp()
install_abs_path = cmd.primaryStorageInstallPath
last_task = self.load_and_save_task(req, rsp, os.path.exists, install_abs_path)
if last_task and last_task.agent_pid == os.getpid():
rsp = self.wait_task_complete(last_task)
return jsonobject.dumps(rsp)
linux.scp_download(cmd.hostname, cmd.sshKey, cmd.backupStorageInstallPath, install_abs_path, cmd.username, cmd.sshPort, cmd.bandWidth)
rsp.format = linux.get_img_fmt(install_abs_path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def cancel_download_from_kvmhost(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = kvmagent.AgentResponse()
install_abs_path = cmd.primaryStorageInstallPath
shell.run("pkill -9 -f '%s'" % install_abs_path)
linux.rm_file_force(cmd.primaryStorageInstallPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_download_bits_from_kvmhost_progress(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetDownloadBitsFromKvmHostProgressRsp()
rsp.totalSize = linux.get_total_file_size(cmd.volumePaths)
return jsonobject.dumps(rsp)
class SharedBlockPlugin(kvmagent.KvmAgent):
CONNECT_PATH = "/sharedblock/connect"
DISCONNECT_PATH = "/sharedblock/disconnect"
CREATE_VOLUME_FROM_CACHE_PATH = "/sharedblock/createrootvolume"
DELETE_BITS_PATH = "/sharedblock/bits/delete"
CREATE_TEMPLATE_FROM_VOLUME_PATH = "/sharedblock/createtemplatefromvolume"
UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH = "/sharedblock/sftp/upload"
DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH = "/sharedblock/sftp/download"
UPLOAD_BITS_TO_IMAGESTORE_PATH = "/sharedblock/imagestore/upload"
COMMIT_BITS_TO_IMAGESTORE_PATH = "/sharedblock/imagestore/commit"
DOWNLOAD_BITS_FROM_IMAGESTORE_PATH = "/sharedblock/imagestore/download"
REVERT_VOLUME_FROM_SNAPSHOT_PATH = "/sharedblock/volume/revertfromsnapshot"
MERGE_SNAPSHOT_PATH = "/sharedblock/snapshot/merge"
OFFLINE_MERGE_SNAPSHOT_PATH = "/sharedblock/snapshot/offlinemerge"
CREATE_EMPTY_VOLUME_PATH = "/sharedblock/volume/createempty"
CHECK_BITS_PATH = "/sharedblock/bits/check"
RESIZE_VOLUME_PATH = "/sharedblock/volume/resize"
CONVERT_IMAGE_TO_VOLUME = "/sharedblock/image/tovolume"
CHANGE_VOLUME_ACTIVE_PATH = "/sharedblock/volume/active"
GET_VOLUME_SIZE_PATH = "/sharedblock/volume/getsize"
CHECK_DISKS_PATH = "/sharedblock/disks/check"
ADD_SHARED_BLOCK = "/sharedblock/disks/add"
MIGRATE_DATA_PATH = "/sharedblock/volume/migrate"
GET_BLOCK_DEVICES_PATH = "/sharedblock/blockdevices"
DOWNLOAD_BITS_FROM_KVM_HOST_PATH = "/sharedblock/kvmhost/download"
CANCEL_DOWNLOAD_BITS_FROM_KVM_HOST_PATH = "/sharedblock/kvmhost/download/cancel"
GET_BACKING_CHAIN_PATH = "/sharedblock/volume/backingchain"
CONVERT_VOLUME_PROVISIONING_PATH = "/sharedblock/volume/convertprovisioning"
def start(self):
http_server = kvmagent.get_http_server()
http_server.register_async_uri(self.CONNECT_PATH, self.connect)
http_server.register_async_uri(self.DISCONNECT_PATH, self.disconnect)
http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH,
self.create_root_volume)
http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete_bits)
http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME_PATH,
self.create_template_from_volume)
http_server.register_async_uri(
self.UPLOAD_BITS_TO_SFTP_BACKUPSTORAGE_PATH, self.upload_to_sftp)
http_server.register_async_uri(
self.DOWNLOAD_BITS_FROM_SFTP_BACKUPSTORAGE_PATH,
self.download_from_sftp)
http_server.register_async_uri(self.UPLOAD_BITS_TO_IMAGESTORE_PATH,
self.upload_to_imagestore)
http_server.register_async_uri(self.COMMIT_BITS_TO_IMAGESTORE_PATH,
self.commit_to_imagestore)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_IMAGESTORE_PATH,
self.download_from_imagestore)
http_server.register_async_uri(self.REVERT_VOLUME_FROM_SNAPSHOT_PATH,
self.revert_volume_from_snapshot)
http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH,
self.merge_snapshot)
http_server.register_async_uri(self.OFFLINE_MERGE_SNAPSHOT_PATH,
self.offline_merge_snapshots)
http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH,
self.create_empty_volume)
http_server.register_async_uri(self.CONVERT_IMAGE_TO_VOLUME,
self.convert_image_to_volume)
http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
http_server.register_async_uri(self.RESIZE_VOLUME_PATH,
self.resize_volume)
http_server.register_async_uri(self.CHANGE_VOLUME_ACTIVE_PATH,
self.active_lv)
http_server.register_async_uri(self.GET_VOLUME_SIZE_PATH,
self.get_volume_size)
http_server.register_async_uri(self.CHECK_DISKS_PATH, self.check_disks)
http_server.register_async_uri(self.ADD_SHARED_BLOCK, self.add_disk)
http_server.register_async_uri(self.MIGRATE_DATA_PATH,
self.migrate_volumes)
http_server.register_async_uri(self.GET_BLOCK_DEVICES_PATH,
self.get_block_devices)
http_server.register_async_uri(self.DOWNLOAD_BITS_FROM_KVM_HOST_PATH,
self.download_from_kvmhost)
http_server.register_async_uri(
self.CANCEL_DOWNLOAD_BITS_FROM_KVM_HOST_PATH,
self.cancel_download_from_kvmhost)
http_server.register_async_uri(self.GET_BACKING_CHAIN_PATH,
self.get_backing_chain)
http_server.register_async_uri(self.CONVERT_VOLUME_PROVISIONING_PATH,
self.convert_volume_provisioning)
self.imagestore_client = ImageStoreClient()
def stop(self):
pass
@kvmagent.replyerror
def check_disks(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if cmd.failIfNoPath:
linux.set_fail_if_no_path()
for diskUuid in cmd.sharedBlockUuids:
disk = CheckDisk(diskUuid)
path = disk.get_path()
if cmd.rescan:
disk.rescan(path.split("/")[-1])
if cmd.vgUuid is not None and lvm.vg_exists(cmd.vgUuid):
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(
cmd.vgUuid, False)
return jsonobject.dumps(rsp)
@staticmethod
def create_vg_if_not_found(vgUuid, diskPaths, hostUuid, forceWipe=False):
@linux.retry(times=5, sleep_time=random.uniform(0.1, 3))
def find_vg(vgUuid, raise_exception=True):
cmd = shell.ShellCmd(
"timeout 5 vgscan --ignorelockingfailure; vgs --nolocking %s -otags | grep %s"
% (vgUuid, INIT_TAG))
cmd(is_exception=False)
if cmd.return_code != 0 and raise_exception:
raise RetryException("can not find vg %s with tag %s" %
(vgUuid, INIT_TAG))
elif cmd.return_code != 0:
return False
return True
try:
find_vg(vgUuid)
except RetryException as e:
if forceWipe is True:
lvm.wipe_fs(diskPaths, vgUuid)
cmd = shell.ShellCmd(
"vgcreate -qq --shared --addtag '%s::%s::%s::%s' --metadatasize %s %s %s"
% (INIT_TAG, hostUuid, time.time(),
bash.bash_o("hostname").strip(), DEFAULT_VG_METADATA_SIZE,
vgUuid, " ".join(diskPaths)))
cmd(is_exception=False)
logger.debug("created vg %s, ret: %s, stdout: %s, stderr: %s" %
(vgUuid, cmd.return_code, cmd.stdout, cmd.stderr))
if cmd.return_code == 0 and find_vg(vgUuid, False) is True:
return True
try:
if find_vg(vgUuid) is True:
return True
except RetryException as ee:
raise Exception(
"can not find vg %s with disks: %s and create vg return: %s %s %s "
% (vgUuid, diskPaths, cmd.return_code, cmd.stdout,
cmd.stderr))
except Exception as ee:
raise ee
except Exception as e:
raise e
return False
@kvmagent.replyerror
@lock.file_lock(LOCK_FILE)
def connect(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ConnectRsp()
diskPaths = set()
def config_lvm(host_id, enableLvmetad=False):
lvm.backup_lvm_config()
lvm.reset_lvm_conf_default()
lvm.config_lvm_by_sed("use_lvmlockd", "use_lvmlockd=1",
["lvm.conf", "lvmlocal.conf"])
if enableLvmetad:
lvm.config_lvm_by_sed("use_lvmetad", "use_lvmetad=1",
["lvm.conf", "lvmlocal.conf"])
else:
lvm.config_lvm_by_sed("use_lvmetad", "use_lvmetad=0",
["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("host_id", "host_id=%s" % host_id,
["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed(
"sanlock_lv_extend",
"sanlock_lv_extend=%s" % DEFAULT_SANLOCK_LV_SIZE,
["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("lvmlockd_lock_retries",
"lvmlockd_lock_retries=6",
["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("issue_discards", "issue_discards=1",
["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("reserved_stack", "reserved_stack=256",
["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_by_sed("reserved_memory", "reserved_memory=131072",
["lvm.conf", "lvmlocal.conf"])
lvm.config_lvm_filter(["lvm.conf", "lvmlocal.conf"])
lvm.config_sanlock_by_sed("sh_retries", "sh_retries=20")
lvm.config_sanlock_by_sed("logfile_priority", "logfile_priority=7")
lvm.config_sanlock_by_sed("renewal_read_extend_sec",
"renewal_read_extend_sec=24")
lvm.config_sanlock_by_sed("debug_renew", "debug_renew=1")
lvm.config_sanlock_by_sed("use_watchdog", "use_watchdog=0")
sanlock_hostname = "%s-%s-%s" % (
cmd.vgUuid[:8], cmd.hostUuid[:8],
bash.bash_o("hostname").strip()[:20])
lvm.config_sanlock_by_sed("our_host_name",
"our_host_name=%s" % sanlock_hostname)
config_lvm(cmd.hostId, cmd.enableLvmetad)
for diskUuid in cmd.sharedBlockUuids:
disk = CheckDisk(diskUuid)
diskPaths.add(disk.get_path())
lvm.start_lvmlockd()
lvm.check_gl_lock()
logger.debug("find/create vg %s lock..." % cmd.vgUuid)
rsp.isFirst = self.create_vg_if_not_found(cmd.vgUuid, diskPaths,
cmd.hostUuid, cmd.forceWipe)
lvm.check_stuck_vglk()
logger.debug("starting vg %s lock..." % cmd.vgUuid)
lvm.start_vg_lock(cmd.vgUuid)
if lvm.lvm_vgck(cmd.vgUuid,
60)[0] is False and lvm.lvm_check_operation(
cmd.vgUuid) is False:
lvm.drop_vg_lock(cmd.vgUuid)
logger.debug("restarting vg %s lock..." % cmd.vgUuid)
lvm.check_gl_lock()
lvm.start_vg_lock(cmd.vgUuid)
lvm.clean_vg_exists_host_tags(cmd.vgUuid, cmd.hostUuid, HEARTBEAT_TAG)
lvm.add_vg_tag(
cmd.vgUuid,
"%s::%s::%s::%s" % (HEARTBEAT_TAG, cmd.hostUuid, time.time(),
bash.bash_o('hostname').strip()))
self.clear_stalled_qmp_socket()
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
rsp.hostId = lvm.get_running_host_id(cmd.vgUuid)
rsp.vgLvmUuid = lvm.get_vg_lvm_uuid(cmd.vgUuid)
rsp.hostUuid = cmd.hostUuid
return jsonobject.dumps(rsp)
@staticmethod
@bash.in_bash
def clear_stalled_qmp_socket():
def get_used_qmp_file():
t = bash.bash_o("ps aux | grep -Eo -- '-qmp unix:%s/\w*\.sock'" %
QMP_SOCKET_PATH).splitlines()
qmp = []
for i in t:
qmp.append(i.split("/")[-1])
return qmp
exists_qmp_files = set(
bash.bash_o("ls %s" % QMP_SOCKET_PATH).splitlines())
if len(exists_qmp_files) == 0:
return
running_qmp_files = set(get_used_qmp_file())
if len(running_qmp_files) == 0:
bash.bash_roe("/bin/rm %s/*" % QMP_SOCKET_PATH)
return
need_delete_qmp_files = exists_qmp_files.difference(running_qmp_files)
if len(need_delete_qmp_files) == 0:
return
for f in need_delete_qmp_files:
bash.bash_roe("/bin/rm %s/%s" % (QMP_SOCKET_PATH, f))
@kvmagent.replyerror
@lock.file_lock(LOCK_FILE)
def disconnect(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
@linux.retry(times=3, sleep_time=random.uniform(0.1, 3))
def find_vg(vgUuid):
cmd = shell.ShellCmd("vgs --nolocking %s -otags | grep %s" %
(vgUuid, INIT_TAG))
cmd(is_exception=False)
if cmd.return_code == 0:
return True
logger.debug("can not find vg %s with tag %s" % (vgUuid, INIT_TAG))
cmd = shell.ShellCmd("vgs %s" % vgUuid)
cmd(is_exception=False)
if cmd.return_code == 0:
logger.warn("found vg %s without tag %s" % (vgUuid, INIT_TAG))
return True
raise RetryException("can not find vg %s with or without tag %s" %
(vgUuid, INIT_TAG))
try:
find_vg(cmd.vgUuid)
except RetryException:
logger.debug("can not find vg %s; return success" % cmd.vgUuid)
return jsonobject.dumps(rsp)
except Exception as e:
raise e
@linux.retry(times=3, sleep_time=random.uniform(0.1, 3))
def deactive_lvs_on_vg(vgUuid):
active_lvs = lvm.list_local_active_lvs(vgUuid)
if len(active_lvs) == 0:
return
logger.warn("active lvs %s will be deactivate" % active_lvs)
lvm.deactive_lv(vgUuid)
active_lvs = lvm.list_local_active_lvs(vgUuid)
if len(active_lvs) != 0:
raise RetryException(
"lvs [%s] still active, retry deactive again" % active_lvs)
deactive_lvs_on_vg(cmd.vgUuid)
lvm.clean_vg_exists_host_tags(cmd.vgUuid, cmd.hostUuid, HEARTBEAT_TAG)
lvm.stop_vg_lock(cmd.vgUuid)
if cmd.stopServices:
lvm.quitLockServices()
lvm.clean_lvm_archive_files(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@lock.file_lock(LOCK_FILE)
def add_disk(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
disk = CheckDisk(cmd.diskUuid)
command = shell.ShellCmd("vgs --nolocking %s -otags | grep %s" %
(cmd.vgUuid, INIT_TAG))
command(is_exception=False)
if command.return_code != 0:
self.create_vg_if_not_found(cmd.vgUuid, [disk.get_path()],
cmd.hostUuid, cmd.forceWipe)
else:
lvm.check_gl_lock()
if cmd.forceWipe is True:
lvm.wipe_fs([disk.get_path()], cmd.vgUuid)
lvm.add_pv(cmd.vgUuid, disk.get_path(), DEFAULT_VG_METADATA_SIZE)
rsp = AgentRsp
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def resize_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
install_abs_path = translate_absolute_path_from_install_path(
cmd.installPath)
with lvm.RecursiveOperateLv(install_abs_path, shared=False):
lvm.resize_lv_from_cmd(install_abs_path, cmd.size, cmd)
if not cmd.live:
shell.call("qemu-img resize %s %s" %
(install_abs_path, cmd.size))
ret = linux.qcow2_virtualsize(install_abs_path)
rsp = ResizeVolumeRsp()
rsp.size = ret
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@lock.file_lock(LOCK_FILE)
def create_root_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
template_abs_path_cache = translate_absolute_path_from_install_path(
cmd.templatePathInCache)
install_abs_path = translate_absolute_path_from_install_path(
cmd.installPath)
qcow2_options = self.calc_qcow2_option(self, cmd.qcow2Options, True,
cmd.provisioning)
with lvm.RecursiveOperateLv(template_abs_path_cache,
shared=True,
skip_deactivate_tags=[IMAGE_TAG]):
virtual_size = linux.qcow2_virtualsize(template_abs_path_cache)
if not lvm.lv_exists(install_abs_path):
lvm.create_lv_from_cmd(
install_abs_path, virtual_size, cmd,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(install_abs_path,
shared=False,
delete_when_exception=True):
linux.qcow2_clone_with_option(template_abs_path_cache,
install_abs_path, qcow2_options)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def delete_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if cmd.folder:
raise Exception("not support this operation")
self.do_delete_bits(cmd.path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
def do_delete_bits(self, path):
install_abs_path = translate_absolute_path_from_install_path(path)
if lvm.has_lv_tag(install_abs_path, IMAGE_TAG):
logger.info('deleting lv image: ' + install_abs_path)
lvm.delete_image(install_abs_path, IMAGE_TAG)
else:
logger.info('deleting lv volume: ' + install_abs_path)
lvm.delete_lv(install_abs_path)
@kvmagent.replyerror
def create_template_from_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
volume_abs_path = translate_absolute_path_from_install_path(
cmd.volumePath)
install_abs_path = translate_absolute_path_from_install_path(
cmd.installPath)
if cmd.sharedVolume:
lvm.do_active_lv(volume_abs_path, lvm.LvmlockdLockType.SHARE, True)
with lvm.RecursiveOperateLv(volume_abs_path,
shared=cmd.sharedVolume,
skip_deactivate_tags=[IMAGE_TAG]):
virtual_size = linux.qcow2_virtualsize(volume_abs_path)
total_size = 0
compress = False
for qcow2 in linux.qcow2_get_file_chain(volume_abs_path):
if bash.bash_r("qemu-img check %s | grep compressed" %
volume_abs_path) == 0:
compress = True
total_size += int(lvm.get_lv_size(qcow2))
if total_size > virtual_size:
total_size = virtual_size
if bash.bash_r("qemu-img info --backing-chain %s | grep compress" %
volume_abs_path) == 0:
compress = True
if not lvm.lv_exists(install_abs_path):
lvm.create_lv_from_absolute_path(
install_abs_path, total_size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(install_abs_path,
shared=False,
delete_when_exception=True):
linux.create_template(volume_abs_path, install_abs_path,
compress)
logger.debug(
'successfully created template[%s] from volume[%s]' %
(cmd.installPath, cmd.volumePath))
if cmd.compareQcow2 is True:
logger.debug("comparing qcow2 between %s and %s")
bash.bash_errorout("time qemu-img compare %s %s" %
(volume_abs_path, install_abs_path))
logger.debug("confirmed qcow2 %s and %s are identical" %
(volume_abs_path, install_abs_path))
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@staticmethod
@bash.in_bash
def compare(src, dst):
return bash.bash_r("cmp %s %s" % (src, dst)) == 0
@kvmagent.replyerror
def upload_to_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
install_abs_path = translate_absolute_path_from_install_path(
cmd.primaryStorageInstallPath)
def upload():
if not os.path.exists(cmd.primaryStorageInstallPath):
raise kvmagent.KvmError('cannot find %s' %
cmd.primaryStorageInstallPath)
linux.scp_upload(cmd.hostname, cmd.sshKey,
cmd.primaryStorageInstallPath,
cmd.backupStorageInstallPath, cmd.username,
cmd.sshPort)
with lvm.OperateLv(install_abs_path, shared=True):
upload()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
install_abs_path = translate_absolute_path_from_install_path(
cmd.primaryStorageInstallPath)
self.do_download_from_sftp(cmd, install_abs_path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
def do_download_from_sftp(self, cmd, install_abs_path):
if not lvm.lv_exists(install_abs_path):
size = linux.sftp_get(cmd.hostname, cmd.sshKey,
cmd.backupStorageInstallPath,
install_abs_path, cmd.username, cmd.sshPort,
True)
lvm.create_lv_from_absolute_path(
install_abs_path, size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(install_abs_path,
shared=False,
delete_when_exception=True):
linux.scp_download(cmd.hostname, cmd.sshKey,
cmd.backupStorageInstallPath, install_abs_path,
cmd.username, cmd.sshPort, cmd.bandWidth)
logger.debug('successfully download %s/%s to %s' %
(cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath))
self.do_active_lv(cmd.primaryStorageInstallPath, cmd.lockType, False)
def cancel_download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
install_abs_path = translate_absolute_path_from_install_path(
cmd.primaryStorageInstallPath)
shell.run("pkill -9 -f '%s'" % install_abs_path)
self.do_delete_bits(cmd.primaryStorageInstallPath)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@completetask
def download_from_kvmhost(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
install_abs_path = translate_absolute_path_from_install_path(
cmd.primaryStorageInstallPath)
# todo: assume agent will not restart, maybe need clean
last_task = self.load_and_save_task(req, rsp, os.path.exists,
install_abs_path)
if last_task and last_task.agent_pid == os.getpid():
rsp = self.wait_task_complete(last_task)
return jsonobject.dumps(rsp)
self.do_download_from_sftp(cmd, install_abs_path)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def cancel_download_from_kvmhost(self, req):
return self.cancel_download_from_sftp(req)
@kvmagent.replyerror
def upload_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.upload_to_imagestore(cmd, req)
@kvmagent.replyerror
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
@kvmagent.replyerror
def download_from_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.imagestore_client.download_from_imagestore(
None, cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath)
self.do_active_lv(cmd.primaryStorageInstallPath, cmd.lockType, True)
rsp = AgentRsp()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def revert_volume_from_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RevertVolumeFromSnapshotRsp()
snapshot_abs_path = translate_absolute_path_from_install_path(
cmd.snapshotInstallPath)
qcow2_options = self.calc_qcow2_option(self, cmd.qcow2Options, True,
cmd.provisioning)
new_volume_path = cmd.installPath
if new_volume_path is None or new_volume_path == "":
new_volume_path = "/dev/%s/%s" % (cmd.vgUuid, uuidhelper.uuid())
else:
new_volume_path = translate_absolute_path_from_install_path(
new_volume_path)
with lvm.RecursiveOperateLv(snapshot_abs_path, shared=True):
size = linux.qcow2_virtualsize(snapshot_abs_path)
lvm.create_lv_from_cmd(
new_volume_path, size, cmd,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(new_volume_path,
shared=False,
delete_when_exception=True):
linux.qcow2_clone_with_option(snapshot_abs_path,
new_volume_path, qcow2_options)
size = linux.qcow2_virtualsize(new_volume_path)
rsp.newVolumeInstallPath = new_volume_path
rsp.size = size
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def merge_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = MergeSnapshotRsp()
snapshot_abs_path = translate_absolute_path_from_install_path(
cmd.snapshotInstallPath)
workspace_abs_path = translate_absolute_path_from_install_path(
cmd.workspaceInstallPath)
with lvm.RecursiveOperateLv(snapshot_abs_path, shared=True):
virtual_size = linux.qcow2_virtualsize(snapshot_abs_path)
if not lvm.lv_exists(workspace_abs_path):
lvm.create_lv_from_absolute_path(
workspace_abs_path, virtual_size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(workspace_abs_path,
shared=False,
delete_when_exception=True):
linux.create_template(snapshot_abs_path, workspace_abs_path)
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
workspace_abs_path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
rsp.actualSize = rsp.size
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def offline_merge_snapshots(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = OfflineMergeSnapshotRsp()
src_abs_path = translate_absolute_path_from_install_path(cmd.srcPath)
dst_abs_path = translate_absolute_path_from_install_path(cmd.destPath)
with lvm.RecursiveOperateLv(src_abs_path, shared=True):
virtual_size = linux.qcow2_virtualsize(src_abs_path)
if not lvm.lv_exists(dst_abs_path):
lvm.create_lv_from_absolute_path(
dst_abs_path, virtual_size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.RecursiveOperateLv(dst_abs_path, shared=False):
if not cmd.fullRebase:
linux.qcow2_rebase(src_abs_path, dst_abs_path)
else:
tmp_lv = 'tmp_%s' % uuidhelper.uuid()
tmp_abs_path = os.path.join(os.path.dirname(dst_abs_path),
tmp_lv)
tmp_abs_path = os.path.join(os.path.dirname(dst_abs_path),
tmp_lv)
logger.debug("creating temp lv %s" % tmp_abs_path)
lvm.create_lv_from_absolute_path(
tmp_abs_path, virtual_size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(tmp_abs_path,
shared=False,
delete_when_exception=True):
linux.create_template(dst_abs_path, tmp_abs_path)
lvm.lv_rename(tmp_abs_path,
dst_abs_path,
overwrite=True)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@lock.file_lock(LOCK_FILE)
def create_empty_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
install_abs_path = translate_absolute_path_from_install_path(
cmd.installPath)
if cmd.backingFile:
qcow2_options = self.calc_qcow2_option(self, cmd.qcow2Options,
True, cmd.provisioning)
backing_abs_path = translate_absolute_path_from_install_path(
cmd.backingFile)
with lvm.RecursiveOperateLv(backing_abs_path, shared=True):
virtual_size = linux.qcow2_virtualsize(backing_abs_path)
if not lvm.lv_exists(install_abs_path):
lvm.create_lv_from_cmd(
install_abs_path, virtual_size, cmd,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(install_abs_path,
shared=False,
delete_when_exception=True):
linux.qcow2_create_with_backing_file_and_option(
backing_abs_path, install_abs_path, qcow2_options)
elif not lvm.lv_exists(install_abs_path):
lvm.create_lv_from_cmd(
install_abs_path, cmd.size, cmd,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
if cmd.volumeFormat != 'raw':
qcow2_options = self.calc_qcow2_option(self, cmd.qcow2Options,
False, cmd.provisioning)
with lvm.OperateLv(install_abs_path,
shared=False,
delete_when_exception=True):
linux.qcow2_create_with_option(install_abs_path, cmd.size,
qcow2_options)
linux.qcow2_fill(0, 1048576, install_abs_path)
logger.debug(
'successfully create empty volume[uuid:%s, size:%s] at %s' %
(cmd.volumeUuid, cmd.size, cmd.installPath))
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def convert_image_to_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
install_abs_path = translate_absolute_path_from_install_path(
cmd.primaryStorageInstallPath)
with lvm.OperateLv(install_abs_path, shared=False):
lvm.clean_lv_tag(install_abs_path, IMAGE_TAG)
lvm.add_lv_tag(
install_abs_path,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckBitsRsp()
install_abs_path = translate_absolute_path_from_install_path(cmd.path)
rsp.existing = lvm.lv_exists(install_abs_path)
if cmd.vgUuid is not None and lvm.vg_exists(cmd.vgUuid):
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(
cmd.vgUuid, False)
return jsonobject.dumps(rsp)
def do_active_lv(self,
installPath,
lockType,
recursive,
killProcess=False):
def handle_lv(lockType, fpath):
if lockType > lvm.LvmlockdLockType.NULL:
lvm.active_lv(fpath, lockType == lvm.LvmlockdLockType.SHARE)
else:
try:
lvm.deactive_lv(fpath)
except Exception as e:
if not killProcess:
return
qemus = lvm.find_qemu_for_lv_in_use(fpath)
if len(qemus) == 0:
return
for qemu in qemus:
if qemu.state != "running":
linux.kill_process(qemu.pid)
lvm.deactive_lv(fpath)
install_abs_path = translate_absolute_path_from_install_path(
installPath)
handle_lv(lockType, install_abs_path)
if recursive is False or lockType is lvm.LvmlockdLockType.NULL:
return
while linux.qcow2_get_backing_file(install_abs_path) != "":
install_abs_path = linux.qcow2_get_backing_file(install_abs_path)
if lockType == lvm.LvmlockdLockType.NULL:
handle_lv(lvm.LvmlockdLockType.NULL, install_abs_path)
else:
# activate backing files only in shared mode
handle_lv(lvm.LvmlockdLockType.SHARE, install_abs_path)
@kvmagent.replyerror
def active_lv(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(
cmd.vgUuid, raise_exception=False)
self.do_active_lv(cmd.installPath, cmd.lockType, cmd.recursive,
cmd.killProcess)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetVolumeSizeRsp()
install_abs_path = translate_absolute_path_from_install_path(
cmd.installPath)
with lvm.OperateLv(install_abs_path, shared=True):
rsp.size = linux.qcow2_virtualsize(install_abs_path)
rsp.actualSize = lvm.get_lv_size(install_abs_path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@bash.in_bash
def migrate_volumes(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
for struct in cmd.migrateVolumeStructs:
target_abs_path = translate_absolute_path_from_install_path(
struct.targetInstallPath)
current_abs_path = translate_absolute_path_from_install_path(
struct.currentInstallPath)
with lvm.OperateLv(current_abs_path, shared=True):
lv_size = lvm.get_lv_size(current_abs_path)
if lvm.lv_exists(target_abs_path):
target_ps_uuid = get_primary_storage_uuid_from_install_path(
struct.targetInstallPath)
raise Exception("found %s already exists on ps %s" %
(target_abs_path, target_ps_uuid))
lvm.create_lv_from_absolute_path(
target_abs_path, lvm.getOriginalSize(lv_size),
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
lvm.active_lv(target_abs_path, lvm.LvmlockdLockType.SHARE)
try:
for struct in cmd.migrateVolumeStructs:
target_abs_path = translate_absolute_path_from_install_path(
struct.targetInstallPath)
current_abs_path = translate_absolute_path_from_install_path(
struct.currentInstallPath)
with lvm.OperateLv(current_abs_path, shared=True):
bash.bash_errorout("cp %s %s" %
(current_abs_path, target_abs_path))
for struct in cmd.migrateVolumeStructs:
target_abs_path = translate_absolute_path_from_install_path(
struct.targetInstallPath)
current_abs_path = translate_absolute_path_from_install_path(
struct.currentInstallPath)
with lvm.RecursiveOperateLv(current_abs_path, shared=True):
previous_ps_uuid = get_primary_storage_uuid_from_install_path(
struct.currentInstallPath)
target_ps_uuid = get_primary_storage_uuid_from_install_path(
struct.targetInstallPath)
current_backing_file = linux.qcow2_get_backing_file(
current_abs_path) # type: str
target_backing_file = current_backing_file.replace(
previous_ps_uuid, target_ps_uuid)
if struct.compareQcow2:
logger.debug("comparing qcow2 between %s and %s" %
(current_abs_path, target_abs_path))
if not self.compare(current_abs_path, target_abs_path):
raise Exception(
"qcow2 %s and %s are not identical" %
(current_abs_path, target_abs_path))
logger.debug(
"confirmed qcow2 %s and %s are identical" %
(current_abs_path, target_abs_path))
if current_backing_file is not None and current_backing_file != "":
lvm.do_active_lv(target_backing_file,
lvm.LvmlockdLockType.SHARE, False)
logger.debug("rebase %s to %s" %
(target_abs_path, target_backing_file))
linux.qcow2_rebase_no_check(target_backing_file,
target_abs_path)
except Exception as e:
for struct in cmd.migrateVolumeStructs:
target_abs_path = translate_absolute_path_from_install_path(
struct.targetInstallPath)
if struct.currentInstallPath == struct.targetInstallPath:
logger.debug(
"current install path %s equals target %s, skip to delete"
%
(struct.currentInstallPath, struct.targetInstallPath))
else:
logger.debug("error happened, delete lv %s" %
target_abs_path)
lvm.delete_lv(target_abs_path, False)
raise e
finally:
for struct in cmd.migrateVolumeStructs:
target_abs_path = translate_absolute_path_from_install_path(
struct.targetInstallPath)
lvm.deactive_lv(target_abs_path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@staticmethod
def calc_qcow2_option(self, options, has_backing_file, provisioning=None):
if options is None or options == "":
return " "
if has_backing_file or provisioning == lvm.VolumeProvisioningStrategy.ThinProvisioning:
return re.sub("-o preallocation=\w* ", " ", options)
return options
@kvmagent.replyerror
def get_block_devices(self, req):
rsp = GetBlockDevicesRsp()
rsp.blockDevices = lvm.get_block_devices()
return jsonobject.dumps(rsp)
@kvmagent.replyerror
def get_backing_chain(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetBackingChainRsp()
abs_path = translate_absolute_path_from_install_path(cmd.installPath)
with lvm.RecursiveOperateLv(abs_path,
shared=True,
skip_deactivate_tags=[IMAGE_TAG],
delete_when_exception=False):
rsp.backingChain = linux.qcow2_get_file_chain(abs_path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
@kvmagent.replyerror
@bash.in_bash
def convert_volume_provisioning(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ConvertVolumeProvisioningRsp()
if cmd.provisioningStrategy != "ThinProvisioning":
raise NotImplementedError
abs_path = translate_absolute_path_from_install_path(cmd.installPath)
with lvm.RecursiveOperateLv(abs_path, shared=False):
image_offest = long(
bash.bash_o(
"qemu-img check %s | grep 'Image end offset' | awk -F ': ' '{print $2}'"
% abs_path).strip())
current_size = long(lvm.get_lv_size(abs_path))
virtual_size = linux.qcow2_virtualsize(abs_path)
size = image_offest + cmd.addons[
lvm.thinProvisioningInitializeSize]
if size > current_size:
size = current_size
if size > virtual_size:
size = virtual_size
lvm.resize_lv(abs_path, size, True)
rsp.actualSize = size
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
