def on_main(self, checkpoint, resource, context, parameters, **kwargs):
original_volume_id = resource.id
restore_name = parameters.get('restore_name',
'%s@%s' % (checkpoint.id,
original_volume_id))
restore_description = parameters.get('restore_description', None)
bank_section = checkpoint.get_resource_bank_section(original_volume_id)
cinder_client = ClientFactory.create_client('cinder', context)
glance_client = ClientFactory.create_client('glance', context)
resource_metadata = bank_section.get_object('metadata')
volume_size = int(resource_metadata['volume_size'])
temporary_image = None
try:
temporary_image = self._create_temporary_image(
bank_section, glance_client, original_volume_id
)
self._create_volume_from_image(cinder_client, temporary_image,
restore_name, original_volume_id,
volume_size, restore_description)
finally:
if temporary_image:
try:
glance_client.images.delete(temporary_image.id)
except Exception as e:
LOG.warning('Failed deleting temporary image: '
'%(temporary_image_id)s. '
'Reason: %(reason)s', {
'temporary_image_id': temporary_image.id,
'reason': e
})
LOG.info("Finish restoring volume backup, volume_id: %s.",
original_volume_id)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
original_volume_id = resource.id
restore_name = parameters.get('restore_name',
'%s@%s' % (checkpoint.id,
original_volume_id))
restore_description = parameters.get('restore_description', None)
bank_section = checkpoint.get_resource_bank_section(original_volume_id)
cinder_client = ClientFactory.create_client('cinder', context)
glance_client = ClientFactory.create_client('glance', context)
resource_metadata = bank_section.get_object('metadata')
volume_size = int(resource_metadata['volume_size'])
temporary_image = None
try:
temporary_image = self._create_temporary_image(
bank_section, glance_client, original_volume_id
)
volume_id = self._create_volume_from_image(
cinder_client, temporary_image, restore_name, volume_size,
restore_description
)
# check and update
update_method = partial(
utils.update_resource_restore_result,
kwargs.get('restore'), resource.type, volume_id
)
update_method(constants.RESOURCE_STATUS_RESTORING)
is_success = self._check_create_complete(cinder_client, volume_id)
if is_success:
update_method(constants.RESOURCE_STATUS_AVAILABLE)
kwargs.get("new_resources")[original_volume_id] = volume_id
else:
LOG.error("Restore volume glance backup failed, so delete "
"the temporary volume: volume_id: %s.",
original_volume_id)
cinder_client.volumes.delete(volume_id)
raise exception.CreateResourceFailed(
name="Volume Glance Backup",
reason='Restored Volume is in erroneous state',
resource_id=volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE,
)
finally:
if temporary_image:
try:
glance_client.images.delete(temporary_image.id)
except Exception as e:
LOG.warning('Failed deleting temporary image: '
'%(temporary_image_id)s. '
'Reason: %(reason)s', {
'temporary_image_id': temporary_image.id,
'reason': e
})
LOG.info("Finish restoring volume backup, volume_id: %s.",
original_volume_id)
def on_complete(self, checkpoint, resource, context, parameters, **kwargs):
original_server_id = resource.id
LOG.info("Restoring server backup, server_id: %s.", original_server_id)
update_method = None
try:
resource_definition = checkpoint.get_resource_bank_section(
original_server_id).get_object("metadata")
nova_client = ClientFactory.create_client("nova", context)
new_resources = kwargs.get("new_resources")
# restore server instance
restore_net_id = parameters.get("restore_net_id", None)
restore_flavor_id = parameters.get("restore_flavor_id", None)
if restore_flavor_id:
resource_definition["server_metadata"]['flavor'] = (
restore_flavor_id)
new_server_id = self._restore_server_instance(
nova_client, new_resources, original_server_id,
parameters.get("restore_name", "karbor-restore-server"),
restore_net_id, resource_definition)
update_method = partial(utils.update_resource_restore_result,
kwargs.get('restore'), resource.type,
new_server_id)
update_method(constants.RESOURCE_STATUS_RESTORING)
self._wait_server_to_active(nova_client, new_server_id)
# restore volume attachment
self._restore_volume_attachment(
nova_client, ClientFactory.create_client("cinder", context),
new_resources, new_server_id, resource_definition)
# restore floating ip association
self._restore_floating_association(nova_client, new_server_id,
resource_definition)
new_resources[original_server_id] = new_server_id
update_method(constants.RESOURCE_STATUS_AVAILABLE)
LOG.info("Finish restore server, server_id: %s.",
original_server_id)
except Exception as e:
if update_method:
update_method(constants.RESOURCE_STATUS_ERROR, str(e))
LOG.exception("Restore server backup failed, server_id: %s.",
original_server_id)
raise exception.RestoreResourceFailed(
name="Server Backup",
reason=e,
resource_id=original_server_id,
resource_type=constants.SERVER_RESOURCE_TYPE)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
instance_id = resource.id
bank_section = checkpoint.get_resource_bank_section(instance_id)
trove_client = ClientFactory.create_client('trove', context)
LOG.info('creating database instance backup, instance_id: %s',
instance_id)
bank_section.update_object('status',
constants.RESOURCE_STATUS_PROTECTING)
instance_info = trove_client.instances.get(instance_id)
if instance_info.status != "ACTIVE":
is_success = utils.status_poll(
partial(get_database_instance_status, trove_client,
instance_id),
interval=self._interval,
success_statuses={'ACTIVE'},
failure_statuses=DATABASE_FAILURE_STATUSES,
ignore_statuses=DATABASE_IGNORE_STATUSES,
)
if not is_success:
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Database instance Backup",
reason='Database instance is in a error status.',
resource_id=instance_id,
resource_type=constants.DATABASE_RESOURCE_TYPE,
)
resource_metadata = {
'instance_id': instance_id,
'datastore': instance_info.datastore,
'flavor': instance_info.flavor,
'size': instance_info.volume['size'],
}
backup_name = parameters.get('backup_name', 'backup%s' % (instance_id))
description = parameters.get('description', None)
try:
backup_id = self._create_backup(trove_client, instance_id,
backup_name, description)
except exception.CreateResourceFailed as e:
LOG.error(
'Error creating backup (instance_id: %(instance_id)s '
': %(reason)s', {
'instance_id': instance_id,
'reason': e
})
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise
resource_metadata['backup_id'] = backup_id
bank_section.update_object('metadata', resource_metadata)
bank_section.update_object('status',
constants.RESOURCE_STATUS_AVAILABLE)
LOG.info(
'Backup database instance (instance_id: %(instance_id)s '
'backup_id: %(backup_id)s ) successfully', {
'instance_id': instance_id,
'backup_id': backup_id
})
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
volume_id = resource.id
bank_section = checkpoint.get_resource_bank_section(volume_id)
cinder_client = ClientFactory.create_client('cinder', context)
LOG.info('Creating volume snapshot, volume_id: %s', volume_id)
bank_section.update_object('status',
constants.RESOURCE_STATUS_PROTECTING)
volume_info = cinder_client.volumes.get(volume_id)
is_success = utils.status_poll(
partial(get_volume_status, cinder_client, volume_id),
interval=self._interval,
success_statuses={
'available', 'in-use', 'error_extending', 'error_restoring'
},
failure_statuses=VOLUME_FAILURE_STATUSES,
ignore_statuses=VOLUME_IGNORE_STATUSES,
)
if not is_success:
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Volume Snapshot",
reason='Volume is in a error status.',
resource_id=volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE,
)
resource_metadata = {'volume_id': volume_id, 'size': volume_info.size}
snapshot_name = parameters.get('snapshot_name', None)
description = parameters.get('description', None)
force = parameters.get('force', False)
try:
snapshot_id = self._create_snapshot(cinder_client, volume_id,
snapshot_name, description,
force)
except Exception as e:
LOG.error(
'Error creating snapshot (volume_id: %(volume_id)s '
': %(reason)s', {
'volume_id': volume_id,
'reason': e
})
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Volume Snapshot",
reason=e,
resource_id=volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE,
)
resource_metadata['snapshot_id'] = snapshot_id
bank_section.update_object('metadata', resource_metadata)
bank_section.update_object('status',
constants.RESOURCE_STATUS_AVAILABLE)
LOG.info(
'Snapshot volume (volume_id: %(volume_id)s snapshot_id: '
'%(snapshot_id)s ) successfully', {
'volume_id': volume_id,
'snapshot_id': snapshot_id
})
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
resource_id = resource.id
bank_section = checkpoint.get_resource_bank_section(resource_id)
resource_metadata = bank_section.get_object('metadata')
cinder_client = ClientFactory.create_client('cinder', context)
LOG.info('Verifying the volume backup, volume_id: %s', resource_id)
update_method = partial(
utils.update_resource_verify_result,
kwargs.get('verify'), resource.type, resource_id)
backup_id = resource_metadata['backup_id']
try:
volume_backup = cinder_client.backups.get(backup_id)
backup_status = volume_backup.status
except Exception as ex:
LOG.error('Error get volume backup (backup_id: %(backup_id)s): '
'%(reason)s', {'backup_id': backup_id, 'reason': ex})
reason = 'Error getting volume backup.'
update_method(constants.RESOURCE_STATUS_ERROR, reason)
raise
if backup_status == 'available':
update_method(constants.RESOURCE_STATUS_AVAILABLE)
else:
reason = (
'The status of volume backup status is %s.' % backup_status)
update_method(backup_status, reason)
raise exception.VerifyResourceFailed(
name="Volume Backup",
reason=reason,
resource_id=resource_id,
resource_type=resource.type
)
def on_prepare_finish(self, checkpoint, resource, context, parameters,
**kwargs):
volume_id = resource.id
if not self._backup_from_snapshot:
LOG.info(
'Skipping taking snapshot of volume %s - backing up '
'directly', volume_id)
return
LOG.info('Taking snapshot of volume %s', volume_id)
bank_section = checkpoint.get_resource_bank_section(volume_id)
bank_section.update_object('status',
constants.RESOURCE_STATUS_PROTECTING)
cinder_client = ClientFactory.create_client('cinder', context)
try:
self.snapshot_id = self._create_snapshot(cinder_client, volume_id)
except Exception:
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Volume Backup",
reason='Error creating snapshot for volume',
resource_id=volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE,
)
def _get_dependent_resources_by_server(self, context, parent_resource):
try:
# get metadata about network from neutron
net_client = self._neutron_client(context)
network_infos = net_client.list_networks().get('networks')
neutron_networks = {network["id"] for network in network_infos}
# get interface info from server
nova_networks = set()
serverid = parent_resource.id
nova_client = ClientFactory.create_client("nova", context)
interface_list = nova_client.servers.interface_list(serverid)
# check net_id in interface
for iface in interface_list:
net_id = iface.net_id
if net_id not in nova_networks:
nova_networks.add(net_id)
# get the exsited networks
valid_networks = nova_networks.intersection(neutron_networks)
if valid_networks:
return [
resource.Resource(type=self._SUPPORT_RESOURCE_TYPE,
id=self._get_network_id(),
name="Network Topology")
]
return []
except Exception as e:
LOG.exception("List all interfaces from nova failed.")
raise exception.ListProtectableResourceFailed(
type=self._SUPPORT_RESOURCE_TYPE, reason=six.text_type(e))
def setUp(self):
super(GlanceProtectionPluginTest, self).setUp()
self.plugin = GlanceProtectionPlugin()
cfg.CONF.set_default("glance_endpoint", "http://127.0.0.1:9292", "glance_client")
self.cntxt = RequestContext(user_id="admin", project_id="abcd", auth_token="efgh")
self.glance_client = ClientFactory.create_client("glance", self.cntxt)
self.checkpoint = CheckpointCollection()
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
original_instance_id = resource.id
bank_section = checkpoint.get_resource_bank_section(
original_instance_id)
trove_client = ClientFactory.create_client('trove', context)
resource_metadata = bank_section.get_object('metadata')
restore_name = parameters.get(
'restore_name', '%s@%s' % (checkpoint.id, original_instance_id))
flavor = resource_metadata['flavor']
size = resource_metadata['size']
backup_id = resource_metadata['backup_id']
restore = kwargs.get('restore')
LOG.info(
"Restoring a database instance from backup, "
"original_instance_id: %s.", original_instance_id)
try:
instance_info = trove_client.instances.create(
restore_name,
flavor["id"],
volume={"size": size},
restorePoint={"backupRef": backup_id})
is_success = utils.status_poll(
partial(get_database_instance_status, trove_client,
instance_info.id),
interval=self._interval,
success_statuses={'ACTIVE'},
failure_statuses=DATABASE_FAILURE_STATUSES,
ignore_statuses=DATABASE_IGNORE_STATUSES)
if is_success is not True:
LOG.error(
'The status of database instance is '
'invalid. status:%s', instance_info.status)
restore.update_resource_status(
constants.DATABASE_RESOURCE_TYPE, instance_info.id,
instance_info.status, "Invalid status.")
restore.save()
raise exception.RestoreResourceFailed(
name="Database instance Backup",
reason="Invalid status.",
resource_id=original_instance_id,
resource_type=constants.DATABASE_RESOURCE_TYPE)
restore.update_resource_status(constants.DATABASE_RESOURCE_TYPE,
instance_info.id,
instance_info.status)
restore.save()
except Exception as e:
LOG.error(
"Restore Database instance from backup "
"failed, instance_id: %s.", original_instance_id)
raise exception.RestoreResourceFailed(
name="Database instance Backup",
reason=e,
resource_id=original_instance_id,
resource_type=constants.DATABASE_RESOURCE_TYPE)
LOG.info(
"Finish restoring a Database instance from backup,"
"instance_id: %s.", original_instance_id)
def _create_server_snapshot(self, context, glance_client, parameters,
resource, resource_definition, resource_id):
server_id = resource.extra_info.get('server_id')
resource_definition['resource_id'] = resource_id
resource_definition['server_id'] = server_id
nova_client = ClientFactory.create_client('nova', context)
is_success = utils.status_poll(
partial(get_server_status, nova_client, server_id),
interval=self._interval,
success_statuses={'ACTIVE', 'STOPPED', 'SUSPENDED', 'PAUSED'},
failure_statuses={
'DELETED', 'ERROR', 'RESIZED', 'SHELVED', 'SHELVED_OFFLOADED',
'SOFT_DELETED', 'RESCUED', 'not-found'
},
ignore_statuses={'BUILDING'},
)
if not is_success:
raise exception.CreateResourceFailed(
name="Image Backup",
reason='The parent server of the image is not in valid'
' status',
resource_id=resource_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
temp_image_name = 'Temp_image_name_for_karbor' + server_id
try:
image_uuid = nova_client.servers.create_image(
server_id, temp_image_name, parameters)
except Exception as e:
msg = "Failed to create the server snapshot: %s" % e
LOG.exception(msg)
raise exception.CreateResourceFailed(
name="Image Backup",
reason=msg,
resource_id=resource_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
else:
is_success = utils.status_poll(
partial(get_image_status, glance_client, image_uuid),
interval=self._interval,
success_statuses={'active'},
failure_statuses={
'killed', 'deleted', 'pending_delete', 'deactivated'
},
ignore_statuses={'queued', 'saving', 'uploading'})
if not is_success:
msg = "Image has been created, but fail to become " \
"active, so delete it and raise exception."
LOG.error(msg)
glance_client.images.delete(image_uuid)
image_uuid = None
if not image_uuid:
raise exception.CreateResourceFailed(
name="Image Backup",
reason="Create parent server snapshot failed.",
resource_id=resource_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
return image_uuid
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
pod_id = resource.id
pod_name = resource.name
bank_section = checkpoint.get_resource_bank_section(pod_id)
k8s_client = ClientFactory.create_client("k8s", context)
resource_definition = {"resource_id": pod_id}
LOG.info("Creating pod backup, id: %(pod_id)s) name: "
"%(pod_name)s.", {
"pod_id": pod_id,
"pod_name": pod_name
})
try:
bank_section.update_object("status",
constants.RESOURCE_STATUS_PROTECTING)
# get metadata about pod
pod_namespace, k8s_pod_name = pod_name.split(":")
pod = k8s_client.read_namespaced_pod(k8s_pod_name, pod_namespace)
resource_definition["resource_name"] = pod_name
resource_definition["namespace"] = pod_namespace
mounted_volumes_list = self._get_mounted_volumes(
k8s_client, pod, pod_namespace)
containers_list = self._get_containers(pod)
# save all pod's metadata
pod_metadata = {
'apiVersion': pod.api_version,
'kind': pod.kind,
'metadata': {
'labels': pod.metadata.labels,
'name': pod.metadata.name,
'namespace': pod.metadata.namespace,
},
'spec': {
'containers': containers_list,
'volumes': mounted_volumes_list,
'restartPolicy': pod.spec.restart_policy
}
}
resource_definition["pod_metadata"] = pod_metadata
LOG.debug("Creating pod backup, pod_metadata: %s.", pod_metadata)
bank_section.update_object("metadata", resource_definition)
bank_section.update_object("status",
constants.RESOURCE_STATUS_AVAILABLE)
LOG.info("Finish backup pod, pod_id: %s.", pod_id)
except Exception as err:
LOG.exception("Create pod backup failed, pod_id: %s.", pod_id)
bank_section.update_object("status",
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Pod Backup",
reason=err,
resource_id=pod_id,
resource_type=constants.POD_RESOURCE_TYPE)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
original_share_id = resource.id
bank_section = checkpoint.get_resource_bank_section(original_share_id)
manila_client = ClientFactory.create_client('manila', context)
resource_metadata = bank_section.get_object('metadata')
restore_name = parameters.get(
'restore_name', '%s@%s' % (checkpoint.id, original_share_id))
restore_description = parameters.get('restore_description', None)
snapshot_id = resource_metadata['snapshot_id']
share_proto = resource_metadata['share_proto']
size = resource_metadata['size']
share_type = resource_metadata['share_type']
share_network_id = resource_metadata['share_network_id']
restore = kwargs.get('restore')
LOG.info("Restoring a share from snapshot, "
"original_share_id: %s.", original_share_id)
try:
share = manila_client.shares.create(
share_proto,
size,
snapshot_id=snapshot_id,
name=restore_name,
description=restore_description,
share_network=share_network_id,
share_type=share_type)
is_success = utils.status_poll(
partial(get_share_status, manila_client, share.id),
interval=self._interval,
success_statuses={'available'},
failure_statuses=SHARE_FAILURE_STATUSES,
ignore_statuses=SHARE_IGNORE_STATUSES)
if is_success is not True:
LOG.error('The status of share is invalid. status:%s',
share.status)
restore.update_resource_status(constants.SHARE_RESOURCE_TYPE,
share.id, share.status,
"Invalid status.")
restore.save()
raise exception.RestoreResourceFailed(
name="Share Snapshot",
reason="Invalid status.",
resource_id=original_share_id,
resource_type=constants.SHARE_RESOURCE_TYPE)
restore.update_resource_status(constants.SHARE_RESOURCE_TYPE,
share.id, share.status)
restore.save()
except Exception as e:
LOG.error("Restore share from snapshot failed, share_id: %s.",
original_share_id)
raise exception.RestoreResourceFailed(
name="Share Snapshot",
reason=e,
resource_id=original_share_id,
resource_type=constants.SHARE_RESOURCE_TYPE)
LOG.info("Finish restoring a share from snapshot, share_id: %s.",
original_share_id)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
original_volume_id = resource.id
bank_section = checkpoint.get_resource_bank_section(original_volume_id)
cinder_client = ClientFactory.create_client('cinder', context)
resource_metadata = bank_section.get_object('metadata')
restore_name = parameters.get(
'restore_name',
'volume-%s@%s' % (checkpoint.id, original_volume_id))
restore_description = parameters.get('restore_description', None)
snapshot_id = resource_metadata['snapshot_id']
size = resource_metadata['size']
restore = kwargs.get('restore')
LOG.info("Restoring a volume from snapshot, "
"original_volume_id: %s", original_volume_id)
try:
volume = cinder_client.volumes.create(
size,
snapshot_id=snapshot_id,
name=restore_name,
description=restore_description)
is_success = utils.status_poll(
partial(get_volume_status, cinder_client, volume.id),
interval=self._interval,
success_statuses={
'available', 'in-use', 'error_extending', 'error_restoring'
},
failure_statuses=VOLUME_FAILURE_STATUSES,
ignore_statuses=VOLUME_IGNORE_STATUSES,
)
volume = cinder_client.volumes.get(volume.id)
if is_success is not True:
LOG.error('The status of volume is invalid. status:%s',
volume.status)
reason = 'Invalid status: %s' % volume.status
restore.update_resource_status(constants.VOLUME_RESOURCE_TYPE,
volume.id, volume.status,
reason)
restore.save()
raise exception.RestoreResourceFailed(
name="Volume Snapshot",
resource_id=original_volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE)
restore.update_resource_status(constants.VOLUME_RESOURCE_TYPE,
volume.id, volume.status)
restore.save()
except Exception as e:
LOG.error("Restore volume from snapshot failed, volume_id: %s",
original_volume_id)
raise exception.RestoreResourceFailed(
name="Volume Snapshot",
reason=e,
resource_id=original_volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE)
LOG.info("Finish restoring a volume from snapshot, volume_id: %s",
original_volume_id)
def setUp(self):
super(GlanceProtectionPluginTest, self).setUp()
self.plugin = GlanceProtectionPlugin()
cfg.CONF.set_default('glance_endpoint', 'http://127.0.0.1:9292',
'glance_client')
self.cntxt = RequestContext(user_id='admin',
project_id='abcd',
auth_token='efgh')
self.glance_client = ClientFactory.create_client("glance", self.cntxt)
self.checkpoint = CheckpointCollection()
def setUp(self):
super(CinderProtectionPluginTest, self).setUp()
self.plugin = CinderProtectionPlugin()
cfg.CONF.set_default('cinder_endpoint', 'http://127.0.0.1:8776/v2',
'cinder_client')
self.cntxt = RequestContext(user_id='admin',
project_id='abcd',
auth_token='efgh')
self.cinder_client = ClientFactory.create_client("cinder", self.cntxt)
self.checkpoint = FakeCheckpoint()
def setUp(self):
super(CinderProtectionPluginTest, self).setUp()
self.plugin = CinderProtectionPlugin()
cfg.CONF.set_default('cinder_endpoint',
'http://127.0.0.1:8776/v2',
'cinder_client')
self.cntxt = RequestContext(user_id='admin',
project_id='abcd',
auth_token='efgh')
self.cinder_client = ClientFactory.create_client("cinder", self.cntxt)
self.checkpoint = FakeCheckpoint()
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
image_id = resource.id
bank_section = checkpoint.get_resource_bank_section(image_id)
resource_definition = {"resource_id": image_id}
glance_client = ClientFactory.create_client('glance', context)
LOG.info("Creating image backup, image_id: %s.", image_id)
try:
bank_section.update_object("status",
constants.RESOURCE_STATUS_PROTECTING)
image_info = glance_client.images.get(image_id)
if image_info.status != "active":
is_success = utils.status_poll(
partial(get_image_status, glance_client, image_info.id),
interval=self._interval,
success_statuses={'active'},
ignore_statuses={'queued', 'saving'},
failure_statuses={
'killed', 'deleted', 'pending_delete', 'deactivated',
'NotFound'
})
if is_success is not True:
LOG.error("The status of image (id: %s) is invalid.",
image_id)
raise exception.CreateResourceFailed(
name="Image Backup",
reason="The status of image is invalid.",
resource_id=image_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
image_metadata = {
"disk_format": image_info.disk_format,
"container_format": image_info.container_format,
"checksum": image_info.checksum
}
resource_definition["image_metadata"] = image_metadata
bank_section.update_object("metadata", resource_definition)
except Exception as err:
LOG.error("Create image backup failed, image_id: %s.", image_id)
bank_section.update_object("status",
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Image Backup",
reason=err,
resource_id=image_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
self._create_backup(glance_client, bank_section, image_id)
def get_flow(context, workflow_engine, operation_type, checkpoint, provider,
restore, restore_auth):
target = restore.get('restore_target', None)
# TODO(luobin): create a heat_client
# Initialize a heat client using current login tenant when the
# restore_target and restore_auth is not provided.
if target is not None:
username = None
password = None
if restore_auth is not None:
auth_type = restore_auth.get("type", None)
if auth_type == "password":
username = restore_auth["username"]
password = restore_auth["password"]
kwargs = {
"auth_url": target,
"username": username,
"password": password
}
else:
kwargs = {}
heat_client = ClientFactory.create_client("heat",
context=context,
**kwargs)
# TODO(luobin): create a heat_template
heat_template = HeatTemplate()
ctx = {
'context': context,
'checkpoint': checkpoint,
'workflow_engine': workflow_engine,
'operation_type': operation_type,
'restore': restore,
'heat_template': heat_template
}
flow_name = "create_restoration_" + checkpoint.id
restoration_flow = workflow_engine.build_flow(flow_name, 'linear')
result = provider.build_task_flow(ctx)
resource_flow = result.get('task_flow')
workflow_engine.add_tasks(
restoration_flow, resource_flow,
CreateStackTask(heat_client, heat_template),
SyncStackStatusTask(checkpoint, heat_client, restore))
flow_engine = workflow_engine.get_engine(restoration_flow)
return flow_engine
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
resource_id = resource.id
bank_section = checkpoint.get_resource_bank_section(resource_id)
resource_metadata = bank_section.get_object('metadata')
cinder_client = ClientFactory.create_client('cinder', context)
# create volume
volume_property = {
'name':
parameters.get('restore_name',
'%s@%s' % (checkpoint.id, resource_id))
}
if 'restore_description' in parameters:
volume_property['description'] = parameters['restore_description']
backup_id = resource_metadata['backup_id']
try:
volume_id = cinder_client.restores.restore(backup_id).volume_id
cinder_client.volumes.update(volume_id, **volume_property)
except Exception as ex:
LOG.error(
'Error creating volume (backup_id: %(backup_id)s): '
'%(reason)s', {
'backup_id': backup_id,
'reason': ex
})
raise
# check and update status
update_method = partial(utils.update_resource_restore_result,
kwargs.get('restore'), resource.type,
volume_id)
update_method(constants.RESOURCE_STATUS_RESTORING)
is_success = self._check_create_complete(cinder_client, volume_id)
if is_success:
update_method(constants.RESOURCE_STATUS_AVAILABLE)
kwargs.get("new_resources")[resource_id] = volume_id
else:
reason = 'Error creating volume'
update_method(constants.RESOURCE_STATUS_ERROR, reason)
raise exception.RestoreResourceFailed(name="Volume Backup",
reason=reason,
resource_id=resource_id,
resource_type=resource.type)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
original_image_id = resource.id
name = parameters.get("restore_name", "karbor-restore-image")
LOG.info("Restoring image backup, image_id: %s.", original_image_id)
glance_client = ClientFactory.create_client('glance', context)
bank_section = checkpoint.get_resource_bank_section(original_image_id)
image_info = None
try:
image_info = utils.restore_image_from_bank(glance_client,
bank_section, name)
if image_info.status != "active":
is_success = utils.status_poll(
partial(get_image_status, glance_client, image_info.id),
interval=self._interval,
success_statuses={'active'},
ignore_statuses={'queued', 'saving'},
failure_statuses={
'killed', 'deleted', 'pending_delete', 'deactivated',
'not-found'
})
if is_success is not True:
LOG.error('The status of image is invalid. status:%s',
image_info.status)
raise exception.RestoreResourceFailed(
name="Image Backup",
resource_id=image_info.id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
kwargs.get("new_resources")[original_image_id] = image_info.id
except Exception as e:
LOG.error("Restore image backup failed, image_id: %s.",
original_image_id)
if image_info is not None and hasattr(image_info, 'id'):
LOG.info("Delete the failed image, image_id: %s.",
image_info.id)
glance_client.images.delete(image_info.id)
raise exception.RestoreResourceFailed(
name="Image Backup",
reason=e,
resource_id=original_image_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
LOG.info("Finish restoring image backup, image_id: %s.",
original_image_id)
def on_complete(self, checkpoint, resource, context, parameters, **kwargs):
original_pod_id = resource.id
LOG.info("Restoring pod backup, pod_id: %s.", original_pod_id)
update_method = None
try:
resource_definition = checkpoint.get_resource_bank_section(
original_pod_id).get_object("metadata")
LOG.debug("Restoring pod backup, metadata: %s.",
resource_definition)
k8s_client = ClientFactory.create_client("k8s", context)
new_resources = kwargs.get("new_resources")
# restore pod
new_pod_name = self._restore_pod_instance(
k8s_client, new_resources, original_pod_id,
parameters.get(
"restore_name",
"karbor-restored-pod-%s" % uuidutils.generate_uuid()),
resource_definition)
update_method = partial(utils.update_resource_restore_result,
kwargs.get('restore'), resource.type,
new_pod_name)
update_method(constants.RESOURCE_STATUS_RESTORING)
pod_namespace = resource_definition["namespace"]
self._wait_pod_to_running(k8s_client, new_pod_name, pod_namespace)
new_resources[original_pod_id] = new_pod_name
update_method(constants.RESOURCE_STATUS_AVAILABLE)
LOG.info("Finish restore pod, pod_id: %s.", original_pod_id)
except Exception as e:
if update_method:
update_method(constants.RESOURCE_STATUS_ERROR, str(e))
LOG.exception("Restore pod backup failed, pod_id: %s.",
original_pod_id)
raise exception.RestoreResourceFailed(
name="Pod Backup",
reason=e,
resource_id=original_pod_id,
resource_type=constants.POD_RESOURCE_TYPE)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
network_id = get_network_id(context)
backup_name = kwargs.get("backup_name", "karbor network backup")
bank_section = checkpoint.get_resource_bank_section(network_id)
neutron_client = ClientFactory.create_client("neutron", context)
resource_definition = {"resource_id": network_id}
resource_definition["backup_name"] = backup_name
resource_definition["network_metadata"] = (
self._get_resources_by_network(context, neutron_client))
resource_definition["subnet_metadata"] = (
self._get_resources_by_subnet(context, neutron_client))
resource_definition["port_metadata"] = (
self._get_resources_by_port(context, neutron_client))
resource_definition["router_metadata"] = (
self._get_resources_by_router(context, neutron_client))
resource_definition["security-group_metadata"] = (
self._get_resources_by_security_group(context, neutron_client))
try:
bank_section.update_object("status",
constants.RESOURCE_STATUS_PROTECTING)
# write resource_definition in bank
bank_section.update_object("metadata", resource_definition)
# update resource_definition backup_status
bank_section.update_object("status",
constants.CHECKPOINT_STATUS_AVAILABLE)
LOG.info("finish backup network, network_id: %s.", network_id)
except Exception as err:
# update resource_definition backup_status
LOG.error("create backup failed, network_id: %s.", network_id)
bank_section.update_object("status",
constants.CHECKPOINT_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Network Backup",
reason=err,
resource_id=network_id,
resource_type=self._SUPPORT_RESOURCE_TYPES)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
resource_id = resource.id
bank_section = checkpoint.get_resource_bank_section(resource_id)
backup_id = None
try:
bank_section.update_object('status',
constants.RESOURCE_STATUS_DELETING)
resource_metadata = bank_section.get_object('metadata')
backup_id = resource_metadata['backup_id']
trove_client = ClientFactory.create_client('trove', context)
try:
backup = trove_client.backups.get(backup_id)
trove_client.backups.delete(backup)
except trove_exc.NotFound:
LOG.info('Backup id: %s not found. Assuming deleted',
backup_id)
is_success = utils.status_poll(
partial(get_backup_status, trove_client, backup_id),
interval=self._interval,
success_statuses={'not-found'},
failure_statuses={'FAILED', 'DELETE_FAILED'},
ignore_statuses={'COMPLETED'},
ignore_unexpected=True
)
if not is_success:
raise exception.NotFound()
bank_section.delete_object('metadata')
bank_section.update_object('status',
constants.RESOURCE_STATUS_DELETED)
except Exception as e:
LOG.error('Delete Database instance Backup failed, backup_id: %s',
backup_id)
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise exception.DeleteResourceFailed(
name="Database instance Backup",
reason=six.text_type(e),
resource_id=resource_id,
resource_type=constants.DATABASE_RESOURCE_TYPE
)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
resource_id = resource.id
bank_section = checkpoint.get_resource_bank_section(resource_id)
snapshot_id = None
try:
bank_section.update_object('status',
constants.RESOURCE_STATUS_DELETING)
resource_metadata = bank_section.get_object('metadata')
snapshot_id = resource_metadata['snapshot_id']
cinder_client = ClientFactory.create_client('cinder', context)
try:
snapshot = cinder_client.volume_snapshots.get(snapshot_id)
cinder_client.volume_snapshots.delete(snapshot)
except cinder_exc.NotFound:
LOG.info('Snapshot id: %s not found. Assuming deleted',
snapshot_id)
is_success = utils.status_poll(
partial(get_snapshot_status, cinder_client, snapshot_id),
interval=self._interval,
success_statuses={'deleted', 'not-found'},
failure_statuses={'error', 'error_deleting'},
ignore_statuses={'deleting'},
ignore_unexpected=True
)
if not is_success:
raise exception.NotFound()
bank_section.delete_object('metadata')
bank_section.update_object('status',
constants.RESOURCE_STATUS_DELETED)
except Exception as e:
LOG.error('Delete volume snapshot failed, snapshot_id: %s',
snapshot_id)
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise exception.DeleteResourceFailed(
name="Volume Snapshot",
reason=six.text_type(e),
resource_id=resource_id,
resource_type=constants.VOLUME_RESOURCE_TYPE
)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
original_volume_id = resource.id
bank_section = checkpoint.get_resource_bank_section(original_volume_id)
cinder_client = ClientFactory.create_client('cinder', context)
resource_metadata = bank_section.get_object('metadata')
LOG.info('Verifying the volume snapshot, volume_id: %s',
original_volume_id)
update_method = partial(utils.update_resource_verify_result,
kwargs.get('verify'), resource.type,
original_volume_id)
snapshot_id = resource_metadata['snapshot_id']
try:
volume_snapshot = cinder_client.volume_snapshots.get(snapshot_id)
snapshot_status = volume_snapshot.status
except Exception as ex:
LOG.error(
'Getting volume snapshot (snapshot_id: %(snapshot_id)s):'
'%(reason)s fails', {
'snapshot_id': snapshot_id,
'reason': ex
})
reason = 'Getting volume backup fails.'
update_method(constants.RESOURCE_STATUS_ERROR, reason)
raise
if snapshot_status == 'available':
update_method(constants.RESOURCE_STATUS_AVAILABLE)
else:
reason = ('The status of volume snapshot status is %s.' %
snapshot_status)
update_method(snapshot_status, reason)
raise exception.VerifyResourceFailed(
name="Volume snapshot",
reason=reason,
resource_id=original_volume_id,
resource_type=resource.type)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
resource_id = resource.id
bank_section = checkpoint.get_resource_bank_section(resource_id)
backup_id = None
try:
bank_section.update_object('status',
constants.RESOURCE_STATUS_DELETING)
resource_metadata = bank_section.get_object('metadata')
backup_id = resource_metadata['backup_id']
cinder_client = ClientFactory.create_client('cinder', context)
try:
backup = cinder_client.backups.get(backup_id)
cinder_client.backups.delete(backup)
except cinder_exc.NotFound:
LOG.info('Backup id: %s not found. Assuming deleted',
backup_id)
is_success = status_poll(
partial(get_backup_status, cinder_client, backup_id),
interval=self._interval,
success_statuses={'deleted', 'not-found'},
failure_statuses={'error', 'error_deleting'},
ignore_statuses={'deleting'},
)
if not is_success:
raise exception.NotFound()
bank_section.delete_object('metadata')
bank_section.update_object('status',
constants.RESOURCE_STATUS_DELETED)
except Exception as e:
LOG.error('delete volume backup failed, backup_id: %s', backup_id)
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise exception.DeleteBackupFailed(
reason=six.text_type(e),
resource_id=resource_id,
resource_type=constants.VOLUME_RESOURCE_TYPE)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
original_instance_id = resource.id
bank_section = checkpoint.get_resource_bank_section(
original_instance_id)
trove_client = ClientFactory.create_client('trove', context)
resource_metadata = bank_section.get_object('metadata')
LOG.info('Verifying the database instance, instance_id: %s',
original_instance_id)
update_method = partial(
utils.update_resource_verify_result,
kwargs.get('verify'), resource.type, original_instance_id)
backup_id = resource_metadata['backup_id']
try:
instance_backup = trove_client.backups.get(backup_id)
backup_status = instance_backup.status
except Exception as ex:
LOG.error('Getting database backup (backup_id: %(backup_id)s):'
'%(reason)s fails',
{'backup_id': backup_id, 'reason': ex})
reason = 'Getting database backup fails.'
update_method(constants.RESOURCE_STATUS_ERROR, reason)
raise
if backup_status == 'COMPLETED':
update_method(constants.RESOURCE_STATUS_AVAILABLE)
else:
reason = ('The status of database backup status is %s.'
% backup_status)
update_method(backup_status, reason)
raise exception.VerifyResourceFailed(
name="Database backup",
reason=reason,
resource_id=original_instance_id,
resource_type=resource.type)
def _k8s_client(self, context):
self._k8s_client_instance = ClientFactory.create_client("k8s", context)
return self._k8s_client_instance
def _client(self, context):
self._client_instance = ClientFactory.create_client("cinder", context)
return self._client_instance
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
server_id = resource.id
bank_section = checkpoint.get_resource_bank_section(server_id)
nova_client = ClientFactory.create_client("nova", context)
cinder_client = ClientFactory.create_client("cinder", context)
neutron_client = ClientFactory.create_client("neutron", context)
resource_definition = {"resource_id": server_id}
# get dependent resources
server_child_nodes = []
resources = checkpoint.resource_graph
for resource_node in resources:
resource = resource_node.value
if resource.id == server_id:
server_child_nodes = resource_node.child_nodes
LOG.info("Creating server backup, server_id: %s. ", server_id)
try:
bank_section.update_object("status",
constants.RESOURCE_STATUS_PROTECTING)
# get attach_metadata about volume
attach_metadata = {}
for server_child_node in server_child_nodes:
child_resource = server_child_node.value
if child_resource.type == constants.VOLUME_RESOURCE_TYPE:
volume = cinder_client.volumes.get(child_resource.id)
attachments = getattr(volume, "attachments")
for attachment in attachments:
if attachment["server_id"] == server_id:
attachment["bootable"] = getattr(
volume, "bootable")
attach_metadata[child_resource.id] = attachment
resource_definition["attach_metadata"] = attach_metadata
# get metadata about AZ
server = nova_client.servers.get(server_id)
availability_zone = getattr(server, "OS-EXT-AZ:availability_zone")
# get metadata about network, flavor, key_name, security_groups
addresses = getattr(server, "addresses")
networks = []
floating_ips = []
for network_infos in addresses.values():
for network_info in network_infos:
addr = network_info.get("addr")
mac = network_info.get("OS-EXT-IPS-MAC:mac_addr")
network_type = network_info.get("OS-EXT-IPS:type")
if network_type == 'fixed':
port = neutron_client.list_ports(
mac_address=mac)["ports"][0]
if port["network_id"] not in networks:
networks.append(port["network_id"])
elif network_type == "floating":
floating_ips.append(addr)
flavor = getattr(server, "flavor")["id"]
key_name = getattr(server, "key_name", None)
security_groups = getattr(server, "security_groups", None)
# get metadata about boot device
boot_metadata = {}
image_info = getattr(server, "image", None)
if image_info is not None and isinstance(image_info, dict):
boot_metadata["boot_device_type"] = "image"
boot_metadata["boot_image_id"] = image_info['id']
else:
boot_metadata["boot_device_type"] = "volume"
volumes_attached = getattr(
server, "os-extended-volumes:volumes_attached", [])
for volume_attached in volumes_attached:
volume_id = volume_attached["id"]
volume_attach_metadata = attach_metadata.get(
volume_id, None)
if volume_attach_metadata is not None and (
volume_attach_metadata["bootable"] == "true"):
boot_metadata["boot_volume_id"] = volume_id
boot_metadata["boot_attach_metadata"] = (
volume_attach_metadata)
resource_definition["boot_metadata"] = boot_metadata
# save all server's metadata
server_metadata = {
"availability_zone": availability_zone,
"networks": networks,
"floating_ips": floating_ips,
"flavor": flavor,
"key_name": key_name,
"security_groups": security_groups,
}
resource_definition["server_metadata"] = server_metadata
LOG.info("Creating server backup, resource_definition: %s.",
resource_definition)
bank_section.update_object("metadata", resource_definition)
# update resource_definition backup_status
bank_section.update_object("status",
constants.RESOURCE_STATUS_AVAILABLE)
LOG.info("Finish backup server, server_id: %s.", server_id)
except Exception as err:
# update resource_definition backup_status
LOG.exception("Create backup failed, server_id: %s.", server_id)
bank_section.update_object("status",
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Server Backup",
reason=err,
resource_id=server_id,
resource_type=constants.SERVER_RESOURCE_TYPE)
def _nova_client(self, context):
self._nova_client_instance = \
ClientFactory.create_client('nova', context)
return self._nova_client_instance
def _glance_client(self, context):
self._glance_client_instance = \
ClientFactory.create_client('glance', context)
return self._glance_client_instance
def _glance_client(self, cntxt):
return ClientFactory.create_client("glance", cntxt)
def _neutron_client(self, cntxt):
return ClientFactory.create_client("neutron", cntxt)
def _nova_client(self, cntxt):
return ClientFactory.create_client("nova", cntxt)
def _cinder_client(self, cntxt):
return ClientFactory.create_client("cinder", cntxt)
def _glance_client(self, cntxt):
return ClientFactory.create_client("glance", cntxt)
def _client(self, context):
self._client_instance = ClientFactory.create_client(
"nova",
context)
return self._client_instance
