def on_main(self, checkpoint, resource, context, parameters, **kwargs):
volume_id = resource.id
bank_section = checkpoint.get_resource_bank_section(volume_id)
cinder_client = ClientFactory.create_client('cinder', context)
LOG.info('Creating volume snapshot, volume_id: %s', volume_id)
bank_section.update_object('status',
constants.RESOURCE_STATUS_PROTECTING)
volume_info = cinder_client.volumes.get(volume_id)
is_success = utils.status_poll(
partial(get_volume_status, cinder_client, volume_id),
interval=self._interval,
success_statuses={
'available', 'in-use', 'error_extending', 'error_restoring'
},
failure_statuses=VOLUME_FAILURE_STATUSES,
ignore_statuses=VOLUME_IGNORE_STATUSES,
)
if not is_success:
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Volume Snapshot",
reason='Volume is in a error status.',
resource_id=volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE,
)
resource_metadata = {'volume_id': volume_id, 'size': volume_info.size}
snapshot_name = parameters.get('snapshot_name', None)
description = parameters.get('description', None)
force = parameters.get('force', False)
try:
snapshot_id = self._create_snapshot(cinder_client, volume_id,
snapshot_name, description,
force)
except Exception as e:
LOG.error(
'Error creating snapshot (volume_id: %(volume_id)s '
': %(reason)s', {
'volume_id': volume_id,
'reason': e
})
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Volume Snapshot",
reason=e,
resource_id=volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE,
)
resource_metadata['snapshot_id'] = snapshot_id
bank_section.update_object('metadata', resource_metadata)
bank_section.update_object('status',
constants.RESOURCE_STATUS_AVAILABLE)
LOG.info(
'Snapshot volume (volume_id: %(volume_id)s snapshot_id: '
'%(snapshot_id)s ) successfully', {
'volume_id': volume_id,
'snapshot_id': snapshot_id
})
def _create_server_snapshot(self, context, glance_client, parameters,
resource, resource_definition, resource_id):
server_id = resource.extra_info.get('server_id')
resource_definition['resource_id'] = resource_id
resource_definition['server_id'] = server_id
nova_client = ClientFactory.create_client('nova', context)
is_success = utils.status_poll(
partial(get_server_status, nova_client, server_id),
interval=self._interval,
success_statuses={'ACTIVE', 'STOPPED', 'SUSPENDED', 'PAUSED'},
failure_statuses={
'DELETED', 'ERROR', 'RESIZED', 'SHELVED', 'SHELVED_OFFLOADED',
'SOFT_DELETED', 'RESCUED', 'not-found'
},
ignore_statuses={'BUILDING'},
)
if not is_success:
raise exception.CreateResourceFailed(
name="Image Backup",
reason='The parent server of the image is not in valid'
' status',
resource_id=resource_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
temp_image_name = 'Temp_image_name_for_karbor' + server_id
try:
image_uuid = nova_client.servers.create_image(
server_id, temp_image_name, parameters)
except Exception as e:
msg = "Failed to create the server snapshot: %s" % e
LOG.exception(msg)
raise exception.CreateResourceFailed(
name="Image Backup",
reason=msg,
resource_id=resource_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
else:
is_success = utils.status_poll(
partial(get_image_status, glance_client, image_uuid),
interval=self._interval,
success_statuses={'active'},
failure_statuses={
'killed', 'deleted', 'pending_delete', 'deactivated'
},
ignore_statuses={'queued', 'saving', 'uploading'})
if not is_success:
msg = "Image has been created, but fail to become " \
"active, so delete it and raise exception."
LOG.error(msg)
glance_client.images.delete(image_uuid)
image_uuid = None
if not image_uuid:
raise exception.CreateResourceFailed(
name="Image Backup",
reason="Create parent server snapshot failed.",
resource_id=resource_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
return image_uuid
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
image_id = resource.id
bank_section = checkpoint.get_resource_bank_section(image_id)
resource_definition = {"resource_id": image_id}
glance_client = ClientFactory.create_client('glance', context)
LOG.info("Creating image backup, image_id: %s.", image_id)
try:
bank_section.update_object("status",
constants.RESOURCE_STATUS_PROTECTING)
image_info = glance_client.images.get(image_id)
if image_info.status != "active":
is_success = utils.status_poll(
partial(get_image_status, glance_client, image_info.id),
interval=self._interval,
success_statuses={'active'},
ignore_statuses={'queued', 'saving'},
failure_statuses={
'killed', 'deleted', 'pending_delete', 'deactivated',
'NotFound'
})
if is_success is not True:
LOG.error("The status of image (id: %s) is invalid.",
image_id)
raise exception.CreateResourceFailed(
name="Image Backup",
reason="The status of image is invalid.",
resource_id=image_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
image_metadata = {
"disk_format": image_info.disk_format,
"container_format": image_info.container_format,
"checksum": image_info.checksum
}
resource_definition["image_metadata"] = image_metadata
bank_section.update_object("metadata", resource_definition)
except Exception as err:
LOG.error("Create image backup failed, image_id: %s.", image_id)
bank_section.update_object("status",
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Image Backup",
reason=err,
resource_id=image_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
self._create_backup(glance_client, bank_section, image_id)
def _create_volume_from_image(self, cinder_client, temporary_image,
restore_name, original_vol_id, volume_size,
description):
volume = cinder_client.volumes.create(
size=volume_size,
imageRef=temporary_image.id,
name=restore_name,
description=description
)
is_success = utils.status_poll(
partial(get_volume_status, cinder_client, volume.id),
interval=self._interval,
success_statuses=VOLUME_SUCCESS_STATUSES,
failure_statuses=VOLUME_FAILURE_STATUSES,
ignore_statuses=VOLUME_IGNORE_STATUSES
)
if not is_success:
LOG.error("Restore volume glance backup failed, volume_id: %s.",
original_vol_id)
if volume is not None and hasattr(volume, 'id'):
LOG.info("Delete the failed volume, volume_id: %s.", volume.id)
cinder_client.volumes.delete(volume.id)
raise exception.CreateResourceFailed(
name="Volume Glance Backup",
reason='Restored Volume is in erroneous state',
resource_id=volume.id,
resource_type=constants.VOLUME_RESOURCE_TYPE,
)
def _backup_temporary_image(self, glance_client, image_id, bank_section):
try:
chunks_num = utils.backup_image_to_bank(
glance_client,
image_id,
bank_section,
self._image_object_size
)
image_info = glance_client.images.get(image_id)
image_resource_definition = {
'chunks_num': chunks_num,
'image_metadata': {
'checksum': image_info.checksum,
'disk_format': image_info.disk_format,
"container_format": image_info.container_format
}
}
return image_resource_definition
except Exception as err:
LOG.exception('Protecting temporary image (id: %s) to bank '
'failed.', image_id)
raise exception.CreateResourceFailed(
name="Volume Glance Backup",
reason=err,
resource_id=image_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
def _create_temporary_volume(self, cinder_client, snapshot_id):
LOG.info("Start creating volume from snapshot({0}) success"
"".format(snapshot_id))
snapshot = cinder_client.volume_snapshots.get(snapshot_id)
volume = cinder_client.volumes.create(
size=snapshot.size,
snapshot_id=snapshot_id,
name='temporary_volume_of_{0}'.format(snapshot_id)
)
is_success = utils.status_poll(
partial(get_volume_status, cinder_client, volume.id),
interval=self._interval,
success_statuses=VOLUME_SUCCESS_STATUSES,
failure_statuses=VOLUME_FAILURE_STATUSES,
ignore_statuses=VOLUME_IGNORE_STATUSES,
)
volume = cinder_client.volumes.get(volume.id)
if is_success is not True:
LOG.error('The status of temporary volume is invalid. status:%s',
volume.status)
reason = 'Invalid status: %s of temporary volume.' % volume.status
raise exception.CreateResourceFailed(
name="Volume Glance Backup",
reason=reason,
resource_id=volume.id,
resource_type=constants.VOLUME_RESOURCE_TYPE,
)
LOG.info("Create volume from snapshot({0}) success, "
"volume({1})".format(snapshot_id, volume.id))
return volume
def _create_snapshot(self, manila_client, share_id, snapshot_name,
description, force):
snapshot = manila_client.share_snapshots.create(
share=share_id,
name=snapshot_name,
description=description,
force=force)
snapshot_id = snapshot.id
is_success = utils.status_poll(partial(get_snapshot_status,
manila_client, snapshot_id),
interval=self._interval,
success_statuses={'available'},
failure_statuses={'error'},
ignore_statuses={'creating'},
ignore_unexpected=True)
if not is_success:
try:
snapshot = manila_client.share_snapshots.get(snapshot_id)
except Exception:
reason = 'Unable to find snapshot.'
else:
reason = 'The status of snapshot is %s' % snapshot.status
raise exception.CreateResourceFailed(
name="Share Snapshot",
reason=reason,
resource_id=share_id,
resource_type=constants.SHARE_RESOURCE_TYPE)
return snapshot_id
def on_prepare_finish(self, checkpoint, resource, context, parameters,
**kwargs):
volume_id = resource.id
if not self._backup_from_snapshot:
LOG.info(
'Skipping taking snapshot of volume %s - backing up '
'directly', volume_id)
return
LOG.info('Taking snapshot of volume %s', volume_id)
bank_section = checkpoint.get_resource_bank_section(volume_id)
bank_section.update_object('status',
constants.RESOURCE_STATUS_PROTECTING)
cinder_client = ClientFactory.create_client('cinder', context)
try:
self.snapshot_id = self._create_snapshot(cinder_client, volume_id)
except Exception:
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Volume Backup",
reason='Error creating snapshot for volume',
resource_id=volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE,
)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
instance_id = resource.id
bank_section = checkpoint.get_resource_bank_section(instance_id)
trove_client = ClientFactory.create_client('trove', context)
LOG.info('creating database instance backup, instance_id: %s',
instance_id)
bank_section.update_object('status',
constants.RESOURCE_STATUS_PROTECTING)
instance_info = trove_client.instances.get(instance_id)
if instance_info.status != "ACTIVE":
is_success = utils.status_poll(
partial(get_database_instance_status, trove_client,
instance_id),
interval=self._interval,
success_statuses={'ACTIVE'},
failure_statuses=DATABASE_FAILURE_STATUSES,
ignore_statuses=DATABASE_IGNORE_STATUSES,
)
if not is_success:
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Database instance Backup",
reason='Database instance is in a error status.',
resource_id=instance_id,
resource_type=constants.DATABASE_RESOURCE_TYPE,
)
resource_metadata = {
'instance_id': instance_id,
'datastore': instance_info.datastore,
'flavor': instance_info.flavor,
'size': instance_info.volume['size'],
}
backup_name = parameters.get('backup_name', 'backup%s' % (instance_id))
description = parameters.get('description', None)
try:
backup_id = self._create_backup(trove_client, instance_id,
backup_name, description)
except exception.CreateResourceFailed as e:
LOG.error(
'Error creating backup (instance_id: %(instance_id)s '
': %(reason)s', {
'instance_id': instance_id,
'reason': e
})
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise
resource_metadata['backup_id'] = backup_id
bank_section.update_object('metadata', resource_metadata)
bank_section.update_object('status',
constants.RESOURCE_STATUS_AVAILABLE)
LOG.info(
'Backup database instance (instance_id: %(instance_id)s '
'backup_id: %(backup_id)s ) successfully', {
'instance_id': instance_id,
'backup_id': backup_id
})
def _create_backup(self, glance_client, bank_section, image_id):
try:
chunks_num = utils.backup_image_to_bank(
glance_client,
image_id, bank_section,
self._data_block_size_bytes
)
# Save the chunks_num to metadata
resource_definition = bank_section.get_object("metadata")
if resource_definition is not None:
resource_definition["chunks_num"] = chunks_num
bank_section.update_object("metadata", resource_definition)
# Update resource_definition backup_status
bank_section.update_object("status",
constants.RESOURCE_STATUS_AVAILABLE)
LOG.info('Protecting image (id: %s) to bank completed '
'successfully', image_id)
except Exception as err:
# update resource_definition backup_status
LOG.exception('Protecting image (id: %s) to bank failed.',
image_id)
bank_section.update_object("status",
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Image Backup",
reason=err,
resource_id=image_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
def _create_snapshot(self, cinder_client, volume_id, snapshot_name,
description, force):
snapshot = cinder_client.volume_snapshots.create(
volume_id=volume_id,
name=snapshot_name,
description=description,
force=force
)
snapshot_id = snapshot.id
is_success = utils.status_poll(
partial(get_snapshot_status, cinder_client, snapshot_id),
interval=self._interval,
success_statuses={'available'},
failure_statuses={'error', 'error_deleting', 'deleting',
'not-found'},
ignore_statuses={'creating'},
ignore_unexpected=True
)
if not is_success:
try:
snapshot = cinder_client.volume_snapshots.get(snapshot_id)
except Exception:
reason = 'Unable to find volume snapshot.'
else:
reason = 'The status of snapshot is %s' % snapshot.status
raise exception.CreateResourceFailed(
name="Volume Snapshot",
reason=reason, resource_id=volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE)
return snapshot_id
def _create_backup(self, trove_client, instance_id, backup_name,
description):
backup = trove_client.backups.create(
backup_name,
instance=instance_id,
description=description
)
backup_id = backup.id
is_success = utils.status_poll(
partial(get_backup_status, trove_client, backup_id),
interval=self._interval,
success_statuses={'COMPLETED'},
failure_statuses={'FAILED'},
ignore_statuses={'BUILDING'},
ignore_unexpected=True
)
if not is_success:
try:
backup = trove_client.backups.get(backup_id)
except Exception:
reason = 'Unable to find backup.'
else:
reason = 'The status of backup is %s' % backup.status
raise exception.CreateResourceFailed(
name="Database Instance Backup",
reason=reason, resource_id=instance_id,
resource_type=constants.DATABASE_RESOURCE_TYPE)
return backup_id
def _create_snapshot(self, cinder_client, volume_id):
LOG.info("Start creating snapshot of volume({0}).".format(volume_id))
snapshot = cinder_client.volume_snapshots.create(
volume_id,
name='temporary_snapshot_of_{0}'.format(volume_id),
force=True
)
snapshot_id = snapshot.id
is_success = utils.status_poll(
partial(get_snapshot_status, cinder_client, snapshot_id),
interval=self._interval,
success_statuses={'available', },
failure_statuses={'error', 'error_deleting', 'deleting',
'not-found'},
ignore_statuses={'creating', },
)
if is_success is not True:
try:
snapshot = cinder_client.volume_snapshots.get(snapshot_id)
except Exception:
reason = 'Unable to find volume snapshot.'
else:
reason = 'The status of snapshot is %s' % snapshot.status
raise exception.CreateResourceFailed(
name="Volume Glance Backup",
reason=reason,
resource_id=volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE
)
LOG.info("Create snapshot of volume({0}) success, "
"snapshot_id({1})".format(volume_id, snapshot_id))
return snapshot_id
def _create_temporary_image(self, cinder_client, glance_client,
temporary_volume):
LOG.info("Start creating image from volume({0})."
"".format(temporary_volume.id))
image = cinder_client.volumes.upload_to_image(
volume=temporary_volume,
force=True,
image_name='temporary_image_of_{0}'.format(temporary_volume.id),
container_format="bare",
disk_format="raw",
visibility="private",
protected=False
)
image_id = image[1]['os-volume_upload_image']['image_id']
is_success = utils.status_poll(
partial(get_image_status, glance_client, image_id),
interval=self._interval, success_statuses={'active'},
ignore_statuses={'queued', 'saving'},
failure_statuses={'killed', 'deleted', 'pending_delete',
'deactivated', 'NotFound'}
)
image_info = glance_client.images.get(image_id)
if is_success is not True:
LOG.error("The status of image (id: %s) is invalid.",
image_id)
reason = "Invalid status: %s of temporary image." % \
image_info.status
raise exception.CreateResourceFailed(
name="Volume Glance Backup",
reason=reason,
resource_id=image_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
LOG.info("Create image({0}) from volume({1}) "
"success.".format(image_id, temporary_volume.id))
return image_id
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
original_volume_id = resource.id
restore_name = parameters.get('restore_name',
'%s@%s' % (checkpoint.id,
original_volume_id))
restore_description = parameters.get('restore_description', None)
bank_section = checkpoint.get_resource_bank_section(original_volume_id)
cinder_client = ClientFactory.create_client('cinder', context)
glance_client = ClientFactory.create_client('glance', context)
resource_metadata = bank_section.get_object('metadata')
volume_size = int(resource_metadata['volume_size'])
temporary_image = None
try:
temporary_image = self._create_temporary_image(
bank_section, glance_client, original_volume_id
)
volume_id = self._create_volume_from_image(
cinder_client, temporary_image, restore_name, volume_size,
restore_description
)
# check and update
update_method = partial(
utils.update_resource_restore_result,
kwargs.get('restore'), resource.type, volume_id
)
update_method(constants.RESOURCE_STATUS_RESTORING)
is_success = self._check_create_complete(cinder_client, volume_id)
if is_success:
update_method(constants.RESOURCE_STATUS_AVAILABLE)
kwargs.get("new_resources")[original_volume_id] = volume_id
else:
LOG.error("Restore volume glance backup failed, so delete "
"the temporary volume: volume_id: %s.",
original_volume_id)
cinder_client.volumes.delete(volume_id)
raise exception.CreateResourceFailed(
name="Volume Glance Backup",
reason='Restored Volume is in erroneous state',
resource_id=volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE,
)
finally:
if temporary_image:
try:
glance_client.images.delete(temporary_image.id)
except Exception as e:
LOG.warning('Failed deleting temporary image: '
'%(temporary_image_id)s. '
'Reason: %(reason)s', {
'temporary_image_id': temporary_image.id,
'reason': e
})
LOG.info("Finish restoring volume backup, volume_id: %s.",
original_volume_id)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
resource_id = resource.id
bank_section = checkpoint.get_resource_bank_section(resource_id)
LOG.info("Deleting freezer protection backup, resource_id: {0}".format(
resource_id))
bank_section.update_object('status',
constants.RESOURCE_STATUS_DELETING)
resource_metadata = bank_section.get_object('metadata')
freezer_task_info = resource_metadata.get('job_info', None)
if not freezer_task_info:
raise exception.DeleteResourceFailed(
name='Freezer Backup FreezerTask',
reason='The content of freezer job is invalid.',
resource_id=resource_id,
resource_type=resource.type)
freezer_job_operation = FreezerTask(context)
job_id, job_info = None, None
try:
job_id, job_info = freezer_job_operation.create_delete_job(
freezer_task_info)
is_success = utils.status_poll(partial(get_job_status,
freezer_job_operation,
job_id),
interval=self._poll_interval,
success_statuses={'success'},
failure_statuses={'fail'},
ignore_statuses={'aborted', ''},
ignore_unexpected=True)
if is_success is not True:
LOG.error(
"The status of freezer job (id: {0}) is invalid.".format(
job_id))
raise exception.CreateResourceFailed(
name="Freezer Backup FreezerTask",
reason="The status of freezer job is invalid.",
resource_id=resource_id,
resource_type=resource.type)
except Exception as e:
LOG.error("Delete freezer backup resource failed, resource_type:"
"{0}, resource_id: {1}".format(resource.type,
resource.id))
if job_id:
freezer_job_operation.delete(job_id)
raise exception.DeleteResourceFailed(
name="Freezer Backup FreezerTask",
reason=e,
resource_id=resource_id,
resource_type=resource.type)
LOG.debug('Finish deleting freezer backup resource')
bank_section.delete_object('metadata')
bank_section.update_object('status', constants.RESOURCE_STATUS_DELETED)
freezer_job_operation.delete(job_id)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
pod_id = resource.id
pod_name = resource.name
bank_section = checkpoint.get_resource_bank_section(pod_id)
k8s_client = ClientFactory.create_client("k8s", context)
resource_definition = {"resource_id": pod_id}
LOG.info("Creating pod backup, id: %(pod_id)s) name: "
"%(pod_name)s.", {
"pod_id": pod_id,
"pod_name": pod_name
})
try:
bank_section.update_object("status",
constants.RESOURCE_STATUS_PROTECTING)
# get metadata about pod
pod_namespace, k8s_pod_name = pod_name.split(":")
pod = k8s_client.read_namespaced_pod(k8s_pod_name, pod_namespace)
resource_definition["resource_name"] = pod_name
resource_definition["namespace"] = pod_namespace
mounted_volumes_list = self._get_mounted_volumes(
k8s_client, pod, pod_namespace)
containers_list = self._get_containers(pod)
# save all pod's metadata
pod_metadata = {
'apiVersion': pod.api_version,
'kind': pod.kind,
'metadata': {
'labels': pod.metadata.labels,
'name': pod.metadata.name,
'namespace': pod.metadata.namespace,
},
'spec': {
'containers': containers_list,
'volumes': mounted_volumes_list,
'restartPolicy': pod.spec.restart_policy
}
}
resource_definition["pod_metadata"] = pod_metadata
LOG.debug("Creating pod backup, pod_metadata: %s.", pod_metadata)
bank_section.update_object("metadata", resource_definition)
bank_section.update_object("status",
constants.RESOURCE_STATUS_AVAILABLE)
LOG.info("Finish backup pod, pod_id: %s.", pod_id)
except Exception as err:
LOG.exception("Create pod backup failed, pod_id: %s.", pod_id)
bank_section.update_object("status",
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Pod Backup",
reason=err,
resource_id=pod_id,
resource_type=constants.POD_RESOURCE_TYPE)
def _create_backup(self, glance_client, bank_section, image_id):
try:
image_response = glance_client.images.data(image_id,
do_checksum=True)
bank_chunk_num = int(self._data_block_size_bytes / 65536)
LOG.debug("Creating image backup, bank_chunk_num: %s.",
bank_chunk_num)
# backup the data of image
image_chunks_num = 0
chunks_num = 1
image_response_data = BytesIO()
for chunk in image_response:
image_response_data.write(chunk)
image_chunks_num += 1
if image_chunks_num == bank_chunk_num:
image_chunks_num = 0
image_response_data.seek(0, os.SEEK_SET)
data = image_response_data.read(
self._data_block_size_bytes)
bank_section.update_object("data_" + str(chunks_num), data)
image_response_data.truncate(0)
image_response_data.seek(0, os.SEEK_SET)
chunks_num += 1
image_response_data.seek(0, os.SEEK_SET)
data = image_response_data.read()
if data != '':
bank_section.update_object("data_" + str(chunks_num), data)
else:
chunks_num -= 1
# Save the chunks_num to metadata
resource_definition = bank_section.get_object("metadata")
if resource_definition is not None:
resource_definition["chunks_num"] = chunks_num
bank_section.update_object("metadata", resource_definition)
# Update resource_definition backup_status
bank_section.update_object("status",
constants.RESOURCE_STATUS_AVAILABLE)
LOG.info(
'Protecting image (id: %s) to bank completed '
'successfully', image_id)
except Exception as err:
# update resource_definition backup_status
LOG.exception('Protecting image (id: %s) to bank failed.',
image_id)
bank_section.update_object("status",
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Image Backup",
reason=err,
resource_id=image_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
network_id = get_network_id(context)
backup_name = kwargs.get("backup_name", "karbor network backup")
bank_section = checkpoint.get_resource_bank_section(network_id)
neutron_client = ClientFactory.create_client("neutron", context)
resource_definition = {"resource_id": network_id}
resource_definition["backup_name"] = backup_name
resource_definition["network_metadata"] = (
self._get_resources_by_network(context, neutron_client))
resource_definition["subnet_metadata"] = (
self._get_resources_by_subnet(context, neutron_client))
resource_definition["port_metadata"] = (
self._get_resources_by_port(context, neutron_client))
resource_definition["router_metadata"] = (
self._get_resources_by_router(context, neutron_client))
resource_definition["security-group_metadata"] = (
self._get_resources_by_security_group(context, neutron_client))
try:
bank_section.update_object("status",
constants.RESOURCE_STATUS_PROTECTING)
# write resource_definition in bank
bank_section.update_object("metadata", resource_definition)
# update resource_definition backup_status
bank_section.update_object("status",
constants.CHECKPOINT_STATUS_AVAILABLE)
LOG.info("finish backup network, network_id: %s.", network_id)
except Exception as err:
# update resource_definition backup_status
LOG.error("create backup failed, network_id: %s.", network_id)
bank_section.update_object("status",
constants.CHECKPOINT_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Network Backup",
reason=err,
resource_id=network_id,
resource_type=self._SUPPORT_RESOURCE_TYPES)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
server_id = resource.id
bank_section = checkpoint.get_resource_bank_section(server_id)
nova_client = ClientFactory.create_client("nova", context)
cinder_client = ClientFactory.create_client("cinder", context)
neutron_client = ClientFactory.create_client("neutron", context)
resource_definition = {"resource_id": server_id}
# get dependent resources
server_child_nodes = []
resources = checkpoint.resource_graph
for resource_node in resources:
resource = resource_node.value
if resource.id == server_id:
server_child_nodes = resource_node.child_nodes
LOG.info("Creating server backup, server_id: %s. ", server_id)
try:
bank_section.update_object("status",
constants.RESOURCE_STATUS_PROTECTING)
# get attach_metadata about volume
attach_metadata = {}
for server_child_node in server_child_nodes:
child_resource = server_child_node.value
if child_resource.type == constants.VOLUME_RESOURCE_TYPE:
volume = cinder_client.volumes.get(child_resource.id)
attachments = getattr(volume, "attachments")
for attachment in attachments:
if attachment["server_id"] == server_id:
attachment["bootable"] = getattr(
volume, "bootable")
attach_metadata[child_resource.id] = attachment
resource_definition["attach_metadata"] = attach_metadata
# get metadata about AZ
server = nova_client.servers.get(server_id)
availability_zone = getattr(server, "OS-EXT-AZ:availability_zone")
# get metadata about network, flavor, key_name, security_groups
addresses = getattr(server, "addresses")
networks = []
floating_ips = []
for network_infos in addresses.values():
for network_info in network_infos:
addr = network_info.get("addr")
mac = network_info.get("OS-EXT-IPS-MAC:mac_addr")
network_type = network_info.get("OS-EXT-IPS:type")
if network_type == 'fixed':
port = neutron_client.list_ports(
mac_address=mac)["ports"][0]
if port["network_id"] not in networks:
networks.append(port["network_id"])
elif network_type == "floating":
floating_ips.append(addr)
flavor = getattr(server, "flavor")["id"]
key_name = getattr(server, "key_name", None)
security_groups = getattr(server, "security_groups", None)
# get metadata about boot device
boot_metadata = {}
image_info = getattr(server, "image", None)
if image_info is not None and isinstance(image_info, dict):
boot_metadata["boot_device_type"] = "image"
boot_metadata["boot_image_id"] = image_info['id']
else:
boot_metadata["boot_device_type"] = "volume"
volumes_attached = getattr(
server, "os-extended-volumes:volumes_attached", [])
for volume_attached in volumes_attached:
volume_id = volume_attached["id"]
volume_attach_metadata = attach_metadata.get(
volume_id, None)
if volume_attach_metadata is not None and (
volume_attach_metadata["bootable"] == "true"):
boot_metadata["boot_volume_id"] = volume_id
boot_metadata["boot_attach_metadata"] = (
volume_attach_metadata)
resource_definition["boot_metadata"] = boot_metadata
# save all server's metadata
server_metadata = {
"availability_zone": availability_zone,
"networks": networks,
"floating_ips": floating_ips,
"flavor": flavor,
"key_name": key_name,
"security_groups": security_groups,
}
resource_definition["server_metadata"] = server_metadata
LOG.info("Creating server backup, resource_definition: %s.",
resource_definition)
bank_section.update_object("metadata", resource_definition)
# update resource_definition backup_status
bank_section.update_object("status",
constants.RESOURCE_STATUS_AVAILABLE)
LOG.info("Finish backup server, server_id: %s.", server_id)
except Exception as err:
# update resource_definition backup_status
LOG.exception("Create backup failed, server_id: %s.", server_id)
bank_section.update_object("status",
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Server Backup",
reason=err,
resource_id=server_id,
resource_type=constants.SERVER_RESOURCE_TYPE)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
resource_id = resource.id
bank_section = checkpoint.get_resource_bank_section(resource_id)
LOG.info('Creating freezer protection backup, resource_id: {0}'.format(
resource_id))
bank_section.update_object('status',
constants.RESOURCE_STATUS_PROTECTING)
backup_name = parameters.get('backup_name',
'backup{0}'.format(resource_id))
description = parameters.get('description', None)
self.freezer_storage.storage_path = "{0}/{1}".format(
self.freezer_storage.storage_path, checkpoint.id)
job_id, job_info = None, None
freezer_task = FreezerTask(context)
try:
job_id, job_info = freezer_task.create(
backup_name=backup_name,
storage=self.freezer_storage,
description=description,
resource=resource,
action_type='backup',
scheduler_client_id=self._scheduler_client_id)
LOG.debug(
'Creating freezer backup job successful, job_id: {0}'.format(
job_id))
is_success = utils.status_poll(partial(get_job_status,
freezer_task, job_id),
interval=self._poll_interval,
success_statuses={'success'},
failure_statuses={'fail'},
ignore_statuses={'aborted', ''},
ignore_unexpected=True)
if is_success is not True:
LOG.error(
"The status of freezer job (id: {0}) is invalid.".format(
job_id))
raise exception.CreateResourceFailed(
name="Freezer Backup FreezerTask",
reason="The status of freezer job is invalid.",
resource_id=resource_id,
resource_type=resource.type)
resource_definition = {'job_id': job_id, 'job_info': job_info}
bank_section.update_object("metadata", resource_definition)
bank_section.update_object("status",
constants.RESOURCE_STATUS_AVAILABLE)
except exception.CreateResourceFailed as e:
LOG.error(
'Error creating backup (resource_id: {0}, reason: {1})'.format(
resource_id, e))
if job_id:
freezer_task.delete(job_id)
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise
LOG.debug('Finish creating freezer backup resource')
freezer_task.delete(job_id)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
volume_id = resource.id
bank_section = checkpoint.get_resource_bank_section(volume_id)
cinder_client = ClientFactory.create_client('cinder', context)
LOG.info('creating volume backup, volume_id: %s', volume_id)
bank_section.update_object('status',
constants.RESOURCE_STATUS_PROTECTING)
resource_metadata = {
'volume_id': volume_id,
}
is_success = utils.status_poll(
partial(get_volume_status, cinder_client, volume_id),
interval=self._interval,
success_statuses={
'available', 'in-use', 'error_extending', 'error_restoring'
},
failure_statuses={
'error', 'error_deleting', 'deleting', 'not-found'
},
ignore_statuses={
'attaching', 'creating', 'backing-up', 'restoring-backup'
},
)
if not is_success:
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Volume Backup",
reason='Volume is in errorneous state',
resource_id=volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE,
)
backup_name = parameters.get('backup_name', None)
description = parameters.get('description', None)
backup_mode = parameters.get('backup_mode', "full")
container = parameters.get('container', None)
force = parameters.get('force', False)
incremental = False
if backup_mode == "incremental":
incremental = True
elif backup_mode == "full":
incremental = False
try:
backup_id = self._create_backup(cinder_client, volume_id,
backup_name, description,
self.snapshot_id, incremental,
container, force)
except Exception as e:
LOG.error(
'Error creating backup (volume_id: %(volume_id)s '
'snapshot_id: %(snapshot_id)s): %(reason)s', {
'volume_id': volume_id,
'snapshot_id': self.snapshot_id,
'reason': e
})
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Volume Backup",
reason=e,
resource_id=volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE,
)
resource_metadata['backup_id'] = backup_id
bank_section.update_object('metadata', resource_metadata)
bank_section.update_object('status',
constants.RESOURCE_STATUS_AVAILABLE)
LOG.info(
'Backed up volume (volume_id: %(volume_id)s snapshot_id: '
'%(snapshot_id)s backup_id: %(backup_id)s) successfully', {
'backup_id': backup_id,
'snapshot_id': self.snapshot_id,
'volume_id': volume_id
})
if self.snapshot_id:
try:
self._delete_snapshot(cinder_client, self.snapshot_id)
except Exception as e:
LOG.warning(
'Failed deleting snapshot: %(snapshot_id)s. '
'Reason: %(reason)s', {
'snapshot_id': self.snapshot_id,
'reason': e
})
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
share_id = resource.id
bank_section = checkpoint.get_resource_bank_section(share_id)
manila_client = ClientFactory.create_client('manila', context)
LOG.info('creating share snapshot, share_id: %s', share_id)
bank_section.update_object('status',
constants.RESOURCE_STATUS_PROTECTING)
share_info = manila_client.shares.get(share_id)
if share_info.status != "available":
is_success = utils.status_poll(
partial(get_share_status, manila_client, share_id),
interval=self._interval,
success_statuses={'available'},
failure_statuses=SHARE_FAILURE_STATUSES,
ignore_statuses=SHARE_IGNORE_STATUSES,
)
if not is_success:
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Share Snapshot",
reason='Share is in a error status.',
resource_id=share_id,
resource_type=constants.SHARE_RESOURCE_TYPE,
)
resource_metadata = {
'share_id': share_id,
'size': share_info.size,
'share_proto': share_info.share_proto,
'share_type': share_info.share_type,
'share_network_id': share_info.share_network_id
}
snapshot_name = parameters.get('snapshot_name', None)
description = parameters.get('description', None)
force = parameters.get('force', False)
try:
snapshot_id = self._create_snapshot(manila_client, share_id,
snapshot_name, description,
force)
except exception.CreateResourceFailed as e:
LOG.error(
'Error creating snapshot (share_id: %(share_id)s '
': %(reason)s', {
'share_id': share_id,
'reason': e
})
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise
resource_metadata['snapshot_id'] = snapshot_id
bank_section.update_object('metadata', resource_metadata)
bank_section.update_object('status',
constants.RESOURCE_STATUS_AVAILABLE)
LOG.info(
'Snapshot share (share_id: %(share_id)s snapshot_id: '
'%(snapshot_id)s ) successfully', {
'share_id': share_id,
'snapshot_id': snapshot_id
})
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
LOG.debug('Start creating image backup, resource info: %s',
resource.to_dict())
resource_id = resource.id
bank_section = checkpoint.get_resource_bank_section(resource_id)
glance_client = ClientFactory.create_client('glance', context)
bank_section.update_object("status",
constants.RESOURCE_STATUS_PROTECTING)
resource_definition = {'resource_id': resource_id}
if resource.extra_info and self._enable_server_snapshot:
image_id = self._create_server_snapshot(context, glance_client,
parameters, resource,
resource_definition,
resource_id)
need_delete_temp_image = True
else:
image_id = resource_id
need_delete_temp_image = False
LOG.info("Creating image backup, image_id: %s.", image_id)
try:
image_info = glance_client.images.get(image_id)
if image_info.status != "active":
is_success = utils.status_poll(
partial(get_image_status, glance_client, image_info.id),
interval=self._interval,
success_statuses={'active'},
ignore_statuses={'queued', 'saving'},
failure_statuses={
'killed', 'deleted', 'pending_delete', 'deactivated',
'NotFound'
})
if is_success is not True:
LOG.error("The status of image (id: %s) is invalid.",
image_id)
raise exception.CreateResourceFailed(
name="Image Backup",
reason="The status of image is invalid.",
resource_id=image_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
image_metadata = {
"disk_format": image_info.disk_format,
"container_format": image_info.container_format,
"checksum": image_info.checksum,
}
resource_definition["image_metadata"] = image_metadata
bank_section.update_object("metadata", resource_definition)
except Exception as err:
LOG.error("Create image backup failed, image_id: %s.", image_id)
bank_section.update_object("status",
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Image Backup",
reason=err,
resource_id=image_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
self._create_backup(glance_client, bank_section, image_id)
if need_delete_temp_image:
try:
glance_client.images.delete(image_id)
except Exception as error:
LOG.warning('Failed to delete temporary image: %s', error)
else:
LOG.debug('Delete temporary image(%s) success', image_id)
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
volume_id = resource.id
bank_section = checkpoint.get_resource_bank_section(volume_id)
cinder_client = ClientFactory.create_client('cinder', context)
glance_client = ClientFactory.create_client('glance', context)
LOG.info('creating volume backup by glance, volume_id: %s', volume_id)
bank_section.update_object('status',
constants.RESOURCE_STATUS_PROTECTING)
resource_metadata = {
'volume_id': volume_id,
}
is_success = utils.status_poll(
partial(get_volume_status, cinder_client, volume_id),
interval=self._interval,
success_statuses=VOLUME_SUCCESS_STATUSES,
failure_statuses=VOLUME_FAILURE_STATUSES,
ignore_statuses=VOLUME_IGNORE_STATUSES,
)
if not is_success:
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Volume Glance Backup",
reason='Volume is in erroneous state',
resource_id=volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE,
)
volume_info = cinder_client.volumes.get(volume_id)
resource_metadata['volume_size'] = volume_info.size
snapshot_id = None
temporary_volume = None
temporary_image_id = None
try:
snapshot_id = self._create_snapshot(cinder_client, volume_id)
temporary_volume = self._create_temporary_volume(
cinder_client, snapshot_id)
temporary_image_id = self._create_temporary_image(
cinder_client, glance_client, temporary_volume)
image_resource_metadata = \
self._backup_temporary_image(glance_client, temporary_image_id,
bank_section)
metadata = dict(resource_metadata, **image_resource_metadata)
bank_section.update_object('metadata', metadata)
bank_section.update_object('status',
constants.RESOURCE_STATUS_AVAILABLE)
LOG.info('Backed up volume '
'(volume_id: %(volume_id)s '
'snapshot_id: %(snapshot_id)s '
'temporary_volume_id: %(temporary_volume_id)s) '
'temporary_image_id: %(temporary_image_id)s '
'successfully', {
'volume_id': volume_id,
'snapshot_id': snapshot_id,
'temporary_volume_id': temporary_volume.id,
'temporary_image_id': temporary_image_id
})
finally:
if snapshot_id:
try:
cinder_client.volume_snapshots.delete(snapshot_id)
except Exception as e:
LOG.warning('Failed deleting snapshot: %(snapshot_id)s. '
'Reason: %(reason)s',
{'snapshot_id': self.snapshot_id, 'reason': e})
if temporary_volume:
try:
cinder_client.volumes.delete(temporary_volume.id)
except Exception as e:
LOG.warning('Failed deleting temporary volume: '
'%(temporary_volume_id)s. '
'Reason: %(reason)s', {
'temporary_volume_id': temporary_volume.id,
'reason': e
})
if temporary_image_id:
try:
glance_client.images.delete(temporary_image_id)
except Exception as e:
LOG.warning('Failed deleting temporary image: '
'%(temporary_image_id)s. '
'Reason: %(reason)s', {
'temporary_image_id': temporary_image_id,
'reason': e})