def _run_operation(self, operation_id, param):
self._update_operation_state(
operation_id, {'state': constants.OPERATION_STATE_RUNNING})
try:
check_item = [self._CHECK_ITEMS['is_canceled']]
if self._check_operation(operation_id, check_item):
return
try:
operation = objects.ScheduledOperation.get_by_id(
context.get_admin_context(), operation_id)
except Exception:
LOG.exception(_LE("Run operation(%s), get operation failed"),
operation_id)
return
try:
param['user_id'] = operation.user_id
param['project_id'] = operation.project_id
self._operation_manager.run_operation(
operation.operation_type,
operation.operation_definition,
param=param)
except Exception:
LOG.exception(_LE("Run operation(%s) failed"), operation_id)
finally:
self._update_operation_state(
operation_id, {'state': constants.OPERATION_STATE_REGISTERED})
def _run_operation(self, operation_id, param):
self._update_operation_state(
operation_id,
{'state': constants.OPERATION_STATE_RUNNING})
try:
check_item = [self._CHECK_ITEMS['is_canceled']]
if self._check_operation(operation_id, check_item):
return
try:
operation = objects.ScheduledOperation.get_by_id(
context.get_admin_context(), operation_id)
except Exception:
LOG.exception(_LE("Run operation(%s), get operation failed"),
operation_id)
return
try:
param['user_id'] = operation.user_id
param['project_id'] = operation.project_id
self._operation_manager.run_operation(
operation.operation_type,
operation.operation_definition,
param=param)
except Exception:
LOG.exception(_LE("Run operation(%s) failed"), operation_id)
finally:
self._update_operation_state(
operation_id,
{'state': constants.OPERATION_STATE_REGISTERED})
def delete(self, context, provider_id, checkpoint_id):
LOG.info(_LI("Starting protection service:delete action"))
LOG.debug('provider_id :%s checkpoint_id:%s', provider_id,
checkpoint_id)
provider = self.provider_registry.show_provider(provider_id)
try:
checkpoint_collection = provider.get_checkpoint_collection()
checkpoint = checkpoint_collection.get(checkpoint_id)
except Exception:
LOG.error(_LE("get checkpoint failed, checkpoint_id:%s"),
checkpoint_id)
raise exception.InvalidInput(
reason=_("Invalid checkpoint_id or provider_id"))
if checkpoint.status not in [
constants.CHECKPOINT_STATUS_AVAILABLE,
constants.CHECKPOINT_STATUS_ERROR,
]:
raise exception.CheckpointNotBeDeleted(checkpoint_id=checkpoint_id)
checkpoint.status = constants.CHECKPOINT_STATUS_DELETING
checkpoint.commit()
try:
delete_checkpoint_flow = self.worker.get_delete_checkpoint_flow(
context, constants.OPERATION_DELETE, checkpoint, provider)
except Exception:
LOG.exception(
_LE("Failed to create delete checkpoint flow, checkpoint:%s."),
checkpoint_id)
raise exception.KarborException(
_("Failed to create delete checkpoint flow."))
self._spawn(self.worker.run_flow, delete_checkpoint_flow)
def restore(self, context, restore, restore_auth):
LOG.info(_LI("Starting restore service:restore action"))
checkpoint_id = restore["checkpoint_id"]
provider_id = restore["provider_id"]
provider = self.provider_registry.show_provider(provider_id)
try:
checkpoint_collection = provider.get_checkpoint_collection()
checkpoint = checkpoint_collection.get(checkpoint_id)
except Exception:
LOG.error(_LE("Invalid checkpoint id: %s"), checkpoint_id)
raise exception.InvalidInput(reason=_("Invalid checkpoint id"))
if checkpoint.status != constants.CHECKPOINT_STATUS_AVAILABLE:
raise exception.CheckpointNotAvailable(checkpoint_id=checkpoint_id)
try:
restoration_flow = self.worker.get_restoration_flow(
context, constants.OPERATION_RESTORE, checkpoint, provider,
restore, restore_auth)
except Exception:
LOG.exception(
_LE("Failed to create restoration flow checkpoint: %s"),
checkpoint_id)
raise exception.FlowError(flow="restore",
error=_("Failed to create flow"))
try:
self.worker.run_flow(restoration_flow)
except Exception:
LOG.exception(_LE("Failed to run restoration flow checkpoint: %s"),
checkpoint_id)
raise exception.FlowError(flow="restore",
error=_("Failed to run flow"))
def create_heat_client_with_tenant(context, conf):
cacert = conf.heat_client.heat_ca_cert_file
insecure = conf.heat_client.heat_auth_insecure
conf.register_opts(heat_client_opts, group=SERVICE + '_client')
try:
url = utils.get_url(SERVICE,
context,
conf,
append_project_fmt='%(url)s/%(project)s')
except Exception:
LOG.error(_LE("Get heat service endpoint url failed."))
raise
try:
LOG.info(_LI('Creating heat client with url %s.'), url)
heat = hc.Client(HEATCLIENT_VERSION,
endpoint=url,
token=context.auth_token,
cacert=cacert,
insecure=insecure)
return heat
except Exception:
LOG.error(_LE('Creating heat client with endpoint fails.'))
raise
def _trigger_operations(self, expect_run_time, trigger_property, timer):
"""Trigger operations once
returns: wait time for next run
"""
# Just for robustness, actually expect_run_time always <= now
# but, if the scheduling of eventlet is not accurate, then we
# can do some adjustments.
entry_time = datetime.utcnow()
if entry_time < expect_run_time and (
int(timeutils.delta_seconds(entry_time, expect_run_time)) > 0):
return expect_run_time
window = trigger_property.get("window")
end_time = expect_run_time + timedelta(seconds=window)
if end_time <= entry_time:
LOG.error(_LE("Can not trigger operations to run. "
"Because it is out of end time of window. "
"expect run time=%(expect)s, end time=%(end)s, "
"entry time=%(now)s"),
{'expect': expect_run_time, 'end': end_time,
'now': entry_time})
else:
# The self._executor.execute_operation may have I/O operation.
# If it is, this green thread will be switched out during looping
# operation_ids. In order to avoid changing self._operation_ids
# during the green thread is switched out, copy self._operation_ids
# as the iterative object.
operation_ids = self._operation_ids.copy()
for operation_id in operation_ids:
try:
self._executor.execute_operation(
operation_id, entry_time, expect_run_time, window)
except Exception:
LOG.exception(_LE("Submit operation to executor "
"failed, id=%s"),
operation_id)
pass
next_time = self._compute_next_run_time(
expect_run_time, trigger_property['end_time'], timer)
now = datetime.utcnow()
if next_time and next_time <= now:
LOG.error(_LE("Next run time:%(next_time)s <= now:%(now)s. Maybe "
"the entry time=%(entry)s is too late, even exceeds "
"the end time of window=%(end)s, or it was blocked "
"where sending the operation to executor."),
{'next_time': next_time, 'now': now,
'entry': entry_time, 'end': end_time})
return next_time
def load_class(namespace, plugin_name):
try:
LOG.debug('Start load plugin %s. ', plugin_name)
# Try to resolve plugin by name
mgr = driver.DriverManager(namespace, plugin_name)
return mgr.driver
except RuntimeError as e1:
# fallback to class name
try:
return importutils.import_class(plugin_name)
except ImportError as e2:
LOG.error(_LE("Error loading plugin by name, %s"), e1)
LOG.error(_LE("Error loading plugin by class, %s"), e2)
raise ImportError(_("Class not found."))
def report_state(self):
"""Update the state of this service in the datastore."""
if not self.manager.is_working():
# NOTE(dulek): If manager reports a problem we're not sending
# heartbeats - to indicate that service is actually down.
LOG.error(
_LE('Manager for service %(binary)s %(host)s is '
'reporting problems, not sending heartbeat. '
'Service will appear "down".'), {
'binary': self.binary,
'host': self.host
})
return
ctxt = context.get_admin_context()
state_catalog = {}
try:
try:
service_ref = db.service_get(ctxt, self.service_id)
except exception.NotFound:
LOG.debug('The service database object disappeared, '
'recreating it.')
self._create_service_ref(ctxt)
service_ref = db.service_get(ctxt, self.service_id)
state_catalog['report_count'] = service_ref['report_count'] + 1
db.service_update(ctxt, self.service_id, state_catalog)
# TODO(termie): make this pattern be more elegant.
if getattr(self, 'model_disconnected', False):
self.model_disconnected = False
LOG.error(_LE('Recovered model server connection!'))
except db_exc.DBConnectionError:
if not getattr(self, 'model_disconnected', False):
self.model_disconnected = True
LOG.exception(_LE('model server went away'))
# NOTE(jsbryant) Other DB errors can happen in HA configurations.
# such errors shouldn't kill this thread, so we handle them here.
except db_exc.DBError:
if not getattr(self, 'model_disconnected', False):
self.model_disconnected = True
LOG.exception(_LE('DBError encountered: '))
except Exception:
if not getattr(self, 'model_disconnected', False):
self.model_disconnected = True
LOG.exception(_LE('Exception encountered: '))
def report_state(self):
"""Update the state of this service in the datastore."""
if not self.manager.is_working():
# NOTE(dulek): If manager reports a problem we're not sending
# heartbeats - to indicate that service is actually down.
LOG.error(_LE('Manager for service %(binary)s %(host)s is '
'reporting problems, not sending heartbeat. '
'Service will appear "down".'),
{'binary': self.binary,
'host': self.host})
return
ctxt = context.get_admin_context()
state_catalog = {}
try:
try:
service_ref = db.service_get(ctxt, self.service_id)
except exception.NotFound:
LOG.debug('The service database object disappeared, '
'recreating it.')
self._create_service_ref(ctxt)
service_ref = db.service_get(ctxt, self.service_id)
state_catalog['report_count'] = service_ref['report_count'] + 1
db.service_update(ctxt,
self.service_id, state_catalog)
# TODO(termie): make this pattern be more elegant.
if getattr(self, 'model_disconnected', False):
self.model_disconnected = False
LOG.error(_LE('Recovered model server connection!'))
except db_exc.DBConnectionError:
if not getattr(self, 'model_disconnected', False):
self.model_disconnected = True
LOG.exception(_LE('model server went away'))
# NOTE(jsbryant) Other DB errors can happen in HA configurations.
# such errors shouldn't kill this thread, so we handle them here.
except db_exc.DBError:
if not getattr(self, 'model_disconnected', False):
self.model_disconnected = True
LOG.exception(_LE('DBError encountered: '))
except Exception:
if not getattr(self, 'model_disconnected', False):
self.model_disconnected = True
LOG.exception(_LE('Exception encountered: '))
def load_plugin(namespace, plugin_name, *args, **kwargs):
try:
LOG.debug('Start load plugin %s. ', plugin_name)
# Try to resolve plugin by name
mgr = driver.DriverManager(namespace, plugin_name)
plugin_class = mgr.driver
except RuntimeError as e1:
# fallback to class name
try:
plugin_class = importutils.import_class(plugin_name)
except ImportError as e2:
LOG.error(_LE("Error loading plugin by name, %s"), e1)
LOG.error(_LE("Error loading plugin by class, %s"), e2)
raise ImportError(_("Class not found."))
return plugin_class(*args, **kwargs)
def list_protectable_dependents(self, context, protectable_id,
protectable_type):
LOG.info(
_LI("Start to list dependents of resource "
"(type:%(type)s, id:%(id)s)"), {
'type': protectable_type,
'id': protectable_id
})
parent_resource = Resource(type=protectable_type,
id=protectable_id,
name="")
try:
dependent_resources = \
self.protectable_registry.fetch_dependent_resources(
context, parent_resource)
except exception.ListProtectableResourceFailed as err:
LOG.error(
_LE("List dependent resources of (%(res)s) "
"failed: %(err)s"), {
'res': parent_resource,
'err': six.text_type(err)
})
raise
result = []
for resource in dependent_resources:
result.append(
dict(type=resource.type, id=resource.id, name=resource.name))
return result
def create(context, conf):
conf.register_opts(nova_client_opts, group=SERVICE + '_client')
try:
url = utils.get_url(SERVICE,
context,
conf,
append_project_fmt='%(url)s/%(project)s')
except Exception:
LOG.error(_LE("Get nova service endpoint url failed."))
raise
LOG.info(_LI('Creating nova client with url %s.'), url)
extensions = nc.discover_extensions(NOVACLIENT_VERSION)
args = {
'project_id': context.project_id,
'auth_token': context.auth_token,
'extensions': extensions,
'cacert': conf.nova_client.nova_ca_cert_file,
'insecure': conf.nova_client.nova_auth_insecure,
}
client = nc.Client(NOVACLIENT_VERSION, **args)
client.client.set_management_url(url)
return client
def __exit__(self, ex_type, ex_value, ex_traceback):
if not ex_value:
return True
if isinstance(ex_value, exception.NotAuthorized):
raise Fault(webob.exc.HTTPForbidden(explanation=ex_value.msg))
elif isinstance(ex_value, exception.Invalid):
raise Fault(
exception.ConvertedException(code=ex_value.code,
explanation=ex_value.msg))
elif isinstance(ex_value, TypeError):
exc_info = (ex_type, ex_value, ex_traceback)
LOG.error(_LE('Exception handling resource: %s'),
ex_value,
exc_info=exc_info)
raise Fault(webob.exc.HTTPBadRequest())
elif isinstance(ex_value, Fault):
LOG.info(_LI("Fault thrown: %s"), ex_value)
raise ex_value
elif isinstance(ex_value, webob.exc.HTTPException):
LOG.info(_LI("HTTP exception thrown: %s"), ex_value)
raise Fault(ex_value)
# We didn't handle the exception
return False
def get_method(self, request, action, content_type, body):
"""Look up the action-specific method and its extensions."""
# Look up the method
try:
if not self.controller:
meth = getattr(self, action)
else:
meth = getattr(self.controller, action)
except AttributeError as e:
with excutils.save_and_reraise_exception(e) as ctxt:
if (not self.wsgi_actions or action
not in ['action', 'create', 'delete', 'update']):
LOG.exception(_LE('Get method error.'))
else:
ctxt.reraise = False
else:
return meth, self.wsgi_extensions.get(action, [])
if action == 'action':
# OK, it's an action; figure out which action...
mtype = _MEDIA_TYPE_MAP.get(content_type)
action_name = self.action_peek[mtype](body)
LOG.debug("Action body: %s", body)
else:
action_name = action
# Look up the action method
return (self.wsgi_actions[action_name],
self.wsgi_action_extensions.get(action_name, []))
def link_task(self, flow, u, v):
if flow is None:
LOG.error(_LE("The flow is None, build it first"))
raise exception.InvalidTaskFlowObject(
reason=_("The flow is None"))
if u and v:
flow.link(u, v)
def delete_backup(self, cntxt, checkpoint, **kwargs):
resource_node = kwargs.get("node")
resource_id = resource_node.value.id
bank_section = checkpoint.get_resource_bank_section(resource_id)
cinder_client = self._cinder_client(cntxt)
LOG.info(_LI("deleting volume backup, volume_id: %s."), resource_id)
try:
bank_section.update_object("status",
constants.RESOURCE_STATUS_DELETING)
resource_definition = bank_section.get_object("metadata")
backup_id = resource_definition["backup_id"]
cinder_client.backups.delete(backup_id)
bank_section.delete_object("metadata")
self.protection_resource_map[resource_id] = {
"bank_section": bank_section,
"backup_id": backup_id,
"cinder_client": cinder_client,
"operation": "delete"
}
except Exception as e:
LOG.error(_LE("delete volume backup failed, volume_id: %s."),
resource_id)
bank_section.update_object("status",
constants.CHECKPOINT_STATUS_ERROR)
raise exception.DeleteBackupFailed(
reason=six.text_type(e),
resource_id=resource_id,
resource_type=constants.VOLUME_RESOURCE_TYPE
)
def sync_status(self):
for resource_id, resource_info in self.protection_resource_map.items():
backup_id = resource_info["backup_id"]
bank_section = resource_info["bank_section"]
cinder_client = resource_info["cinder_client"]
operation = resource_info["operation"]
try:
backup = cinder_client.backups.get(backup_id)
if backup.status == "available":
bank_section.update_object(
"status", constants.RESOURCE_STATUS_AVAILABLE)
self.protection_resource_map.pop(resource_id)
elif backup.status in ["error", "error-deleting"]:
bank_section.update_object(
"status", constants.RESOURCE_STATUS_ERROR)
self.protection_resource_map.pop(resource_id)
else:
continue
except Exception as exc:
if operation == "delete" and type(exc) == NotFound:
bank_section.update_object(
"status",
constants.RESOURCE_STATUS_DELETED)
LOG.info(_LI("deleting volume backup finished, "
"backup id: %s"), backup_id)
else:
LOG.error(_LE("deleting volume backup error.exc: %s"),
six.text_type(exc))
self.protection_resource_map.pop(resource_id)
def restore_backup(self, cntxt, checkpoint, **kwargs):
resource_node = kwargs.get("node")
resource_id = resource_node.value.id
heat_template = kwargs.get("heat_template")
name = kwargs.get("restore_name",
"%s@%s" % (checkpoint.id, resource_id))
description = kwargs.get("restore_description")
heat_resource_id = uuidutils.generate_uuid()
heat_resource = HeatResource(heat_resource_id,
constants.VOLUME_RESOURCE_TYPE)
bank_section = checkpoint.get_resource_bank_section(resource_id)
try:
resource_definition = bank_section.get_object("metadata")
backup_id = resource_definition["backup_id"]
properties = {"backup_id": backup_id,
"name": name}
if description is not None:
properties["description"] = description
for key, value in properties.items():
heat_resource.set_property(key, value)
heat_template.put_resource(resource_id, heat_resource)
except Exception as e:
LOG.error(_LE("restore volume backup failed, volume_id: %s."),
resource_id)
raise exception.RestoreBackupFailed(
reason=six.text_type(e),
resource_id=resource_id,
resource_type=constants.VOLUME_RESOURCE_TYPE
)
def list_protectable_instances(self, context,
protectable_type=None,
marker=None,
limit=None,
sort_keys=None,
sort_dirs=None,
filters=None,
parameters=None):
LOG.info(_LI("Start to list protectable instances of type: %s"),
protectable_type)
try:
resource_instances = self.protectable_registry.list_resources(
context, protectable_type, parameters)
except exception.ListProtectableResourceFailed as err:
LOG.error(_LE("List resources of type %(type)s failed: %(err)s"),
{'type': protectable_type,
'err': six.text_type(err)})
raise
result = []
for resource in resource_instances:
result.append(dict(id=resource.id, name=resource.name))
return result
def list_protectable_dependents(self, context,
protectable_id,
protectable_type):
LOG.info(_LI("Start to list dependents of resource "
"(type:%(type)s, id:%(id)s)"),
{'type': protectable_type,
'id': protectable_id})
parent_resource = Resource(type=protectable_type, id=protectable_id,
name="")
try:
dependent_resources = \
self.protectable_registry.fetch_dependent_resources(
context, parent_resource)
except exception.ListProtectableResourceFailed as err:
LOG.error(_LE("List dependent resources of (%(res)s) "
"failed: %(err)s"),
{'res': parent_resource,
'err': six.text_type(err)})
raise
result = []
for resource in dependent_resources:
result.append(dict(type=resource.type, id=resource.id,
name=resource.name))
return result
def restore(self, context, restore, restore_auth):
LOG.info(_LI("Starting restore service:restore action"))
checkpoint_id = restore["checkpoint_id"]
provider_id = restore["provider_id"]
provider = self.provider_registry.show_provider(provider_id)
if not provider:
raise exception.ProviderNotFound(provider_id=provider_id)
checkpoint_collection = provider.get_checkpoint_collection()
checkpoint = checkpoint_collection.get(checkpoint_id)
if checkpoint.status != constants.CHECKPOINT_STATUS_AVAILABLE:
raise exception.CheckpointNotAvailable(
checkpoint_id=checkpoint_id)
try:
restoration_flow = self.worker.get_restoration_flow(
context,
constants.OPERATION_RESTORE,
checkpoint,
provider,
restore,
restore_auth)
except Exception:
LOG.exception(
_LE("Failed to create restoration flow checkpoint: %s"),
checkpoint_id)
raise exception.FlowError(
flow="restore",
error=_("Failed to create flow"))
self._spawn(self.worker.run_flow, restoration_flow)
def output(self, flow_engine, target=None):
if flow_engine is None:
LOG.error(_LE("Flow engine is None,return nothing"))
return
if target:
return flow_engine.storage.fetch(target)
return flow_engine.storage.fetch_all()
def show_protectable_instance(self,
context,
protectable_type,
protectable_id,
parameters=None):
LOG.info(_LI("Start to show protectable instance of type: %s"),
protectable_type)
try:
resource_instance = \
self.protectable_registry.show_resource(context,
protectable_type,
protectable_id,
parameters=parameters)
except exception.ListProtectableResourceFailed as err:
LOG.error(
_LE("Show resources of type %(type)s id %(id)s "
"failed: %(err)s"), {
'type': protectable_type,
'id': protectable_id,
'err': six.text_type(err)
})
raise
return dict(id=resource_instance.id,
name=resource_instance.name,
type=resource_instance.type)
def list_protectable_instances(self,
context,
protectable_type=None,
marker=None,
limit=None,
sort_keys=None,
sort_dirs=None,
filters=None,
parameters=None):
LOG.info(_LI("Start to list protectable instances of type: %s"),
protectable_type)
try:
resource_instances = self.protectable_registry.list_resources(
context, protectable_type, parameters)
except exception.ListProtectableResourceFailed as err:
LOG.error(_LE("List resources of type %(type)s failed: %(err)s"), {
'type': protectable_type,
'err': six.text_type(err)
})
raise
result = []
for resource in resource_instances:
result.append(dict(id=resource.id, name=resource.name))
return result
def create(context, conf):
register_opts(conf)
if hasattr(conf.swift_client, 'swift_auth_url') and \
conf.swift_client.swift_auth_url:
connection = swift.Connection(
authurl=conf.swift_client.swift_auth_url,
auth_version=conf.swift_client.swift_auth_version,
tenant_name=conf.swift_client.swift_tenant_name,
user=conf.swift_client.swift_user,
key=conf.swift_client.swift_key,
retries=conf.swift_client.swift_retry_attempts,
starting_backoff=conf.swift_client.swift_retry_backoff,
insecure=conf.swift_client.swift_auth_insecure,
cacert=conf.swift_client.swift_ca_cert_file)
else:
try:
url = utils.get_url(SERVICE,
context,
conf,
append_project_fmt='%(url)s/AUTH_%(project)s')
except Exception:
LOG.error(_LE("Get swift service endpoint url failed"))
raise
LOG.info(_LI("Creating swift client with url %s."), url)
connection = swift.Connection(
preauthurl=url,
preauthtoken=context.auth_token,
retries=conf.swift_client.swift_retry_attempts,
starting_backoff=conf.swift_client.swift_retry_backoff,
insecure=conf.swift_client.swift_auth_insecure,
cacert=conf.swift_client.swift_ca_cert_file)
return connection
def create_backup(self, cntxt, checkpoint, **kwargs):
resource_node = kwargs.get("node")
image_id = resource_node.value.id
bank_section = checkpoint.get_resource_bank_section(image_id)
resource_definition = {"resource_id": image_id}
glance_client = self._glance_client(cntxt)
LOG.info(_LI("creating image backup, image_id: %s."), image_id)
try:
bank_section.create_object("status",
constants.RESOURCE_STATUS_PROTECTING)
image_info = glance_client.images.get(image_id)
image_metadata = {
"disk_format": image_info.disk_format,
"container_format": image_info.container_format
}
resource_definition["image_metadata"] = image_metadata
resource_definition["backup_id"] = image_id
bank_section.create_object("metadata", resource_definition)
except Exception as err:
LOG.error(_LE("create image backup failed, image_id: %s."),
image_id)
bank_section.update_object("status",
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateBackupFailed(
reason=err,
resource_id=image_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
self._add_to_threadpool(self._create_backup, glance_client,
bank_section, image_id)
def create_backup(self, cntxt, checkpoint, **kwargs):
resource_node = kwargs.get("node")
image_id = resource_node.value.id
bank_section = checkpoint.get_resource_bank_section(image_id)
resource_definition = {"resource_id": image_id}
glance_client = self._glance_client(cntxt)
LOG.info(_LI("creating image backup, image_id: %s."), image_id)
try:
bank_section.create_object("status",
constants.RESOURCE_STATUS_PROTECTING)
image_info = glance_client.images.get(image_id)
image_metadata = {
"disk_format": image_info.disk_format,
"container_format": image_info.container_format
}
resource_definition["image_metadata"] = image_metadata
resource_definition["backup_id"] = image_id
bank_section.create_object("metadata", resource_definition)
except Exception as err:
LOG.error(_LE("create image backup failed, image_id: %s."),
image_id)
bank_section.update_object("status",
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateBackupFailed(
reason=err,
resource_id=image_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
self._add_to_threadpool(self._create_backup, glance_client,
bank_section, image_id)
def get_object(self, key):
try:
return self._get_object(container=self.bank_object_container,
obj=key)
except SwiftConnectionFailed as err:
LOG.error(_LE("get object failed, err: %s."), err)
raise exception.BankGetObjectFailed(reason=err, key=key)
def sync_status(self):
for resource_id, resource_info in self.protection_resource_map.items():
backup_id = resource_info["backup_id"]
bank_section = resource_info["bank_section"]
cinder_client = resource_info["cinder_client"]
operation = resource_info["operation"]
try:
backup = cinder_client.backups.get(backup_id)
if backup.status == "available":
bank_section.update_object(
"status", constants.RESOURCE_STATUS_AVAILABLE)
self.protection_resource_map.pop(resource_id)
elif backup.status in ["error", "error-deleting"]:
bank_section.update_object("status",
constants.RESOURCE_STATUS_ERROR)
self.protection_resource_map.pop(resource_id)
else:
continue
except Exception as exc:
if operation == "delete" and type(exc) == NotFound:
bank_section.update_object(
"status", constants.RESOURCE_STATUS_DELETED)
LOG.info(
_LI("deleting volume backup finished, "
"backup id: %s"), backup_id)
else:
LOG.error(_LE("deleting volume backup error.exc: %s"),
six.text_type(exc))
self.protection_resource_map.pop(resource_id)
def restore_backup(self, cntxt, checkpoint, **kwargs):
resource_node = kwargs.get("node")
resource_id = resource_node.value.id
heat_template = kwargs.get("heat_template")
name = kwargs.get("restore_name",
"%s@%s" % (checkpoint.id, resource_id))
description = kwargs.get("restore_description")
heat_resource_id = uuidutils.generate_uuid()
heat_resource = HeatResource(heat_resource_id,
constants.VOLUME_RESOURCE_TYPE)
bank_section = checkpoint.get_resource_bank_section(resource_id)
try:
resource_definition = bank_section.get_object("metadata")
backup_id = resource_definition["backup_id"]
properties = {"backup_id": backup_id, "name": name}
if description is not None:
properties["description"] = description
for key, value in properties.items():
heat_resource.set_property(key, value)
heat_template.put_resource(resource_id, heat_resource)
except Exception as e:
LOG.error(_LE("restore volume backup failed, volume_id: %s."),
resource_id)
raise exception.RestoreBackupFailed(
reason=six.text_type(e),
resource_id=resource_id,
resource_type=constants.VOLUME_RESOURCE_TYPE)
def create(context, conf):
register_opts(conf)
if hasattr(conf.swift_client, 'swift_auth_url') and \
conf.swift_client.swift_auth_url:
connection = swift.Connection(
authurl=conf.swift_client.swift_auth_url,
auth_version=conf.swift_client.swift_auth_version,
tenant_name=conf.swift_client.swift_tenant_name,
user=conf.swift_client.swift_user,
key=conf.swift_client.swift_key,
retries=conf.swift_client.swift_retry_attempts,
starting_backoff=conf.swift_client.swift_retry_backoff,
insecure=conf.swift_client.swift_auth_insecure,
cacert=conf.swift_client.swift_ca_cert_file)
else:
try:
url = utils.get_url(SERVICE, context, conf,
append_project_fmt='%(url)s/AUTH_%(project)s')
except Exception:
LOG.error(_LE("Get swift service endpoint url failed"))
raise
LOG.info(_LI("Creating swift client with url %s."), url)
connection = swift.Connection(
preauthurl=url,
preauthtoken=context.auth_token,
retries=conf.swift_client.swift_retry_attempts,
starting_backoff=conf.swift_client.swift_retry_backoff,
insecure=conf.swift_client.swift_auth_insecure,
cacert=conf.swift_client.swift_ca_cert_file)
return connection
def get_method(self, request, action, content_type, body):
"""Look up the action-specific method and its extensions."""
# Look up the method
try:
if not self.controller:
meth = getattr(self, action)
else:
meth = getattr(self.controller, action)
except AttributeError as e:
with excutils.save_and_reraise_exception(e) as ctxt:
if (not self.wsgi_actions or action not in ['action',
'create',
'delete',
'update']):
LOG.exception(_LE('Get method error.'))
else:
ctxt.reraise = False
else:
return meth, self.wsgi_extensions.get(action, [])
if action == 'action':
# OK, it's an action; figure out which action...
mtype = _MEDIA_TYPE_MAP.get(content_type)
action_name = self.action_peek[mtype](body)
LOG.debug("Action body: %s", body)
else:
action_name = action
# Look up the action method
return (self.wsgi_actions[action_name],
self.wsgi_action_extensions.get(action_name, []))
def output(self, flow_engine, target=None):
if flow_engine is None:
LOG.error(_LE("Flow engine is None,return nothing"))
return
if target:
return flow_engine.storage.fetch(target)
return flow_engine.storage.fetch_all()
def create_heat_client_with_auth_url(context, conf, **kwargs):
auth_url = kwargs["auth_url"]
username = kwargs["username"]
password = kwargs["password"]
tenant_name = context.project_name
cacert = conf.heat_client.heat_ca_cert_file
insecure = conf.heat_client.heat_auth_insecure
LOG.info(_LI('Creating heat client with auth url %s.'), auth_url)
try:
keystone = kc_v3.Client(version=KEYSTONECLIENT_VERSION,
username=username,
tenant_name=tenant_name,
password=password,
auth_url=auth_url)
auth_token = keystone.auth_token
heat_endpoint = ''
services = keystone.service_catalog.catalog['catalog']
for service in services:
if service['name'] == 'heat':
endpoints = service['endpoints']
for endpoint in endpoints:
if endpoint['interface'] == 'public':
heat_endpoint = endpoint['url']
heat = hc.Client(HEATCLIENT_VERSION,
endpoint=heat_endpoint,
token=auth_token,
cacert=cacert,
insecure=insecure)
return heat
except Exception:
LOG.error(_LE('Creating heat client with url %s.'), auth_url)
raise
def delete_backup(self, cntxt, checkpoint, **kwargs):
resource_node = kwargs.get("node")
image_id = resource_node.value.id
bank_section = checkpoint.get_resource_bank_section(image_id)
LOG.info(_LI("deleting image backup failed, image_id: %s."), image_id)
try:
bank_section.update_object("status",
constants.RESOURCE_STATUS_DELETING)
objects = bank_section.list_objects()
for obj in objects:
if obj == "status":
continue
bank_section.delete_object(obj)
bank_section.update_object("status",
constants.RESOURCE_STATUS_DELETED)
except Exception as err:
LOG.error(_LE("delete image backup failed, image_id: %s."),
image_id)
bank_section.update_object("status",
constants.RESOURCE_STATUS_ERROR)
raise exception.DeleteBackupFailed(
reason=err,
resource_id=image_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
def delete_backup(self, cntxt, checkpoint, **kwargs):
resource_node = kwargs.get("node")
image_id = resource_node.value.id
bank_section = checkpoint.get_resource_bank_section(image_id)
LOG.info(_LI("deleting image backup failed, image_id: %s."),
image_id)
try:
bank_section.update_object("status",
constants.RESOURCE_STATUS_DELETING)
objects = bank_section.list_objects()
for obj in objects:
if obj == "status":
continue
bank_section.delete_object(obj)
bank_section.update_object("status",
constants.RESOURCE_STATUS_DELETED)
except Exception as err:
LOG.error(_LE("delete image backup failed, image_id: %s."),
image_id)
bank_section.update_object("status",
constants.RESOURCE_STATUS_ERROR)
raise exception.DeleteBackupFailed(
reason=err,
resource_id=image_id,
resource_type=constants.IMAGE_RESOURCE_TYPE)
def delete_backup(self, cntxt, checkpoint, **kwargs):
resource_node = kwargs.get("node")
resource_id = resource_node.value.id
bank_section = checkpoint.get_resource_bank_section(resource_id)
cinder_client = self._cinder_client(cntxt)
LOG.info(_LI("deleting volume backup, volume_id: %s."), resource_id)
try:
bank_section.update_object("status",
constants.RESOURCE_STATUS_DELETING)
resource_definition = bank_section.get_object("metadata")
backup_id = resource_definition["backup_id"]
cinder_client.backups.delete(backup_id)
bank_section.delete_object("metadata")
self.protection_resource_map[resource_id] = {
"bank_section": bank_section,
"backup_id": backup_id,
"cinder_client": cinder_client,
"operation": "delete"
}
except Exception as e:
LOG.error(_LE("delete volume backup failed, volume_id: %s."),
resource_id)
bank_section.update_object("status",
constants.CHECKPOINT_STATUS_ERROR)
raise exception.DeleteBackupFailed(
reason=six.text_type(e),
resource_id=resource_id,
resource_type=constants.VOLUME_RESOURCE_TYPE)
def __init__(self, message=None, **kwargs):
"""Initiate the instance of KarborException
There are two ways to initiate the instance.
1. Specify the value of 'message' and leave the 'kwargs' None.
2. Leave 'message' None, and specify the keyword arguments matched
with the format of KarborException.message. Especially, can't
use the 'message' as the key in the 'kwargs', otherwise, the
first argument('message') will be set.
Note: This class doesn't support to create instance of KarborException
with another instance.
"""
self.kwargs = kwargs
if 'code' not in self.kwargs:
try:
self.kwargs['code'] = self.code
except AttributeError:
pass
if not message:
try:
message = self.message % kwargs
except Exception:
exc_info = sys.exc_info()
# kwargs doesn't match a variable in the message
# log the issue and the kwargs
LOG.exception(_LE('Exception in string format operation'))
for name, value in kwargs.items():
LOG.error(_LE("%(name)s: %(value)s"), {
'name': name,
'value': value
})
if CONF.fatal_exception_format_errors:
six.reraise(*exc_info)
# at least get the core message out if something happened
message = self.message
elif isinstance(message, Exception):
message = six.text_type(message)
# NOTE(luisg): We put the actual message in 'msg' so that we can access
# it, because if we try to access the message via 'message' it will be
# overshadowed by the class' message attribute
self.msg = message
super(KarborException, self).__init__(message)
def output(self, flow_engine, target=None):
if flow_engine is None:
LOG.error(_LE("Flow engine is None,return nothing"))
raise exception.InvalidTaskFlowObject(
reason=_("The flow_engine is None"))
if target:
return flow_engine.storage.fetch(target)
return flow_engine.storage.fetch_all()
def _get_protection_plugin(self, resource_type):
for plugin in self.plugin_map.values():
if hasattr(plugin, "get_supported_resources_types"):
if resource_type in plugin.get_supported_resources_types():
return plugin
LOG.error(_LE("no plugin support this resource_type:%s"),
resource_type)
raise Exception(_("No plugin support this resource_type"))
def build_flow(self, flow_name, flow_type='graph'):
if flow_type == 'linear':
return linear_flow.Flow(flow_name)
elif flow_type == 'graph':
return graph_flow.Flow(flow_name)
else:
LOG.error(_LE("unsupported flow type:%s"), flow_type)
return
def output(self, flow_engine, target=None):
if flow_engine is None:
LOG.error(_LE("Flow engine is None,return nothing"))
raise exception.InvalidTaskFlowObject(
reason=_("The flow_engine is None"))
if target:
return flow_engine.storage.fetch(target)
return flow_engine.storage.fetch_all()
def get_object(self, key):
try:
return self._get_object(container=self.bank_object_container,
obj=key)
except SwiftConnectionFailed as err:
LOG.error(_LE("get object failed, err: %s."), err)
raise exception.BankGetObjectFailed(reason=err,
key=key)
def build_flow(self, flow_name, flow_type='graph'):
if flow_type == 'linear':
return linear_flow.Flow(flow_name)
elif flow_type == 'graph':
return graph_flow.Flow(flow_name)
else:
LOG.error(_LE("unsupported flow type:%s"), flow_type)
return
def delete_object(self, key):
try:
self._delete_object(container=self.bank_object_container,
obj=key)
except SwiftConnectionFailed as err:
LOG.error(_LE("delete object failed, err: %s."), err)
raise exception.BankDeleteObjectFailed(reason=err,
key=key)
def start(self):
"""Start serving a WSGI application.
:returns: None
:raises: karbor.exception.InvalidInput
"""
# The server socket object will be closed after server exits,
# but the underlying file descriptor will remain open, and will
# give bad file descriptor error. So duplicating the socket object,
# to keep file descriptor usable.
dup_socket = self._socket.dup()
dup_socket.setsockopt(socket.SOL_SOCKET,
socket.SO_REUSEADDR, 1)
# NOTE(praneshp): Call set_tcp_keepalive in oslo to set
# tcp keepalive parameters. Sockets can hang around forever
# without keepalive
netutils.set_tcp_keepalive(dup_socket,
CONF.tcp_keepalive,
CONF.tcp_keepidle,
CONF.tcp_keepalive_count,
CONF.tcp_keepalive_interval)
if self._use_ssl:
try:
ssl_kwargs = {
'server_side': True,
'certfile': CONF.ssl_cert_file,
'keyfile': CONF.ssl_key_file,
'cert_reqs': ssl.CERT_NONE,
}
if CONF.ssl_ca_file:
ssl_kwargs['ca_certs'] = CONF.ssl_ca_file
ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED
dup_socket = ssl.wrap_socket(dup_socket,
**ssl_kwargs)
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_LE("Failed to start %(name)s on %(_host)s: "
"%(_port)s with SSL "
"support."), self.__dict__)
wsgi_kwargs = {
'func': eventlet.wsgi.server,
'sock': dup_socket,
'site': self.app,
'protocol': self._protocol,
'custom_pool': self._pool,
'log': self._logger,
'socket_timeout': self.client_socket_timeout,
'keepalive': CONF.wsgi_keep_alive
}
self._server = eventlet.spawn(**wsgi_kwargs)
def __init__(self, config, context=None):
super(SwiftBankPlugin, self).__init__(config)
self._config.register_opts(swift_bank_plugin_opts,
"swift_bank_plugin")
self._config.register_opts(lease_opt,
"swift_bank_plugin")
self.bank_object_container = \
self._config.swift_bank_plugin.bank_swift_object_container
self.lease_expire_window = \
self._config.swift_bank_plugin.lease_expire_window
self.lease_renew_window = \
self._config.swift_bank_plugin.lease_renew_window
self.context = context
# TODO(luobin):
# init lease_validity_window
# according to lease_renew_window if not configured
self.lease_validity_window = \
self._config.swift_bank_plugin.lease_validity_window
# TODO(luobin): create a uuid of this bank_plugin
self.owner_id = uuidutils.generate_uuid()
self.lease_expire_time = 0
self.bank_leases_container = "leases"
self.connection = self._setup_connection()
# create container
try:
self._put_container(self.bank_object_container)
self._put_container(self.bank_leases_container)
except SwiftConnectionFailed as err:
LOG.error(_LE("bank plugin create container failed."))
raise exception.CreateContainerFailed(reason=err)
# acquire lease
try:
self.acquire_lease()
except exception.AcquireLeaseFailed:
LOG.error(_LE("bank plugin acquire lease failed."))
raise
# start renew lease
renew_lease_loop = loopingcall.FixedIntervalLoopingCall(
self.renew_lease)
renew_lease_loop.start(interval=self.lease_renew_window,
initial_delay=self.lease_renew_window)
def __init__(self, message=None, **kwargs):
"""Initiate the instance of KarborException
There are two ways to initiate the instance.
1. Specify the value of 'message' and leave the 'kwargs' None.
2. Leave 'message' None, and specify the keyword arguments matched
with the format of KarborException.message. Especially, can't
use the 'message' as the key in the 'kwargs', otherwise, the
first argument('message') will be set.
Note: This class doesn't support to create instance of KarborException
with another instance.
"""
self.kwargs = kwargs
if 'code' not in self.kwargs:
try:
self.kwargs['code'] = self.code
except AttributeError:
pass
if not message:
try:
message = self.message % kwargs
except Exception:
exc_info = sys.exc_info()
# kwargs doesn't match a variable in the message
# log the issue and the kwargs
LOG.exception(_LE('Exception in string format operation'))
for name, value in kwargs.items():
LOG.error(_LE("%(name)s: %(value)s"),
{'name': name, 'value': value})
if CONF.fatal_exception_format_errors:
six.reraise(*exc_info)
# at least get the core message out if something happened
message = self.message
elif isinstance(message, Exception):
message = six.text_type(message)
# NOTE(luisg): We put the actual message in 'msg' so that we can access
# it, because if we try to access the message via 'message' it will be
# overshadowed by the class' message attribute
self.msg = message
super(KarborException, self).__init__(message)
def search_task(self, flow, task_id):
if not isinstance(flow, graph_flow.Flow):
LOG.error(_LE("this is not a graph flow,flow name:%s"), flow.name)
return
for node, meta in flow.iter_nodes():
if not isinstance(node, task.FunctorTask):
continue
if task_id == getattr(node, 'name'):
return node
def search_task(self, flow, task_id):
if not isinstance(flow, graph_flow.Flow):
LOG.error(_LE("this is not a graph flow,flow name:%s"), flow.name)
return
for node, meta in flow.iter_nodes():
if not isinstance(node, task.FunctorTask):
continue
if task_id == getattr(node, 'name'):
return node
def reload_meta_data(self):
try:
new_md = self._checkpoint_section.get_object(_INDEX_FILE_NAME)
except exception.BankGetObjectFailed:
LOG.error(_LE("unable to reload metadata for checkpoint id: %s"),
self.id)
raise exception.CheckpointNotFound(checkpoint_id=self.id)
self._assert_supported_version(new_md)
self._md_cache = new_md
def reload_meta_data(self):
try:
new_md = self._checkpoint_section.get_object(_INDEX_FILE_NAME)
except exception.BankGetObjectFailed:
LOG.error(_LE("unable to reload metadata for checkpoint id: %s"),
self.id)
raise exception.CheckpointNotFound(checkpoint_id=self.id)
self._assert_supported_version(new_md)
self._md_cache = new_md
def _load_bank(self, bank_name):
try:
plugin = utils.load_plugin(PROTECTION_NAMESPACE, bank_name,
self._config)
except Exception:
LOG.exception(_LE("Load bank plugin: '%s' failed."), bank_name)
raise
else:
self._bank_plugin = plugin
def _raise_unknown_exception(self, exception_instance):
LOG.exception(_LE('An unknown exception happened'))
value = exception_instance.msg if isinstance(
exception_instance, exception.KarborException) else type(
exception_instance)
msg = (_('Unexpected API Error. Please report this at '
'http://bugs.launchpad.net/karbor/ and attach the '
'Karbor API log if possible.\n%s') % value)
raise exc.HTTPInternalServerError(explanation=msg)
def _load_bank(self, bank_name):
try:
plugin = utils.load_plugin(PROTECTION_NAMESPACE, bank_name,
self._config)
except Exception:
LOG.exception(_LE("Load bank plugin: '%s' failed."),
bank_name)
raise
else:
self._bank_plugin = plugin
def _raise_unknown_exception(self, exception_instance):
LOG.exception(_LE('An unknown exception happened'))
value = exception_instance.msg if isinstance(
exception_instance,
exception.KarborException) else type(exception_instance)
msg = (_('Unexpected API Error. Please report this at '
'http://bugs.launchpad.net/karbor/ and attach the '
'Karbor API log if possible.\n%s') % value)
raise exc.HTTPInternalServerError(explanation=msg)
def _update_operation_log(cls, log_ref, updates):
if not log_ref:
return
for item in updates:
setattr(log_ref, item, updates.get(item))
try:
log_ref.save()
except Exception:
LOG.exception(_LE("Execute operation(%s), save log failed"), log_ref.operation_id)
def get_resource_reference(self, original_id):
if original_id in self._original_id_resource_map:
return {
"get_resource": (self._original_id_resource_map[original_id])
}
elif original_id in self._original_id_parameter_map:
return self._original_id_parameter_map[original_id]
else:
LOG.error(_LE("The reference is not found, original_id:%s"),
original_id)
raise InvalidOriginalId
def _list_clients():
clients_dir = os.path.join(os.path.dirname(__file__), "clients")
if not os.path.isdir(clients_dir):
LOG.error(_LE('clients directory "%s" not found'), clients_dir)
return
for file in os.listdir(clients_dir):
name, ext = os.path.splitext(file)
if name != "__init__" and ext == ".py":
LOG.debug('Found client "%s"', name)
yield "%s.clients.%s" % (__package__, name)
def show_resource(self, context, resource_id, parameters=None):
try:
image = self._glance_client(context).images.get(resource_id)
except Exception as e:
LOG.exception(_LE("Show a image from glance failed."))
raise exception.ListProtectableResourceFailed(
type=self._SUPPORT_RESOURCE_TYPE,
reason=six.text_type(e))
else:
return resource.Resource(type=self._SUPPORT_RESOURCE_TYPE,
id=image.id, name=image.name)
