def get_flow(context, workflow_engine, operation_type, plan, provider, checkpoint): protectable_registry = ProtectableRegistry() resources = set(Resource(**item) for item in plan.get("resources")) resource_graph = protectable_registry.build_graph(context, resources) checkpoint.resource_graph = resource_graph checkpoint.commit() flow_name = "Protect_" + plan.get('id') protection_flow = workflow_engine.build_flow(flow_name, 'linear') plugins = provider.load_plugins() resources_task_flow = resource_flow.build_resource_flow( operation_type=operation_type, context=context, workflow_engine=workflow_engine, resource_graph=resource_graph, plugins=plugins, parameters=plan.get('parameters'), ) workflow_engine.add_tasks( protection_flow, InitiateProtectTask(), resources_task_flow, CompleteProtectTask(), ) flow_engine = workflow_engine.get_engine(protection_flow, store={ 'checkpoint': checkpoint }) return flow_engine
def get_flow(context, workflow_engine, operation_type, plan, provider, checkpoint): protectable_registry = ProtectableRegistry() resources = set(Resource(**item) for item in plan.get("resources")) resource_graph = protectable_registry.build_graph(context, resources) checkpoint.resource_graph = resource_graph checkpoint.commit() flow_name = "Protect_" + plan.get('id') protection_flow = workflow_engine.build_flow(flow_name, 'linear') plugins = provider.load_plugins() resources_task_flow = resource_flow.build_resource_flow( operation_type=operation_type, context=context, workflow_engine=workflow_engine, resource_graph=resource_graph, plugins=plugins, parameters=plan.get('parameters'), ) workflow_engine.add_tasks( protection_flow, InitiateProtectTask(), resources_task_flow, CompleteProtectTask(), ) flow_engine = workflow_engine.get_engine(protection_flow, store={'checkpoint': checkpoint}) return flow_engine
def __init__(self, service_name=None, *args, **kwargs): super(ProtectionManager, self).__init__(*args, **kwargs) provider_reg = CONF.provider_registry self.provider_registry = utils.load_plugin(PROVIDER_NAMESPACE, provider_reg) self.protectable_registry = ProtectableRegistry() self.protectable_registry.load_plugins() self.worker = flow_manager.Worker()
def __init__(self, service_name=None, *args, **kwargs): super(ProtectionManager, self).__init__(*args, **kwargs) provider_reg = CONF.provider_registry self.provider_registry = utils.load_plugin(PROVIDER_NAMESPACE, provider_reg) self.protectable_registry = ProtectableRegistry() self.protectable_registry.load_plugins() self.worker = flow_manager.Worker() self._greenpool = None self._greenpool_size = CONF.max_concurrent_operations if self._greenpool_size != 0: self._greenpool = greenpool.GreenPool(self._greenpool_size)
class ProtectableRegistryTest(base.TestCase): def setUp(self): super(ProtectableRegistryTest, self).setUp() self.protectable_registry = ProtectableRegistry() self._fake_plugin = _FakeProtectablePlugin(None) self.protectable_registry.register_plugin(self._fake_plugin) def test_graph_building(self): A = Resource(_FAKE_TYPE, "A", 'nameA') B = Resource(_FAKE_TYPE, "B", 'nameB') C = Resource(_FAKE_TYPE, "C", 'nameC') test_matrix = ( ( {A: [B], B: [C], C: []}, (A, C) ), ( {A: [C], B: [C], C: []}, (A, C) ), ) for g, resources in test_matrix: self._fake_plugin.graph = g result_graph = self.protectable_registry.build_graph(None, resources) self.assert_graph(result_graph, g) self.protectable_registry._protectable_map = {} def assert_graph(self, g, g_dict): for item in g: expected = set(g_dict[item.value]) found = set(child.value for child in item.child_nodes) self.assertEqual(found, expected) self.assert_graph(item.child_nodes, g_dict)
def __init__(self, service_name=None, *args, **kwargs): super(ProtectionManager, self).__init__(*args, **kwargs) provider_reg = CONF.provider_registry self.provider_registry = utils.load_plugin(PROVIDER_NAMESPACE, provider_reg) self.protectable_registry = ProtectableRegistry() self.protectable_registry.load_plugins() self.worker = flow_manager.Worker() self._greenpool = None self._greenpool_size = CONF.max_concurrent_operations if self._greenpool_size != 0: self._greenpool = greenpool.GreenPool(self._greenpool_size)
class ProtectionManager(manager.Manager): """karbor Protection Manager.""" RPC_API_VERSION = '1.0' target = messaging.Target(version=RPC_API_VERSION) def __init__(self, service_name=None, *args, **kwargs): super(ProtectionManager, self).__init__(*args, **kwargs) provider_reg = CONF.provider_registry self.provider_registry = utils.load_plugin(PROVIDER_NAMESPACE, provider_reg) self.protectable_registry = ProtectableRegistry() self.protectable_registry.load_plugins() self.worker = flow_manager.Worker() self._greenpool = None self._greenpool_size = CONF.max_concurrent_operations if self._greenpool_size != 0: self._greenpool = greenpool.GreenPool(self._greenpool_size) def _spawn(self, func, *args, **kwargs): if self._greenpool is not None: return self._greenpool.spawn_n(func, *args, **kwargs) else: return greenthread.spawn_n(func, *args, **kwargs) def init_host(self, **kwargs): """Handle initialization if this is a standalone service""" # TODO(wangliuan) LOG.info("Starting protection service") @messaging.expected_exceptions(exception.InvalidPlan, exception.ProviderNotFound, exception.FlowError) def protect(self, context, plan, checkpoint_properties=None): """create protection for the given plan :param plan: Define that protection plan should be done """ LOG.info("Starting protection service:protect action") LOG.debug("protecting: %s checkpoint_properties:%s", plan, checkpoint_properties) if not plan: raise exception.InvalidPlan( reason=_('the protection plan is None')) provider_id = plan.get('provider_id', None) plan_id = plan.get('id', None) provider = self.provider_registry.show_provider(provider_id) checkpoint_collection = provider.get_checkpoint_collection() try: checkpoint = checkpoint_collection.create(plan, checkpoint_properties, context=context) except Exception as e: LOG.exception("Failed to create checkpoint, plan: %s", plan_id) exc = exception.FlowError(flow="protect", error="Error creating checkpoint") six.raise_from(exc, e) try: flow = self.worker.get_flow( context=context, protectable_registry=self.protectable_registry, operation_type=constants.OPERATION_PROTECT, plan=plan, provider=provider, checkpoint=checkpoint) except Exception as e: LOG.exception("Failed to create protection flow, plan: %s", plan_id) raise exception.FlowError( flow="protect", error=e.msg if hasattr(e, 'msg') else 'Internal error') self._spawn(self.worker.run_flow, flow) return checkpoint.id @messaging.expected_exceptions(exception.InvalidPlan, exception.ProviderNotFound, exception.FlowError) def copy(self, context, plan): """create copy of checkpoint for the given plan :param plan: Define that protection plan should be done """ LOG.info("Starting protection service:copy action.") LOG.debug("Creating the checkpoint copy for the plan: %s", plan) if not plan: raise exception.InvalidPlan( reason=_('The protection plan is None')) provider_id = plan.get('provider_id', None) plan_id = plan.get('id', None) provider = self.provider_registry.show_provider(provider_id) checkpoints = None checkpoint_collection = provider.get_checkpoint_collection() try: checkpoints = self.list_checkpoints(context, provider_id, filters={'plan_id': plan_id}) except Exception as e: LOG.exception("Failed to get checkpoints for the plan: %s", plan_id) exc = exception.FlowError(flow="copy", error="Failed to get checkpoints") six.raise_from(exc, e) try: flow, checkpoint_copy = self.worker.get_flow( context=context, protectable_registry=self.protectable_registry, operation_type=constants.OPERATION_COPY, plan=plan, provider=provider, checkpoint=checkpoints, checkpoint_collection=checkpoint_collection) except Exception as e: LOG.exception("Failed to create copy flow, plan: %s", plan_id) raise exception.FlowError( flow="copy", error=e.msg if hasattr(e, 'msg') else 'Internal error') self._spawn(self.worker.run_flow, flow) return checkpoint_copy @messaging.expected_exceptions(exception.ProviderNotFound, exception.CheckpointNotFound, exception.CheckpointNotAvailable, exception.FlowError, exception.InvalidInput, exception.AccessCheckpointNotAllowed) def restore(self, context, restore, restore_auth): LOG.info("Starting restore service:restore action") checkpoint_id = restore["checkpoint_id"] provider_id = restore["provider_id"] provider = self.provider_registry.show_provider(provider_id) if not provider: raise exception.ProviderNotFound(provider_id=provider_id) self.validate_restore_parameters(restore, provider) checkpoint_collection = provider.get_checkpoint_collection() checkpoint = checkpoint_collection.get(checkpoint_id) if not context.is_admin and (checkpoint.project_id != context.project_id): raise exception.AccessCheckpointNotAllowed( checkpoint_id=checkpoint_id) if checkpoint.status != constants.CHECKPOINT_STATUS_AVAILABLE: raise exception.CheckpointNotAvailable(checkpoint_id=checkpoint_id) try: flow = self.worker.get_flow( context=context, operation_type=constants.OPERATION_RESTORE, checkpoint=checkpoint, provider=provider, restore=restore, restore_auth=restore_auth) except Exception: LOG.exception("Failed to create restore flow checkpoint: %s", checkpoint_id) raise exception.FlowError(flow="restore", error=_("Failed to create flow")) self._spawn(self.worker.run_flow, flow) @messaging.expected_exceptions(exception.ProviderNotFound, exception.CheckpointNotFound, exception.CheckpointNotAvailable, exception.FlowError, exception.InvalidInput) def verification(self, context, verification): LOG.info("Starting verify service:verify action") checkpoint_id = verification["checkpoint_id"] provider_id = verification["provider_id"] provider = self.provider_registry.show_provider(provider_id) if not provider: raise exception.ProviderNotFound(provider_id=provider_id) self.validate_verify_parameters(verification, provider) checkpoint_collection = provider.get_checkpoint_collection() checkpoint = checkpoint_collection.get(checkpoint_id) if checkpoint.status != constants.CHECKPOINT_STATUS_AVAILABLE: raise exception.CheckpointNotAvailable(checkpoint_id=checkpoint_id) try: flow = self.worker.get_flow( context=context, operation_type=constants.OPERATION_VERIFY, checkpoint=checkpoint, provider=provider, verify=verification) except Exception: LOG.exception("Failed to create verify flow checkpoint: %s", checkpoint_id) raise exception.FlowError(flow="verify", error=_("Failed to create flow")) self._spawn(self.worker.run_flow, flow) def validate_restore_parameters(self, restore, provider): parameters = restore["parameters"] if not parameters: return restore_schema = provider.extended_info_schema.get( "restore_schema", None) if restore_schema is None: msg = _("The restore schema of plugin must be provided.") raise exception.InvalidInput(reason=msg) for resource_key, parameter_value in parameters.items(): if "#" in resource_key: resource_type, resource_id = resource_key.split("#") if not uuidutils.is_uuid_like(resource_id): msg = _("The resource_id must be a uuid.") raise exception.InvalidInput(reason=msg) else: resource_type = resource_key if (resource_type not in constants.RESOURCE_TYPES) or ( resource_type not in restore_schema): msg = _("The key of restore parameters is invalid.") raise exception.InvalidInput(reason=msg) properties = restore_schema[resource_type]["properties"] if not set(parameter_value.keys()).issubset(set( properties.keys())): msg = _("The restore property of restore parameters " "is invalid.") raise exception.InvalidInput(reason=msg) def validate_verify_parameters(self, verify, provider): parameters = verify["parameters"] if not parameters: return verify_schema = provider.extended_info_schema.get( "verify_schema", None) if verify_schema is None: msg = _("The verify schema of plugin must be provided.") raise exception.InvalidInput(reason=msg) for resource_key, parameter_value in parameters.items(): if "#" in resource_key: resource_type, resource_id = resource_key.split("#") if not uuidutils.is_uuid_like(resource_id): msg = _("The resource_id must be a uuid.") raise exception.InvalidInput(reason=msg) else: resource_type = resource_key if (resource_type not in constants.RESOURCE_TYPES) or (resource_type not in verify_schema): msg = _("The key of verify parameters is invalid.") raise exception.InvalidInput(reason=msg) properties = verify_schema[resource_type]["properties"] if not set(parameter_value.keys()).issubset(set( properties.keys())): msg = _("The verify property of verify parameters " "is invalid.") raise exception.InvalidInput(reason=msg) @messaging.expected_exceptions(exception.DeleteCheckpointNotAllowed) def delete(self, context, provider_id, checkpoint_id): LOG.info("Starting protection service:delete action") LOG.debug('provider_id :%s checkpoint_id:%s', provider_id, checkpoint_id) provider = self.provider_registry.show_provider(provider_id) try: checkpoint_collection = provider.get_checkpoint_collection() checkpoint = checkpoint_collection.get(checkpoint_id, context=context) except Exception: LOG.error("get checkpoint failed, checkpoint_id:%s", checkpoint_id) raise exception.InvalidInput( reason=_("Invalid checkpoint_id or provider_id")) checkpoint_dict = checkpoint.to_dict() if not context.is_admin and (context.project_id != checkpoint_dict['project_id']): LOG.warn("Delete checkpoint(%s) is not allowed." % checkpoint_id) raise exception.DeleteCheckpointNotAllowed( checkpoint_id=checkpoint_id) if checkpoint.status not in [ constants.CHECKPOINT_STATUS_AVAILABLE, constants.CHECKPOINT_STATUS_ERROR, ]: raise exception.CheckpointNotBeDeleted(checkpoint_id=checkpoint_id) checkpoint.status = constants.CHECKPOINT_STATUS_DELETING checkpoint.commit() try: flow = self.worker.get_flow( context=context, operation_type=constants.OPERATION_DELETE, checkpoint=checkpoint, provider=provider) except Exception: LOG.exception( "Failed to create delete checkpoint flow," "checkpoint:%s.", checkpoint_id) raise exception.KarborException( _("Failed to create delete checkpoint flow.")) self._spawn(self.worker.run_flow, flow) @messaging.expected_exceptions(exception.AccessCheckpointNotAllowed, exception.CheckpointNotBeReset) def reset_state(self, context, provider_id, checkpoint_id, state): provider = self.provider_registry.show_provider(provider_id) checkpoint = provider.get_checkpoint(checkpoint_id, context=context) checkpoint_dict = checkpoint.to_dict() if not context.is_admin and (context.project_id != checkpoint_dict['project_id']): raise exception.AccessCheckpointNotAllowed( checkpoint_id=checkpoint_id) if checkpoint.status not in [ constants.CHECKPOINT_STATUS_AVAILABLE, constants.CHECKPOINT_STATUS_ERROR, constants.CHECKPOINT_STATUS_COPYING, constants.CHECKPOINT_STATUS_WAIT_COPYING, constants.CHECKPOINT_STATUS_COPY_FINISHED ]: raise exception.CheckpointNotBeReset(checkpoint_id=checkpoint_id) checkpoint.status = state checkpoint.commit() def start(self, plan): # TODO(wangliuan) pass def suspend(self, plan): # TODO(wangliuan) pass @messaging.expected_exceptions(exception.ProviderNotFound, exception.CheckpointNotFound, exception.BankListObjectsFailed) def list_checkpoints(self, context, provider_id, marker=None, limit=None, sort_keys=None, sort_dirs=None, filters=None, all_tenants=False): LOG.info("Starting list checkpoints. provider_id:%s", provider_id) plan_id = filters.get("plan_id", None) start_date = None end_date = None if filters.get("start_date", None): start_date = datetime.strptime(filters.get("start_date"), "%Y-%m-%d") if filters.get("end_date", None): end_date = datetime.strptime(filters.get("end_date"), "%Y-%m-%d") sort_dir = None if sort_dirs is None else sort_dirs[0] provider = self.provider_registry.show_provider(provider_id) if filters.get('project_id', None) and all_tenants: project_id = filters.get('project_id') all_tenants = False else: project_id = context.project_id checkpoint_ids = provider.list_checkpoints(project_id, provider_id, limit=limit, marker=marker, plan_id=plan_id, start_date=start_date, end_date=end_date, sort_dir=sort_dir, context=context, all_tenants=all_tenants) checkpoints = [] for checkpoint_id in checkpoint_ids: checkpoint = provider.get_checkpoint(checkpoint_id, context=context) checkpoints.append(checkpoint.to_dict()) return checkpoints @messaging.expected_exceptions(exception.ProviderNotFound, exception.CheckpointNotFound, exception.AccessCheckpointNotAllowed) def show_checkpoint(self, context, provider_id, checkpoint_id): provider = self.provider_registry.show_provider(provider_id) checkpoint = provider.get_checkpoint(checkpoint_id, context=context) checkpoint_dict = checkpoint.to_dict() if not context.is_admin and (context.project_id != checkpoint_dict['project_id']): raise exception.AccessCheckpointNotAllowed( checkpoint_id=checkpoint_id) return checkpoint_dict def list_protectable_types(self, context): LOG.info("Start to list protectable types.") return self.protectable_registry.list_resource_types() @messaging.expected_exceptions(exception.ProtectableTypeNotFound) def show_protectable_type(self, context, protectable_type): LOG.info("Start to show protectable type %s", protectable_type) plugin = self.protectable_registry.get_protectable_resource_plugin( protectable_type) if not plugin: raise exception.ProtectableTypeNotFound( protectable_type=protectable_type) dependents = [] for t in self.protectable_registry.list_resource_types(): if t == protectable_type: continue p = self.protectable_registry.get_protectable_resource_plugin(t) if p and protectable_type in p.get_parent_resource_types(): dependents.append(t) return { 'name': plugin.get_resource_type(), "dependent_types": dependents } @messaging.expected_exceptions(exception.ListProtectableResourceFailed) def list_protectable_instances(self, context, protectable_type=None, marker=None, limit=None, sort_keys=None, sort_dirs=None, filters=None, parameters=None): LOG.info("Start to list protectable instances of type: %s", protectable_type) try: resource_instances = self.protectable_registry.list_resources( context, protectable_type, parameters) except exception.ListProtectableResourceFailed as err: LOG.error("List resources of type %(type)s failed: %(err)s", { 'type': protectable_type, 'err': six.text_type(err) }) raise result = [] for resource in resource_instances: result.append( dict(id=resource.id, name=resource.name, extra_info=resource.extra_info)) return result @messaging.expected_exceptions(exception.ListProtectableResourceFailed) def show_protectable_instance(self, context, protectable_type, protectable_id, parameters=None): LOG.info("Start to show protectable instance of type: %s", protectable_type) registry = self.protectable_registry try: resource_instance = registry.show_resource(context, protectable_type, protectable_id, parameters=parameters) except exception.ListProtectableResourceFailed as err: LOG.error( "Show resources of type %(type)s id %(id)s " "failed: %(err)s", { 'type': protectable_type, 'id': protectable_id, 'err': six.text_type(err) }) raise return resource_instance.to_dict() if resource_instance else None @messaging.expected_exceptions(exception.ListProtectableResourceFailed) def list_protectable_dependents(self, context, protectable_id, protectable_type, protectable_name): LOG.info( "Start to list dependents of resource (type:%(type)s, " "id:%(id)s, name:%(name)s)", { 'type': protectable_type, 'id': protectable_id, 'name': protectable_name }) parent_resource = Resource(type=protectable_type, id=protectable_id, name=protectable_name) registry = self.protectable_registry try: dependent_resources = registry.fetch_dependent_resources( context, parent_resource) except exception.ListProtectableResourceFailed as err: LOG.error("List dependent resources of (%(res)s) failed: %(err)s", { 'res': parent_resource, 'err': six.text_type(err) }) raise return [resource.to_dict() for resource in dependent_resources] def list_providers(self, context, marker=None, limit=None, sort_keys=None, sort_dirs=None, filters=None): return self.provider_registry.list_providers(marker=marker, limit=limit, sort_keys=sort_keys, sort_dirs=sort_dirs, filters=filters) @messaging.expected_exceptions(exception.ProviderNotFound) def show_provider(self, context, provider_id): provider = self.provider_registry.show_provider(provider_id) response = { 'id': provider.id, 'name': provider.name, 'description': provider.description, 'extended_info_schema': provider.extended_info_schema, } return response
class ProtectionManager(manager.Manager): """karbor Protection Manager.""" RPC_API_VERSION = '1.0' target = messaging.Target(version=RPC_API_VERSION) def __init__(self, service_name=None, *args, **kwargs): super(ProtectionManager, self).__init__(*args, **kwargs) provider_reg = CONF.provider_registry self.provider_registry = utils.load_plugin(PROVIDER_NAMESPACE, provider_reg) self.protectable_registry = ProtectableRegistry() self.protectable_registry.load_plugins() self.worker = flow_manager.Worker() self._greenpool = None self._greenpool_size = CONF.max_concurrent_operations if self._greenpool_size != 0: self._greenpool = greenpool.GreenPool(self._greenpool_size) def _spawn(self, func, *args, **kwargs): if self._greenpool is not None: return self._greenpool.spawn_n(func, *args, **kwargs) else: return greenthread.spawn_n(func, *args, **kwargs) def init_host(self, **kwargs): """Handle initialization if this is a standalone service""" # TODO(wangliuan) LOG.info("Starting protection service") @messaging.expected_exceptions(exception.InvalidPlan, exception.ProviderNotFound, exception.FlowError) def protect(self, context, plan, checkpoint_properties=None): """create protection for the given plan :param plan: Define that protection plan should be done """ LOG.info("Starting protection service:protect action") LOG.debug("protecting: %s checkpoint_properties:%s", plan, checkpoint_properties) if not plan: raise exception.InvalidPlan( reason=_('the protection plan is None')) provider_id = plan.get('provider_id', None) plan_id = plan.get('id', None) provider = self.provider_registry.show_provider(provider_id) checkpoint_collection = provider.get_checkpoint_collection() try: checkpoint = checkpoint_collection.create(plan, checkpoint_properties) except Exception as e: LOG.exception("Failed to create checkpoint, plan: %s", plan_id) exc = exception.FlowError(flow="protect", error="Error creating checkpoint") six.raise_from(exc, e) try: flow = self.worker.get_flow( context=context, protectable_registry=self.protectable_registry, operation_type=constants.OPERATION_PROTECT, plan=plan, provider=provider, checkpoint=checkpoint) except Exception as e: LOG.exception("Failed to create protection flow, plan: %s", plan_id) raise exception.FlowError( flow="protect", error=e.msg if hasattr(e, 'msg') else 'Internal error') self._spawn(self.worker.run_flow, flow) return checkpoint.id @messaging.expected_exceptions(exception.ProviderNotFound, exception.CheckpointNotFound, exception.CheckpointNotAvailable, exception.FlowError) def restore(self, context, restore, restore_auth): LOG.info("Starting restore service:restore action") checkpoint_id = restore["checkpoint_id"] provider_id = restore["provider_id"] provider = self.provider_registry.show_provider(provider_id) if not provider: raise exception.ProviderNotFound(provider_id=provider_id) checkpoint_collection = provider.get_checkpoint_collection() checkpoint = checkpoint_collection.get(checkpoint_id) if checkpoint.status != constants.CHECKPOINT_STATUS_AVAILABLE: raise exception.CheckpointNotAvailable(checkpoint_id=checkpoint_id) try: flow = self.worker.get_flow( context=context, operation_type=constants.OPERATION_RESTORE, checkpoint=checkpoint, provider=provider, restore=restore, restore_auth=restore_auth) except Exception: LOG.exception("Failed to create restore flow checkpoint: %s", checkpoint_id) raise exception.FlowError(flow="restore", error=_("Failed to create flow")) self._spawn(self.worker.run_flow, flow) def delete(self, context, provider_id, checkpoint_id): LOG.info("Starting protection service:delete action") LOG.debug('provider_id :%s checkpoint_id:%s', provider_id, checkpoint_id) provider = self.provider_registry.show_provider(provider_id) try: checkpoint_collection = provider.get_checkpoint_collection() checkpoint = checkpoint_collection.get(checkpoint_id) except Exception: LOG.error("get checkpoint failed, checkpoint_id:%s", checkpoint_id) raise exception.InvalidInput( reason=_("Invalid checkpoint_id or provider_id")) if checkpoint.status not in [ constants.CHECKPOINT_STATUS_AVAILABLE, constants.CHECKPOINT_STATUS_ERROR, ]: raise exception.CheckpointNotBeDeleted(checkpoint_id=checkpoint_id) checkpoint.status = constants.CHECKPOINT_STATUS_DELETING checkpoint.commit() try: flow = self.worker.get_flow( context=context, operation_type=constants.OPERATION_DELETE, checkpoint=checkpoint, provider=provider) except Exception: LOG.exception( "Failed to create delete checkpoint flow," "checkpoint:%s.", checkpoint_id) raise exception.KarborException( _("Failed to create delete checkpoint flow.")) self._spawn(self.worker.run_flow, flow) def start(self, plan): # TODO(wangliuan) pass def suspend(self, plan): # TODO(wangliuan) pass @messaging.expected_exceptions(exception.ProviderNotFound, exception.CheckpointNotFound) def list_checkpoints(self, context, provider_id, marker=None, limit=None, sort_keys=None, sort_dirs=None, filters=None): LOG.info("Starting list checkpoints. provider_id:%s", provider_id) plan_id = filters.get("plan_id", None) start_date = None end_date = None if filters.get("start_date", None): start_date = datetime.strptime(filters.get("start_date"), "%Y-%m-%d") if filters.get("end_date", None): end_date = datetime.strptime(filters.get("end_date"), "%Y-%m-%d") sort_dir = None if sort_dirs is None else sort_dirs[0] provider = self.provider_registry.show_provider(provider_id) checkpoint_ids = provider.list_checkpoints(provider_id, limit=limit, marker=marker, plan_id=plan_id, start_date=start_date, end_date=end_date, sort_dir=sort_dir) checkpoints = [] for checkpoint_id in checkpoint_ids: checkpoint = provider.get_checkpoint(checkpoint_id) checkpoints.append(checkpoint.to_dict()) return checkpoints @messaging.expected_exceptions(exception.ProviderNotFound, exception.CheckpointNotFound) def show_checkpoint(self, context, provider_id, checkpoint_id): provider = self.provider_registry.show_provider(provider_id) checkpoint = provider.get_checkpoint(checkpoint_id) return checkpoint.to_dict() def list_protectable_types(self, context): LOG.info("Start to list protectable types.") return self.protectable_registry.list_resource_types() @messaging.expected_exceptions(exception.ProtectableTypeNotFound) def show_protectable_type(self, context, protectable_type): LOG.info("Start to show protectable type %s", protectable_type) plugin = self.protectable_registry.get_protectable_resource_plugin( protectable_type) if not plugin: raise exception.ProtectableTypeNotFound( protectable_type=protectable_type) dependents = [] for t in self.protectable_registry.list_resource_types(): if t == protectable_type: continue p = self.protectable_registry.get_protectable_resource_plugin(t) if p and protectable_type in p.get_parent_resource_types(): dependents.append(t) return { 'name': plugin.get_resource_type(), "dependent_types": dependents } @messaging.expected_exceptions(exception.ListProtectableResourceFailed) def list_protectable_instances(self, context, protectable_type=None, marker=None, limit=None, sort_keys=None, sort_dirs=None, filters=None, parameters=None): LOG.info("Start to list protectable instances of type: %s", protectable_type) try: resource_instances = self.protectable_registry.list_resources( context, protectable_type, parameters) except exception.ListProtectableResourceFailed as err: LOG.error("List resources of type %(type)s failed: %(err)s", { 'type': protectable_type, 'err': six.text_type(err) }) raise result = [] for resource in resource_instances: result.append( dict(id=resource.id, name=resource.name, extra_info=resource.extra_info)) return result @messaging.expected_exceptions(exception.ListProtectableResourceFailed) def show_protectable_instance(self, context, protectable_type, protectable_id, parameters=None): LOG.info("Start to show protectable instance of type: %s", protectable_type) registry = self.protectable_registry try: resource_instance = registry.show_resource(context, protectable_type, protectable_id, parameters=parameters) except exception.ListProtectableResourceFailed as err: LOG.error( "Show resources of type %(type)s id %(id)s " "failed: %(err)s", { 'type': protectable_type, 'id': protectable_id, 'err': six.text_type(err) }) raise return dict(id=resource_instance.id, name=resource_instance.name, type=resource_instance.type, extra_info=resource_instance.extra_info) @messaging.expected_exceptions(exception.ListProtectableResourceFailed) def list_protectable_dependents(self, context, protectable_id, protectable_type): LOG.info( "Start to list dependents of resource (type:%(type)s, " "id:%(id)s)", { 'type': protectable_type, 'id': protectable_id }) parent_resource = Resource(type=protectable_type, id=protectable_id, name="", extra_info=None) registry = self.protectable_registry try: dependent_resources = registry.fetch_dependent_resources( context, parent_resource) except exception.ListProtectableResourceFailed as err: LOG.error("List dependent resources of (%(res)s) failed: %(err)s", { 'res': parent_resource, 'err': six.text_type(err) }) raise result = [] for resource in dependent_resources: result.append( dict(type=resource.type, id=resource.id, name=resource.name, extra_info=resource.extra_info)) return result def list_providers(self, context, marker=None, limit=None, sort_keys=None, sort_dirs=None, filters=None): return self.provider_registry.list_providers(marker=marker, limit=limit, sort_keys=sort_keys, sort_dirs=sort_dirs, filters=filters) @messaging.expected_exceptions(exception.ProviderNotFound) def show_provider(self, context, provider_id): provider = self.provider_registry.show_provider(provider_id) response = { 'id': provider.id, 'name': provider.name, 'description': provider.description, 'extended_info_schema': provider.extended_info_schema, } return response
def setUp(self): super(ProtectableRegistryTest, self).setUp() self.protectable_registry = ProtectableRegistry() self._fake_plugin = _FakeProtectablePlugin(None) self.protectable_registry.register_plugin(self._fake_plugin)
def build_task_flow(self, ctx): cntxt = ctx["context"] workflow_engine = ctx["workflow_engine"] operation = ctx["operation_type"] resource_context = None resource_graph = None if operation == constants.OPERATION_PROTECT: plan = ctx["plan"] task_flow = workflow_engine.build_flow(flow_name=plan.get('id')) resources = plan.get('resources') parameters = plan.get('parameters') graph_resources = [] for resource in resources: graph_resources.append( Resource(type=resource['type'], id=resource['id'], name=resource['name'])) # TODO(luobin): pass registry in ctx registry = ProtectableRegistry() registry.load_plugins() resource_graph = registry.build_graph(cntxt, graph_resources) resource_context = ResourceGraphContext( cntxt=cntxt, operation=operation, workflow_engine=workflow_engine, task_flow=task_flow, plugin_map=self._plugin_map, parameters=parameters) if operation == constants.OPERATION_RESTORE: restore = ctx['restore'] task_flow = workflow_engine.build_flow(flow_name=restore.get('id')) checkpoint = ctx["checkpoint"] resource_graph = checkpoint.resource_graph parameters = restore.get('parameters') heat_template = ctx["heat_template"] resource_context = ResourceGraphContext( cntxt=cntxt, checkpoint=checkpoint, operation=operation, workflow_engine=workflow_engine, task_flow=task_flow, plugin_map=self._plugin_map, parameters=parameters, heat_template=heat_template) if operation == constants.OPERATION_DELETE: checkpoint = ctx['checkpoint'] task_flow = workflow_engine.build_flow(flow_name=checkpoint.id) resource_graph = checkpoint.resource_graph resource_context = ResourceGraphContext( cntxt=cntxt, checkpoint=checkpoint, operation=operation, workflow_engine=workflow_engine, task_flow=task_flow, plugin_map=self._plugin_map) # TODO(luobin): for other type operations walker_listener = ResourceGraphWalkerListener(resource_context) graph_walker = GraphWalker() graph_walker.register_listener(walker_listener) graph_walker.walk_graph(resource_graph) if operation == constants.OPERATION_PROTECT: return { "task_flow": walker_listener.context.task_flow, "status_getters": walker_listener.context.status_getters, "resource_graph": resource_graph } if operation == constants.OPERATION_RESTORE: return {"task_flow": walker_listener.context.task_flow} if operation == constants.OPERATION_DELETE: return { "task_flow": walker_listener.context.task_flow, "status_getters": walker_listener.context.status_getters }
class ProtectionManager(manager.Manager): """karbor Protection Manager.""" RPC_API_VERSION = '1.0' target = messaging.Target(version=RPC_API_VERSION) def __init__(self, service_name=None, *args, **kwargs): super(ProtectionManager, self).__init__(*args, **kwargs) provider_reg = CONF.provider_registry self.provider_registry = utils.load_plugin(PROVIDER_NAMESPACE, provider_reg) self.protectable_registry = ProtectableRegistry() self.protectable_registry.load_plugins() self.worker = flow_manager.Worker() self._greenpool = None self._greenpool_size = CONF.max_concurrent_operations if self._greenpool_size != 0: self._greenpool = greenpool.GreenPool(self._greenpool_size) def _spawn(self, func, *args, **kwargs): if self._greenpool is not None: return self._greenpool.spawn_n(func, *args, **kwargs) else: return greenthread.spawn_n(func, *args, **kwargs) def init_host(self, **kwargs): """Handle initialization if this is a standalone service""" # TODO(wangliuan) LOG.info(_LI("Starting protection service")) @messaging.expected_exceptions(exception.InvalidPlan, exception.ProviderNotFound, exception.FlowError) def protect(self, context, plan): """create protection for the given plan :param plan: Define that protection plan should be done """ LOG.info(_LI("Starting protection service:protect action")) LOG.debug("protecting: %s type: %s", plan, type(plan)) if not plan: raise exception.InvalidPlan( reason=_('the protection plan is None')) provider_id = plan.get('provider_id', None) plan_id = plan.get('id', None) provider = self.provider_registry.show_provider(provider_id) checkpoint_collection = provider.get_checkpoint_collection() try: checkpoint = checkpoint_collection.create(plan) except Exception as e: LOG.exception(_LE("Failed to create checkpoint, plan: %s"), plan_id) exc = exception.FlowError(flow="protect", error="Error creating checkpoint") six.raise_from(exc, e) try: protection_flow = self.worker.get_flow(context, constants.OPERATION_PROTECT, plan=plan, provider=provider, checkpoint=checkpoint) except Exception as e: LOG.exception(_LE("Failed to create protection flow, plan: %s"), plan_id) raise exception.FlowError( flow="protect", error=e.msg if hasattr(e, 'msg') else 'Internal error') self._spawn(self.worker.run_flow, protection_flow) return checkpoint.id @messaging.expected_exceptions(exception.ProviderNotFound, exception.CheckpointNotFound, exception.CheckpointNotAvailable, exception.FlowError) def restore(self, context, restore, restore_auth): LOG.info(_LI("Starting restore service:restore action")) checkpoint_id = restore["checkpoint_id"] provider_id = restore["provider_id"] provider = self.provider_registry.show_provider(provider_id) if not provider: raise exception.ProviderNotFound(provider_id=provider_id) checkpoint_collection = provider.get_checkpoint_collection() checkpoint = checkpoint_collection.get(checkpoint_id) if checkpoint.status != constants.CHECKPOINT_STATUS_AVAILABLE: raise exception.CheckpointNotAvailable( checkpoint_id=checkpoint_id) try: restoration_flow = self.worker.get_restoration_flow( context, constants.OPERATION_RESTORE, checkpoint, provider, restore, restore_auth) except Exception: LOG.exception( _LE("Failed to create restoration flow checkpoint: %s"), checkpoint_id) raise exception.FlowError( flow="restore", error=_("Failed to create flow")) self._spawn(self.worker.run_flow, restoration_flow) def delete(self, context, provider_id, checkpoint_id): LOG.info(_LI("Starting protection service:delete action")) LOG.debug('provider_id :%s checkpoint_id:%s', provider_id, checkpoint_id) provider = self.provider_registry.show_provider(provider_id) try: checkpoint_collection = provider.get_checkpoint_collection() checkpoint = checkpoint_collection.get(checkpoint_id) except Exception: LOG.error(_LE("get checkpoint failed, checkpoint_id:%s"), checkpoint_id) raise exception.InvalidInput( reason=_("Invalid checkpoint_id or provider_id")) if checkpoint.status not in [ constants.CHECKPOINT_STATUS_AVAILABLE, constants.CHECKPOINT_STATUS_ERROR, ]: raise exception.CheckpointNotBeDeleted( checkpoint_id=checkpoint_id) checkpoint.status = constants.CHECKPOINT_STATUS_DELETING checkpoint.commit() try: delete_checkpoint_flow = self.worker.get_delete_checkpoint_flow( context, constants.OPERATION_DELETE, checkpoint, provider) except Exception: LOG.exception( _LE("Failed to create delete checkpoint flow, checkpoint:%s."), checkpoint_id) raise exception.KarborException(_( "Failed to create delete checkpoint flow." )) self._spawn(self.worker.run_flow, delete_checkpoint_flow) def start(self, plan): # TODO(wangliuan) pass def suspend(self, plan): # TODO(wangliuan) pass @messaging.expected_exceptions(exception.ProviderNotFound, exception.CheckpointNotFound) def list_checkpoints(self, context, provider_id, marker=None, limit=None, sort_keys=None, sort_dirs=None, filters=None): LOG.info(_LI("Starting list checkpoints. " "provider_id:%s"), provider_id) plan_id = filters.get("plan_id", None) start_date = None end_date = None if filters.get("start_date", None): start_date = datetime.strptime( filters.get("start_date"), "%Y-%m-%d") if filters.get("end_date", None): end_date = datetime.strptime( filters.get("end_date"), "%Y-%m-%d") sort_dir = None if sort_dirs is None else sort_dirs[0] provider = self.provider_registry.show_provider(provider_id) checkpoint_ids = provider.list_checkpoints( limit=limit, marker=marker, plan_id=plan_id, start_date=start_date, end_date=end_date, sort_dir=sort_dir) checkpoints = [] for checkpoint_id in checkpoint_ids: checkpoint = provider.get_checkpoint(checkpoint_id) checkpoints.append(checkpoint.to_dict()) return checkpoints @messaging.expected_exceptions(exception.ProviderNotFound, exception.CheckpointNotFound) def show_checkpoint(self, context, provider_id, checkpoint_id): provider = self.provider_registry.show_provider(provider_id) checkpoint = provider.get_checkpoint(checkpoint_id) return checkpoint.to_dict() def list_protectable_types(self, context): LOG.info(_LI("Start to list protectable types.")) return self.protectable_registry.list_resource_types() @messaging.expected_exceptions(exception.ProtectableTypeNotFound) def show_protectable_type(self, context, protectable_type): LOG.info(_LI("Start to show protectable type %s"), protectable_type) plugin = self.protectable_registry.get_protectable_resource_plugin( protectable_type) if not plugin: raise exception.ProtectableTypeNotFound( protectable_type=protectable_type) dependents = [] for t in self.protectable_registry.list_resource_types(): if t == protectable_type: continue p = self.protectable_registry.get_protectable_resource_plugin(t) if p and protectable_type in p.get_parent_resource_types(): dependents.append(t) return { 'name': plugin.get_resource_type(), "dependent_types": dependents } @messaging.expected_exceptions(exception.ListProtectableResourceFailed) def list_protectable_instances(self, context, protectable_type=None, marker=None, limit=None, sort_keys=None, sort_dirs=None, filters=None, parameters=None): LOG.info(_LI("Start to list protectable instances of type: %s"), protectable_type) try: resource_instances = self.protectable_registry.list_resources( context, protectable_type, parameters) except exception.ListProtectableResourceFailed as err: LOG.error(_LE("List resources of type %(type)s failed: %(err)s"), {'type': protectable_type, 'err': six.text_type(err)}) raise result = [] for resource in resource_instances: result.append(dict(id=resource.id, name=resource.name)) return result @messaging.expected_exceptions(exception.ListProtectableResourceFailed) def show_protectable_instance(self, context, protectable_type, protectable_id, parameters=None): LOG.info(_LI("Start to show protectable instance of type: %s"), protectable_type) try: resource_instance = \ self.protectable_registry.show_resource(context, protectable_type, protectable_id, parameters=parameters) except exception.ListProtectableResourceFailed as err: LOG.error(_LE("Show resources of type %(type)s id %(id)s " "failed: %(err)s"), {'type': protectable_type, 'id': protectable_id, 'err': six.text_type(err)}) raise return dict(id=resource_instance.id, name=resource_instance.name, type=resource_instance.type) @messaging.expected_exceptions(exception.ListProtectableResourceFailed) def list_protectable_dependents(self, context, protectable_id, protectable_type): LOG.info(_LI("Start to list dependents of resource " "(type:%(type)s, id:%(id)s)"), {'type': protectable_type, 'id': protectable_id}) parent_resource = Resource(type=protectable_type, id=protectable_id, name="") try: dependent_resources = \ self.protectable_registry.fetch_dependent_resources( context, parent_resource) except exception.ListProtectableResourceFailed as err: LOG.error(_LE("List dependent resources of (%(res)s) " "failed: %(err)s"), {'res': parent_resource, 'err': six.text_type(err)}) raise result = [] for resource in dependent_resources: result.append(dict(type=resource.type, id=resource.id, name=resource.name)) return result def list_providers(self, context, marker=None, limit=None, sort_keys=None, sort_dirs=None, filters=None): return self.provider_registry.list_providers(marker=marker, limit=limit, sort_keys=sort_keys, sort_dirs=sort_dirs, filters=filters) @messaging.expected_exceptions(exception.ProviderNotFound) def show_provider(self, context, provider_id): provider = self.provider_registry.show_provider(provider_id) response = {'id': provider.id, 'name': provider.name, 'description': provider.description, 'extended_info_schema': provider.extended_info_schema, } return response