def on_initial_bootstrap(self, process, config, **kwargs): org_ms_client = OrgManagementServiceProcessClient(process=process) ex_ms_client = ExchangeManagementServiceProcessClient(process=process) system_actor, _ = process.container.resource_registry.find_resources( restype=RT.ActorIdentity, name=config.system.system_actor, id_only=True ) if not system_actor: raise AbortBootstrap("Cannot find system actor") system_actor_id = system_actor[0] # Create root Org: ION root_orgname = config.system.root_org org = Org(name=root_orgname, description="ION Root Org") self.org_id = org_ms_client.create_org(org) # Instantiate initial set of User Roles for this Org ion_manager = UserRole(governance_name=ION_MANAGER, name="ION Manager", description="ION Manager") org_ms_client.add_user_role(self.org_id, ion_manager) org_ms_client.grant_role(self.org_id, system_actor_id, ION_MANAGER) # Make the ION system agent a manager for the ION Org org_ms_client.grant_role(self.org_id, system_actor_id, ORG_MANAGER_ROLE) # Create root ExchangeSpace xs = ExchangeSpace(name=ION_ROOT_XS, description="ION service XS") self.xs_id = ex_ms_client.create_exchange_space(xs, self.org_id)
def __init__(self, container): log.debug("ExchangeManager initializing ...") self.container = container # Define the callables that can be added to Container public API # @TODO: remove self.container_api = [ self.create_xs, self.create_xp, self.create_xn_service, self.create_xn_process, self.create_xn_queue ] # Add the public callables to Container for call in self.container_api: setattr(self.container, call.__name__, call) self.default_xs = ExchangeSpace(self, ION_ROOT_XS) self._xs_cache = {} # caching of xs names to RR objects self._default_xs_obj = None # default XS registry object self.org_id = None # mappings self.xs_by_name = { ION_ROOT_XS: self.default_xs } # friendly named XS to XSO self.xn_by_name = {} # friendly named XN to XNO # xn by xs is a property self._chan = None # @TODO specify our own to_name here so we don't get auto-behavior - tricky chicken/egg self._ems_client = ExchangeManagementServiceProcessClient( process=self.container) self._rr_client = ResourceRegistryServiceProcessClient( process=self.container)
def on_initial_bootstrap(self, process, config, **kwargs): org_ms_client = OrgManagementServiceProcessClient(process=process) ex_ms_client = ExchangeManagementServiceProcessClient(process=process) system_actor, _ = process.container.resource_registry.find_resources( restype=RT.ActorIdentity, name=config.system.system_actor, id_only=True) if not system_actor: raise AbortBootstrap("Cannot find system actor") system_actor_id = system_actor[0] # Create root Org: ION root_orgname = config.system.root_org org = Org(name=root_orgname, description="ION Root Org") self.org_id = org_ms_client.create_org(org) # Instantiate initial set of User Roles for this Org ion_manager = UserRole(name=ION_MANAGER, label='ION Manager', description='ION Manager') org_ms_client.add_user_role(self.org_id, ion_manager) org_ms_client.grant_role(self.org_id, system_actor_id, ION_MANAGER) # Make the ION system agent a manager for the ION Org org_ms_client.grant_role(self.org_id, system_actor_id, ORG_MANAGER_ROLE) # Create root ExchangeSpace xs = ExchangeSpace(name=ION_ROOT_XS, description="ION service XS") self.xs_id = ex_ms_client.create_exchange_space(xs, self.org_id)
def on_initial_bootstrap(self, process, config, **kwargs): """ Bootstraps initial objects in the system from configuration (pyon.yml) via EMS calls. """ # get default org_id # @TODO: single org assumed for now org_ids = process.container.resource_registry.find_resources( RT.Org, id_only=True) if not (len(org_ids) and len(org_ids[0]) == 1): raise StandardError("Could not determine org_id") org_id = org_ids[0][0] ems_client = ExchangeManagementServiceProcessClient(process=process) # # Create XSs and XPs # for xsname, xsdict in config.get_safe('exchange_spaces', {}).iteritems(): xso = ResExchangeSpace(name=xsname) xso_id = ems_client.create_exchange_space(xso, org_id) log.info("ExchangeSpace %s, id %s", xsname, xso_id) for xpname, xpopts in xsdict.get('exchange_points', {}).iteritems(): # @TODO: some translation for types CFG currentl has it as "topic_tree" and we've been using "ttree" ttype = xpopts.get('type', 'topic_tree') if ttype == "topic_tree": ttype = "ttree" xpo = ResExchangePoint(name=xpname, topology_type=ttype) xpo_id = ems_client.create_exchange_point(xpo, xso_id) log.info("\tExchangePoint %s, id %s", xpname, xpo_id) # # Create and associate brokers with XSs # for brokername in xsdict.get('brokers', []): xbo = ResExchangeBroker(name=brokername) xbo_id = ems_client.create_exchange_broker(xbo) log.info("\tExchangeBroker %s, id %s", brokername, xbo_id) # directly associate broker with XS # @TODO: should EMS provide this? # first find out if the assoc exists already assocs = process.container.resource_registry.find_associations( xso_id, PRED.hasExchangeBroker, id_only=True) if len(assocs) > 0: continue process.container.resource_registry.create_association( xso_id, PRED.hasExchangeBroker, xbo_id)
def on_initial_bootstrap(self, process, config, **kwargs): """ Bootstraps initial objects in the system from configuration (pyon.yml) via EMS calls. """ # Get ION Org root_org_name = config.get_safe('system.root_org' , "ION") org_ids, _ = process.container.resource_registry.find_resources(restype=RT.Org, name=root_org_name, id_only=True) if not org_ids or len(org_ids) > 1: raise StandardError("Could not determine root Org") org_id = org_ids[0] ems_client = ExchangeManagementServiceProcessClient(process=process) # Create XSs and XPs resource objects xs_by_name = {} # Name to resource ID mapping for ExchangeSpace xs_defs = config.get_safe("exchange.exchange_spaces", {}) for xsname, xsdict in xs_defs.iteritems(): xso = ResExchangeSpace(name=xsname, description=xsdict.get("description", "")) xso_id = ems_client.create_exchange_space(xso, org_id) xs_by_name[xsname] = xso_id log.info("ExchangeSpace %s, id %s", xsname, xso_id) for xpname, xpopts in xsdict.get("exchange_points", {}).iteritems(): # Translation for types. CFG currently has it as "topic_tree" and Pyon uses "ttree" xpo = ResExchangePoint(name=xpname, description=xpopts.get("description", ""), topology_type=xpopts.get('type', 'ttree')) xpo_id = ems_client.create_exchange_point(xpo, xso_id) log.info("\tExchangePoint %s, id %s", xpname, xpo_id) # Create XSs and XPs resource objects for brokername, bdict in config.get_safe("exchange.exchange_brokers", {}).iteritems(): xbo = ResExchangeBroker(name=brokername, description=bdict.get("description", "")) xbo_id = ems_client.create_exchange_broker(xbo) log.info("\tExchangeBroker %s, id %s", brokername, xbo_id) for xs_name in bdict.get("join_xs", []): if xs_name in xs_by_name: xs_id = xs_by_name[xs_name] # directly associate broker with XS # @TODO: should EMS provide this? # first find out if the assoc exists already assocs = process.container.resource_registry.find_associations(xso_id, PRED.hasExchangeBroker, id_only=True) if len(assocs) > 0: continue process.container.resource_registry.create_association(xso_id, PRED.hasExchangeBroker, xbo_id) else: log.warn("ExchangeSpace %s unknown. Broker %s cannot join", xs_name, brokername) for xp_name in bdict.get("join_xp", []): pass
def on_initial_bootstrap(self, process, config, **kwargs): """ Bootstraps initial objects in the system from configuration (pyon.yml) via EMS calls. """ # get default org_id # @TODO: single org assumed for now org_ids = process.container.resource_registry.find_resources(RT.Org, id_only=True) if not (len(org_ids) and len(org_ids[0]) == 1): raise StandardError("Could not determine org_id") org_id = org_ids[0][0] ems_client = ExchangeManagementServiceProcessClient(process=process) # # Create XSs and XPs # for xsname, xsdict in config.get_safe('exchange_spaces', {}).iteritems(): xso = ResExchangeSpace(name=xsname) xso_id = ems_client.create_exchange_space(xso, org_id) log.info("ExchangeSpace %s, id %s", xsname, xso_id) for xpname, xpopts in xsdict.get('exchange_points', {}).iteritems(): # @TODO: some translation for types CFG currentl has it as "topic_tree" and we've been using "ttree" ttype = xpopts.get('type', 'topic_tree') if ttype == "topic_tree": ttype = "ttree" xpo = ResExchangePoint(name=xpname, topology_type=ttype) xpo_id = ems_client.create_exchange_point(xpo, xso_id) log.info("\tExchangePoint %s, id %s", xpname, xpo_id) # # Create and associate brokers with XSs # for brokername in xsdict.get('brokers', []): xbo = ResExchangeBroker(name=brokername) xbo_id = ems_client.create_exchange_broker(xbo) log.info("\tExchangeBroker %s, id %s", brokername, xbo_id) # directly associate broker with XS # @TODO: should EMS provide this? # first find out if the assoc exists already assocs = process.container.resource_registry.find_associations(xso_id, PRED.hasExchangeBroker, id_only=True) if len(assocs) > 0: continue process.container.resource_registry.create_association(xso_id, PRED.hasExchangeBroker, xbo_id)
def on_initial_bootstrap(self, process, config, **kwargs): org_ms_client = OrgManagementServiceProcessClient(process=process) ex_ms_client = ExchangeManagementServiceProcessClient(process=process) system_actor, _ = process.container.resource_registry.find_resources( restype=RT.ActorIdentity, name=config.system.system_actor, id_only=True) if not system_actor: raise AbortBootstrap("Cannot find system actor") system_actor_id = system_actor[0] # Create root Org: ION root_orgname = config.system.root_org org = Org(name=root_orgname, description="ION Root Org") self.org_id = org_ms_client.create_org(org, headers={'ion-actor-id': system_actor_id}) # Instantiate initial set of User Roles for this Org ion_manager = UserRole(name=ION_MANAGER,label='ION Manager', description='ION Manager') org_ms_client.add_user_role(self.org_id, ion_manager) org_ms_client.grant_role(self.org_id, system_actor_id, ION_MANAGER, headers={'ion-actor-id': system_actor_id} ) # Make the ION system agent a manager for the ION Org org_ms_client.grant_role(self.org_id, system_actor_id, MANAGER_ROLE, headers={'ion-actor-id': system_actor_id} ) # Create root ExchangeSpace xs = ExchangeSpace(name=ION_ROOT_XS, description="ION service XS") self.xs_id = ex_ms_client.create_exchange_space(xs, self.org_id, headers={'ion-actor-id': system_actor_id}) # Now load the base set of negotiation definitions used by the request operations (enroll/role/resource, etc) neg_def = NegotiationDefinition(name=RT.EnrollmentRequest, description='Definition of Enrollment Request Negotiation', pre_condition = ['is_registered(user_id)', 'is_not_enrolled(org_id,user_id)', 'enroll_req_not_exist(org_id,user_id)'], accept_action = 'enroll_member(org_id,user_id)' ) process.container.resource_registry.create(neg_def) neg_def = NegotiationDefinition(name=RT.RoleRequest, description='Definition of Role Request Negotiation', pre_condition = ['is_enrolled(org_id,user_id)'], accept_action = 'grant_role(org_id,user_id,role_name)' ) process.container.resource_registry.create(neg_def) neg_def = NegotiationDefinition(name=RT.ResourceRequest, description='Definition of Role Request Negotiation', pre_condition = ['is_enrolled(org_id,user_id)'], accept_action = 'acquire_resource(org_id,user_id,resource_id)' ) process.container.resource_registry.create(neg_def)
def __init__(self, container): log.debug("ExchangeManager initializing ...") self.container = container # Define the callables that can be added to Container public API # @TODO: remove self.container_api = [self.create_xs, self.create_xp, self.create_xn_service, self.create_xn_process, self.create_xn_queue] # Add the public callables to Container for call in self.container_api: setattr(self.container, call.__name__, call) self.default_xs = ExchangeSpace(self, ION_ROOT_XS) self._xs_cache = {} # caching of xs names to RR objects self._default_xs_obj = None # default XS registry object self.org_id = None # mappings self.xs_by_name = { ION_ROOT_XS: self.default_xs } # friendly named XS to XSO self.xn_by_name = {} # friendly named XN to XNO # xn by xs is a property self._chan = None # @TODO specify our own to_name here so we don't get auto-behavior - tricky chicken/egg self._ems_client = ExchangeManagementServiceProcessClient(process=self.container) self._rr_client = ResourceRegistryServiceProcessClient(process=self.container)
def setUp(self): # Start container self._start_container() #Load a deploy file self.container.start_rel_from_url('res/deploy/r2deploy.yml') #Instantiate a process to represent the test process = GovernanceTestProcess() #Load system policies after container has started all of the services LoadSystemPolicy.op_load_system_policies(process) self.rr_client = ResourceRegistryServiceProcessClient( node=self.container.node, process=process) self.id_client = IdentityManagementServiceProcessClient( node=self.container.node, process=process) self.pol_client = PolicyManagementServiceProcessClient( node=self.container.node, process=process) self.org_client = OrgManagementServiceProcessClient( node=self.container.node, process=process) self.ims_client = InstrumentManagementServiceProcessClient( node=self.container.node, process=process) self.ems_client = ExchangeManagementServiceProcessClient( node=self.container.node, process=process) self.ion_org = self.org_client.find_org() self.system_actor = self.id_client.find_actor_identity_by_name( name=CFG.system.system_actor) log.debug('system actor:' + self.system_actor._id) sa_header_roles = get_role_message_headers( self.org_client.find_all_roles_by_user(self.system_actor._id)) self.sa_user_header = { 'ion-actor-id': self.system_actor._id, 'ion-actor-roles': sa_header_roles }
def __init__(self, container): log.debug("ExchangeManager initializing ...") self.container = container # Define the callables that can be added to Container public API # @TODO: remove self.container_api = [ self.create_xs, self.create_xp, self.create_xn_service, self.create_xn_process, self.create_xn_queue ] # Add the public callables to Container for call in self.container_api: setattr(self.container, call.__name__, call) self.default_xs = None self._xs_cache = {} # caching of xs names to RR objects self._default_xs_obj = None # default XS registry object self.org_id = None self._default_xs_declared = False # mappings self.xs_by_name = {} # friendly named XS to XSO self.xn_by_name = {} # friendly named XN to XNO # xn by xs is a property # @TODO specify our own to_name here so we don't get auto-behavior - tricky chicken/egg self._ems_client = ExchangeManagementServiceProcessClient( process=self.container) self._rr_client = ResourceRegistryServiceProcessClient( process=self.container) # mapping of node/ioloop runner by connection name (in config, named via container.messaging.server keys) self._nodes = {} self._ioloops = {} # public toggle switch for if EMS should be used by default self.use_ems = True
def setUp(self): # Start container self._start_container() #Load a deploy file self.container.start_rel_from_url('res/deploy/r2deploy.yml') #Instantiate a process to represent the test process=GovernanceTestProcess() #Load system policies after container has started all of the services LoadSystemPolicy.op_load_system_policies(process) gevent.sleep(1) # Wait for events to be fired and policy updated self.rr_client = ResourceRegistryServiceProcessClient(node=self.container.node, process=process) self.id_client = IdentityManagementServiceProcessClient(node=self.container.node, process=process) self.pol_client = PolicyManagementServiceProcessClient(node=self.container.node, process=process) self.org_client = OrgManagementServiceProcessClient(node=self.container.node, process=process) self.ims_client = InstrumentManagementServiceProcessClient(node=self.container.node, process=process) self.ems_client = ExchangeManagementServiceProcessClient(node=self.container.node, process=process) self.ion_org = self.org_client.find_org() self.system_actor = self.id_client.find_actor_identity_by_name(name=CFG.system.system_actor) log.debug('system actor:' + self.system_actor._id) sa_header_roles = get_role_message_headers(self.org_client.find_all_roles_by_user(self.system_actor._id)) self.sa_user_header = {'ion-actor-id': self.system_actor._id, 'ion-actor-roles': sa_header_roles }
def __init__(self, container): log.debug("ExchangeManager initializing ...") self.container = container # Define the callables that can be added to Container public API # @TODO: remove self.container_api = [self.create_xs, self.create_xp, self.create_xn_service, self.create_xn_process, self.create_xn_queue] # Add the public callables to Container for call in self.container_api: setattr(self.container, call.__name__, call) self.default_xs = None self._xs_cache = {} # caching of xs names to RR objects self._default_xs_obj = None # default XS registry object self.org_id = None self._default_xs_declared = False # mappings self.xs_by_name = {} # friendly named XS to XSO self.xn_by_name = {} # friendly named XN to XNO # xn by xs is a property # @TODO specify our own to_name here so we don't get auto-behavior - tricky chicken/egg self._ems_client = ExchangeManagementServiceProcessClient(process=self.container) self._rr_client = ResourceRegistryServiceProcessClient(process=self.container) # mapping of node/ioloop runner by connection name (in config, named via container.messaging.server keys) self._nodes = {} self._ioloops = {} # public toggle switch for if EMS should be used by default self.use_ems = True
class ExchangeManager(object): """ Manager object for the CC to manage Exchange related resources. """ def __init__(self, container): log.debug("ExchangeManager initializing ...") self.container = container # Define the callables that can be added to Container public API # @TODO: remove self.container_api = [ self.create_xs, self.create_xp, self.create_xn_service, self.create_xn_process, self.create_xn_queue ] # Add the public callables to Container for call in self.container_api: setattr(self.container, call.__name__, call) self.default_xs = ExchangeSpace(self, ION_ROOT_XS) self._xs_cache = {} # caching of xs names to RR objects self._default_xs_obj = None # default XS registry object self.org_id = None # mappings self.xs_by_name = { ION_ROOT_XS: self.default_xs } # friendly named XS to XSO self.xn_by_name = {} # friendly named XN to XNO # xn by xs is a property self._chan = None # @TODO specify our own to_name here so we don't get auto-behavior - tricky chicken/egg self._ems_client = ExchangeManagementServiceProcessClient( process=self.container) self._rr_client = ResourceRegistryServiceProcessClient( process=self.container) # TODO: Do more initializing here, not in container @property def xn_by_xs(self): """ Get a list of XNs associated by XS (friendly name). """ ret = {} for xnname, xn in self.xn_by_name.iteritems(): xsn = xn._xs._exchange if not xsn in ret: ret[xsn] = [] ret[xsn].append(xn) return ret def _get_xs_obj(self, name=ION_ROOT_XS): """ Gets a resource-registry represented XS, either via cache or RR request. """ if name in self._xs_cache: return self._xs_cache[name] xs_objs, _ = self._rr_client.find_resources(RT.ExchangeSpace, name=name) if not len(xs_objs) == 1: log.warn("Could not find RR XS object with name: %s", name) return None self._xs_cache[name] = xs_objs[0] return xs_objs[0] def start(self): log.debug("ExchangeManager starting ...") # Establish connection to broker # @TODO: raise error if sux node, ioloop = messaging.make_node() self._transport = AMQPTransport.get_instance() self._client = self._get_channel(node) # Declare root exchange #self.default_xs.ensure_exists(self._get_channel()) return node, ioloop def _ems_available(self): """ Returns True if the EMS is (likely) available and the auto_register CFG entry is True. Has the side effect of bootstrapping the org_id and default_xs's id/rev from the RR. Therefore, cannot be a property. """ if CFG.container.get('exchange', {}).get('auto_register', False): # ok now make sure it's in the directory svc_de = self.container.directory.lookup( '/Services/exchange_management') if svc_de is not None: if not self.org_id: # find the default Org org_ids = self._rr_client.find_resources(RT.Org, id_only=True) if not (len(org_ids) and len(org_ids[0]) == 1): log.warn("EMS available but could not find Org") return False self.org_id = org_ids[0][0] log.debug( "Bootstrapped Container exchange manager with org id: %s", self.org_id) return True return False def _get_channel(self, node): """ Get a raw channel to be used by all the ensure_exists methods. """ assert self.container # @TODO: needs lock, but so do all these methods if not self._chan: self._chan = blocking_cb(node.client.channel, 'on_open_callback') return self._chan def create_xs(self, name, use_ems=True, exchange_type='topic', durable=False, auto_delete=True): log.debug("ExchangeManager.create_xs: %s", name) xs = ExchangeSpace(self, name, exchange_type=exchange_type, durable=durable, auto_delete=auto_delete) self.xs_by_name[name] = xs if use_ems and self._ems_available(): log.debug("Using EMS to create_xs") # create a RR object xso = ResExchangeSpace(name=name) xso_id = self._ems_client.create_exchange_space(xso, self.org_id) log.debug("Created RR XS object, id: %s", xso_id) else: xs.declare() return xs def delete_xs(self, xs, use_ems=True): """ @type xs ExchangeSpace """ log.debug("ExchangeManager.delete_xs: %s", xs) name = xs._exchange # @TODO this feels wrong del self.xs_by_name[name] if use_ems and self._ems_available(): log.debug("Using EMS to delete_xs") xso = self._get_xs_obj(name) self._ems_client.delete_exchange_space(xso._id) del self._xs_cache[name] else: xs.delete() def create_xp(self, name, xs=None, use_ems=True, **kwargs): log.debug("ExchangeManager.create_xp: %s", name) xs = xs or self.default_xs xp = ExchangePoint(self, name, xs, **kwargs) # put in xn_by_name anyway self.xn_by_name[name] = xp if use_ems and self._ems_available(): log.debug("Using EMS to create_xp") # create an RR object xpo = ResExchangePoint(name=name, topology_type=xp._xptype) xpo_id = self._ems_client.create_exchange_point( xpo, self._get_xs_obj( xs._exchange)._id) # @TODO: _exchange is wrong else: xp.declare() return xp def delete_xp(self, xp, use_ems=True): log.debug("ExchangeManager.delete_xp: name=%s", 'TODO') #xp.build_xname()) name = xp._exchange # @TODO: not right del self.xn_by_name[name] if use_ems and self._ems_available(): log.debug("Using EMS to delete_xp") # find the XP object via RR xpo_ids = self._rr_client.find_resources(RT.ExchangePoint, name=name, id_only=True) if not (len(xpo_ids) and len(xpo_ids[0]) == 1): log.warn("Could not find XP in RR with name of %s", name) xpo_id = xpo_ids[0][0] self._ems_client.delete_exchange_point(xpo_id) else: xp.delete() def _create_xn(self, xn_type, name, xs=None, use_ems=True, **kwargs): xs = xs or self.default_xs log.debug( "ExchangeManager._create_xn: type: %s, name=%s, xs=%s, kwargs=%s", xn_type, name, xs, kwargs) if xn_type == "service": xn = ExchangeNameService(self, name, xs, **kwargs) elif xn_type == "process": xn = ExchangeNameProcess(self, name, xs, **kwargs) elif xn_type == "queue": xn = ExchangeNameQueue(self, name, xs, **kwargs) else: raise StandardError("Unknown XN type: %s" % xn_type) self.xn_by_name[name] = xn if use_ems and self._ems_available(): log.debug("Using EMS to create_xn") xno = ResExchangeName(name=name, xn_type=xn.xn_type) self._ems_client.declare_exchange_name( xno, self._get_xs_obj(xs._exchange)._id) # @TODO: exchange is wrong else: xn.declare() return xn def create_xn_service(self, name, xs=None, **kwargs): return self._create_xn('service', name, xs=xs, **kwargs) def create_xn_process(self, name, xs=None, **kwargs): return self._create_xn('process', name, xs=xs, **kwargs) def create_xn_queue(self, name, xs=None, **kwargs): return self._create_xn('queue', name, xs=xs, **kwargs) def delete_xn(self, xn, use_ems=False): log.debug("ExchangeManager.delete_xn: name=%s", "TODO") #xn.build_xlname()) name = xn._queue # @TODO feels wrong del self.xn_by_name[name] if use_ems and self._ems_available(): log.debug("Using EMS to delete_xn") # find the XN object via RR? xno_ids = self._rr_client.find_resources(RT.ExchangeName, name=name, id_only=True) if not (len(xno_ids) and len(xno_ids[0]) == 1): log.warn("Could not find XN in RR with name of %s", name) xno_id = xno_ids[0][0] self._ems_client.undeclare_exchange_name( xno_id) # "canonical name" currently understood to be RR id else: xn.delete() def stop(self, *args, **kwargs): log.debug("ExchangeManager stopping ...") # transport implementations - XOTransport objects call here def declare_exchange(self, exchange, exchange_type='topic', durable=False, auto_delete=True): log.info("ExchangeManager.declare_exchange") self._transport.declare_exchange_impl(self._client, exchange, exchange_type=exchange_type, durable=durable, auto_delete=auto_delete) def delete_exchange(self, exchange, **kwargs): log.info("ExchangeManager.delete_exchange") self._transport.delete_exchange_impl(self._client, exchange, **kwargs) def declare_queue(self, queue, durable=False, auto_delete=True): log.info("ExchangeManager.declare_queue") return self._transport.declare_queue_impl(self._client, queue, durable=durable, auto_delete=auto_delete) def delete_queue(self, queue, **kwargs): log.info("ExchangeManager.delete_queue") self._transport.delete_queue_impl(self._client, queue, **kwargs) def bind(self, exchange, queue, binding): log.info("ExchangeManager.bind") self._transport.bind_impl(self._client, exchange, queue, binding) def unbind(self, exchange, queue, binding): log.info("ExchangeManager.unbind") self._transport.unbind_impl(self._client, exchange, queue, binding)
class ExchangeManager(object): """ Manager object for the CC to manage Exchange related resources. """ def __init__(self, container): log.debug("ExchangeManager initializing ...") self.container = container # Define the callables that can be added to Container public API # @TODO: remove self.container_api = [ self.create_xs, self.create_xp, self.create_xn_service, self.create_xn_process, self.create_xn_queue ] # Add the public callables to Container for call in self.container_api: setattr(self.container, call.__name__, call) self.default_xs = ExchangeSpace(self, ION_ROOT_XS) self._xs_cache = {} # caching of xs names to RR objects self._default_xs_obj = None # default XS registry object self.org_id = None # mappings self.xs_by_name = { ION_ROOT_XS: self.default_xs } # friendly named XS to XSO self.xn_by_name = {} # friendly named XN to XNO # xn by xs is a property self._chan = None # @TODO specify our own to_name here so we don't get auto-behavior - tricky chicken/egg self._ems_client = ExchangeManagementServiceProcessClient( process=self.container) self._rr_client = ResourceRegistryServiceProcessClient( process=self.container) # mapping of node/ioloop runner by connection name (in config, named via container.messaging.server keys) self._nodes = {} self._ioloops = {} self._client = None self._transport = None self._default_xs_declared = False def start(self): log.debug("ExchangeManager.start") total_count = 0 def handle_failure(name, node): log.warn("Node %s could not be started", name) node.ready.set() # let it fall out below # Establish connection(s) to broker for name, cfgkey in CFG.container.messaging.server.iteritems(): if not cfgkey: continue if cfgkey not in CFG.server: raise ExchangeManagerError( "Config key %s (name: %s) (from CFG.container.messaging.server) not in CFG.server" % (cfgkey, name)) total_count += 1 log.debug("Starting connection: %s", name) # start it with a zero timeout so it comes right back to us try: node, ioloop = messaging.make_node(CFG.server[cfgkey], name, 0) # install a finished handler directly on the ioloop just for this startup period fail_handle = lambda _: handle_failure(name, node) ioloop.link(fail_handle) # wait for the node ready event, with a large timeout just in case node_ready = node.ready.wait(timeout=15) # remove the finished handler, we don't care about it here ioloop.unlink(fail_handle) # only add to our list if we started successfully if not node.running: ioloop.kill() # make sure ioloop dead else: self._nodes[name] = node self._ioloops[name] = ioloop except socket.error as e: log.warn( "Could not start connection %s due to socket error, continuing", name) fail_count = total_count - len(self._nodes) if fail_count > 0 or total_count == 0: if fail_count == total_count: raise ExchangeManagerError( "No node connection was able to start (%d nodes attempted, %d nodes failed)" % (total_count, fail_count)) log.warn("Some nodes could not be started, ignoring for now" ) # @TODO change when ready self._transport = AMQPTransport.get_instance() self._client = self._get_channel( self._nodes.get('priviledged', self._nodes.values()[0])) # @TODO log.debug("Started %d connections (%s)", len(self._nodes), ",".join(self._nodes.iterkeys())) def stop(self, *args, **kwargs): # ############## # HACK HACK HACK # # It appears during shutdown that when a channel is closed, it's not FULLY closed by the pika connection # until the next round of _handle_events. We have to yield here to let that happen, in order to have close # work fine without blowing up. # ############## time.sleep(0.1) # ############## # /HACK # ############## log.debug("ExchangeManager.stopping (%d connections)", len(self._nodes)) for name in self._nodes: self._nodes[name].stop_node() self._ioloops[name].kill() self._nodes[name].client.ioloop.start( ) # loop until connection closes # @TODO undeclare root xs?? need to know if last container #self.default_xs.delete() @property def default_node(self): """ Returns the default node connection. """ if 'primary' in self._nodes: return self._nodes['primary'] elif len(self._nodes): log.warn("No primary connection, returning first available") return self._nodes.values()[0] return None @property def xn_by_xs(self): """ Get a list of XNs associated by XS (friendly name). """ ret = {} for xnname, xn in self.xn_by_name.iteritems(): xsn = xn._xs._exchange if not xsn in ret: ret[xsn] = [] ret[xsn].append(xn) return ret def _get_xs_obj(self, name=ION_ROOT_XS): """ Gets a resource-registry represented XS, either via cache or RR request. """ if name in self._xs_cache: return self._xs_cache[name] xs_objs, _ = self._rr_client.find_resources(RT.ExchangeSpace, name=name) if not len(xs_objs) == 1: log.warn("Could not find RR XS object with name: %s", name) return None self._xs_cache[name] = xs_objs[0] return xs_objs[0] def _ems_available(self): """ Returns True if the EMS is (likely) available and the auto_register CFG entry is True. Has the side effect of bootstrapping the org_id and default_xs's id/rev from the RR. Therefore, cannot be a property. """ if CFG.container.get('exchange', {}).get('auto_register', False): # ok now make sure it's in the directory svc_de = self.container.directory.lookup( '/Services/exchange_management') if svc_de is not None: if not self.org_id: # find the default Org org_ids = self._rr_client.find_resources(RT.Org, id_only=True) if not (len(org_ids) and len(org_ids[0]) == 1): log.warn("EMS available but could not find Org") return False self.org_id = org_ids[0][0] log.debug( "Bootstrapped Container exchange manager with org id: %s", self.org_id) return True return False def _get_channel(self, node): """ Get a raw channel to be used by all the ensure_exists methods. """ assert self.container # @TODO: needs lock, but so do all these methods if not self._chan: self._chan = blocking_cb(node.client.channel, 'on_open_callback') return self._chan def create_xs(self, name, use_ems=True, exchange_type='topic', durable=False, auto_delete=True): log.debug("ExchangeManager.create_xs: %s", name) xs = ExchangeSpace(self, name, exchange_type=exchange_type, durable=durable, auto_delete=auto_delete) self.xs_by_name[name] = xs if use_ems and self._ems_available(): log.debug("Using EMS to create_xs") # create a RR object xso = ResExchangeSpace(name=name) xso_id = self._ems_client.create_exchange_space(xso, self.org_id) log.debug("Created RR XS object, id: %s", xso_id) else: xs.declare() return xs def delete_xs(self, xs, use_ems=True): """ @type xs ExchangeSpace """ log.debug("ExchangeManager.delete_xs: %s", xs) name = xs._exchange # @TODO this feels wrong del self.xs_by_name[name] if use_ems and self._ems_available(): log.debug("Using EMS to delete_xs") xso = self._get_xs_obj(name) self._ems_client.delete_exchange_space(xso._id) del self._xs_cache[name] else: xs.delete() def create_xp(self, name, xs=None, use_ems=True, **kwargs): log.debug("ExchangeManager.create_xp: %s", name) xs = xs or self.default_xs xp = ExchangePoint(self, name, xs, **kwargs) # put in xn_by_name anyway self.xn_by_name[name] = xp if use_ems and self._ems_available(): log.debug("Using EMS to create_xp") # create an RR object xpo = ResExchangePoint(name=name, topology_type=xp._xptype) xpo_id = self._ems_client.create_exchange_point( xpo, self._get_xs_obj( xs._exchange)._id) # @TODO: _exchange is wrong else: xp.declare() return xp def delete_xp(self, xp, use_ems=True): log.debug("ExchangeManager.delete_xp: name=%s", 'TODO') #xp.build_xname()) name = xp._exchange # @TODO: not right del self.xn_by_name[name] if use_ems and self._ems_available(): log.debug("Using EMS to delete_xp") # find the XP object via RR xpo_ids = self._rr_client.find_resources(RT.ExchangePoint, name=name, id_only=True) if not (len(xpo_ids) and len(xpo_ids[0]) == 1): log.warn("Could not find XP in RR with name of %s", name) xpo_id = xpo_ids[0][0] self._ems_client.delete_exchange_point(xpo_id) else: xp.delete() def _create_xn(self, xn_type, name, xs=None, use_ems=True, **kwargs): xs = xs or self.default_xs log.debug( "ExchangeManager._create_xn: type: %s, name=%s, xs=%s, kwargs=%s", xn_type, name, xs, kwargs) if xn_type == "service": xn = ExchangeNameService(self, name, xs, **kwargs) elif xn_type == "process": xn = ExchangeNameProcess(self, name, xs, **kwargs) elif xn_type == "queue": xn = ExchangeNameQueue(self, name, xs, **kwargs) else: raise StandardError("Unknown XN type: %s" % xn_type) self.xn_by_name[name] = xn if use_ems and self._ems_available(): log.debug("Using EMS to create_xn") xno = ResExchangeName(name=name, xn_type=xn.xn_type) self._ems_client.declare_exchange_name( xno, self._get_xs_obj(xs._exchange)._id) # @TODO: exchange is wrong else: xn.declare() return xn def create_xn_service(self, name, xs=None, **kwargs): return self._create_xn('service', name, xs=xs, **kwargs) def create_xn_process(self, name, xs=None, **kwargs): return self._create_xn('process', name, xs=xs, **kwargs) def create_xn_queue(self, name, xs=None, **kwargs): return self._create_xn('queue', name, xs=xs, **kwargs) def delete_xn(self, xn, use_ems=False): log.debug("ExchangeManager.delete_xn: name=%s", "TODO") #xn.build_xlname()) name = xn._queue # @TODO feels wrong del self.xn_by_name[name] if use_ems and self._ems_available(): log.debug("Using EMS to delete_xn") # find the XN object via RR? xno_ids = self._rr_client.find_resources(RT.ExchangeName, name=name, id_only=True) if not (len(xno_ids) and len(xno_ids[0]) == 1): log.warn("Could not find XN in RR with name of %s", name) xno_id = xno_ids[0][0] self._ems_client.undeclare_exchange_name( xno_id) # "canonical name" currently understood to be RR id else: xn.delete() def _ensure_default_declared(self): """ Ensures we declared the default exchange space. Needed by most exchange object calls, so each one calls here. """ if not self._default_xs_declared: log.debug( "ExchangeManager._ensure_default_declared, declaring default xs" ) self._default_xs_declared = True self.default_xs.declare() # transport implementations - XOTransport objects call here def declare_exchange(self, exchange, exchange_type='topic', durable=False, auto_delete=True): log.info("ExchangeManager.declare_exchange") self._ensure_default_declared() self._transport.declare_exchange_impl(self._client, exchange, exchange_type=exchange_type, durable=durable, auto_delete=auto_delete) def delete_exchange(self, exchange, **kwargs): log.info("ExchangeManager.delete_exchange") self._ensure_default_declared() self._transport.delete_exchange_impl(self._client, exchange, **kwargs) def declare_queue(self, queue, durable=False, auto_delete=False): log.info("ExchangeManager.declare_queue (queue %s, durable %s, AD %s)", queue, durable, auto_delete) self._ensure_default_declared() return self._transport.declare_queue_impl(self._client, queue, durable=durable, auto_delete=auto_delete) def delete_queue(self, queue, **kwargs): log.info("ExchangeManager.delete_queue") self._ensure_default_declared() self._transport.delete_queue_impl(self._client, queue, **kwargs) def bind(self, exchange, queue, binding): log.info("ExchangeManager.bind") self._ensure_default_declared() self._transport.bind_impl(self._client, exchange, queue, binding) def unbind(self, exchange, queue, binding): log.info("ExchangeManager.unbind") self._ensure_default_declared() self._transport.unbind_impl(self._client, exchange, queue, binding) def get_stats(self, queue): log.info("ExchangeManager.get_stats") self._ensure_default_declared() return self._transport.get_stats(self._client, queue) def purge(self, queue): log.info("ExchangeManager.purge") self._ensure_default_declared() self._transport.purge(self._client, queue)
class TestGovernanceInt(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() #Load a deploy file self.container.start_rel_from_url('res/deploy/r2deploy.yml') #Instantiate a process to represent the test process = GovernanceTestProcess() #Load system policies after container has started all of the services LoadSystemPolicy.op_load_system_policies(process) self.rr_client = ResourceRegistryServiceProcessClient( node=self.container.node, process=process) self.id_client = IdentityManagementServiceProcessClient( node=self.container.node, process=process) self.pol_client = PolicyManagementServiceProcessClient( node=self.container.node, process=process) self.org_client = OrgManagementServiceProcessClient( node=self.container.node, process=process) self.ims_client = InstrumentManagementServiceProcessClient( node=self.container.node, process=process) self.ems_client = ExchangeManagementServiceProcessClient( node=self.container.node, process=process) self.ion_org = self.org_client.find_org() self.system_actor = self.id_client.find_actor_identity_by_name( name=CFG.system.system_actor) log.debug('system actor:' + self.system_actor._id) sa_header_roles = get_role_message_headers( self.org_client.find_all_roles_by_user(self.system_actor._id)) self.sa_user_header = { 'ion-actor-id': self.system_actor._id, 'ion-actor-roles': sa_header_roles } @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') def test_basic_policy(self): #Make sure that the system policies have been loaded policy_list, _ = self.rr_client.find_resources(restype=RT.Policy) self.assertNotEqual( len(policy_list), 0, "The system policies have not been loaded into the Resource Registry" ) #Attempt to access an operation in service which does not have specific policies set es_obj = IonObject(RT.ExchangeSpace, description='ION test XS', name='ioncore2') with self.assertRaises(Unauthorized) as cm: self.ems_client.create_exchange_space(es_obj) self.assertIn( 'exchange_management(create_exchange_space) has been denied', cm.exception.message) #Add a new policy to allow the the above service call. policy_obj = IonObject( RT.Policy, name='Exchange_Management_Test_Policy', definition_type="Service", rule=TEST_POLICY_TEXT, description= 'Allow specific operations in the Exchange Management Service for anonymous user' ) test_policy_id = self.pol_client.create_policy( policy_obj, headers=self.sa_user_header) self.pol_client.add_service_policy('exchange_management', test_policy_id, headers=self.sa_user_header) log.info('Policy created: ' + policy_obj.name) gevent.sleep(2) # Wait for events to be fired and policy updated #The previous attempt at this operations should now be allowed. es_obj = IonObject(RT.ExchangeSpace, description='ION test XS', name='ioncore2') with self.assertRaises(BadRequest) as cm: self.ems_client.create_exchange_space(es_obj) self.assertIn('Arguments not set', cm.exception.message) #disable the test policy to try again self.pol_client.disable_policy(test_policy_id, headers=self.sa_user_header) gevent.sleep(2) # Wait for events to be fired and policy updated #The same request that previously was allowed should not be denied es_obj = IonObject(RT.ExchangeSpace, description='ION test XS', name='ioncore2') with self.assertRaises(Unauthorized) as cm: self.ems_client.create_exchange_space(es_obj) self.assertIn( 'exchange_management(create_exchange_space) has been denied', cm.exception.message) #now enable the test policy to try again self.pol_client.enable_policy(test_policy_id, headers=self.sa_user_header) gevent.sleep(2) # Wait for events to be fired and policy updated #The previous attempt at this operations should now be allowed. es_obj = IonObject(RT.ExchangeSpace, description='ION test XS', name='ioncore2') with self.assertRaises(BadRequest) as cm: self.ems_client.create_exchange_space(es_obj) self.assertIn('Arguments not set', cm.exception.message) self.pol_client.remove_service_policy('exchange_management', test_policy_id, headers=self.sa_user_header) self.pol_client.delete_policy(test_policy_id, headers=self.sa_user_header) gevent.sleep(2) # Wait for events to be fired and policy updated #The same request that previously was allowed should not be denied es_obj = IonObject(RT.ExchangeSpace, description='ION test XS', name='ioncore2') with self.assertRaises(Unauthorized) as cm: self.ems_client.create_exchange_space(es_obj) self.assertIn( 'exchange_management(create_exchange_space) has been denied', cm.exception.message) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') def test_org_policy(self): #Make sure that the system policies have been loaded policy_list, _ = self.rr_client.find_resources(restype=RT.Policy) self.assertNotEqual( len(policy_list), 0, "The system policies have not been loaded into the Resource Registry" ) with self.assertRaises(BadRequest) as cm: myorg = self.org_client.read_org() self.assertTrue( cm.exception.message == 'The org_id parameter is missing') user_id, valid_until, registered = self.id_client.signon( USER1_CERTIFICATE, True) log.debug("user id=" + user_id) user_roles = get_role_message_headers( self.org_client.find_all_roles_by_user(user_id)) user_header = {'ion-actor-id': user_id, 'ion-actor-roles': user_roles} #Attempt to enroll a user anonymously - should not be allowed with self.assertRaises(Unauthorized) as cm: self.org_client.enroll_member(self.ion_org._id, user_id) self.assertIn('org_management(enroll_member) has been denied', cm.exception.message) #Attempt to let a user enroll themselves - should not be allowed with self.assertRaises(Unauthorized) as cm: self.org_client.enroll_member(self.ion_org._id, user_id, headers=user_header) self.assertIn('org_management(enroll_member) has been denied', cm.exception.message) #Attept to enroll the user in the ION Root org as a manager - should not be allowed since #registration with the system implies membership in the ROOT Org. with self.assertRaises(BadRequest) as cm: self.org_client.enroll_member(self.ion_org._id, user_id, headers=self.sa_user_header) self.assertTrue( cm.exception.message == 'A request to enroll in the root ION Org is not allowed') with self.assertRaises(Unauthorized) as cm: users = self.org_client.find_enrolled_users(self.ion_org._id) self.assertIn('org_management(find_enrolled_users) has been denied', cm.exception.message) with self.assertRaises(Unauthorized) as cm: users = self.org_client.find_enrolled_users(self.ion_org._id, headers=user_header) self.assertIn('org_management(find_enrolled_users) has been denied', cm.exception.message) users = self.org_client.find_enrolled_users( self.ion_org._id, headers=self.sa_user_header) self.assertEqual(len(users), 2) ## test_org_roles and policies roles = self.org_client.find_org_roles(self.ion_org._id) self.assertEqual(len(roles), 3) self.assertItemsEqual([r.name for r in roles], [MANAGER_ROLE, MEMBER_ROLE, ION_MANAGER]) roles = self.org_client.find_roles_by_user(self.ion_org._id, self.system_actor._id, headers=self.sa_user_header) self.assertEqual(len(roles), 3) self.assertItemsEqual([r.name for r in roles], [MEMBER_ROLE, MANAGER_ROLE, ION_MANAGER]) roles = self.org_client.find_roles_by_user(self.ion_org._id, user_id, headers=self.sa_user_header) self.assertEqual(len(roles), 1) self.assertItemsEqual([r.name for r in roles], [MEMBER_ROLE]) with self.assertRaises(NotFound) as nf: org2 = self.org_client.find_org(ORG2) self.assertIn('The Org with name Org2 does not exist', nf.exception.message) org2 = IonObject(RT.Org, name=ORG2, description='A second Org') org2_id = self.org_client.create_org(org2, headers=self.sa_user_header) org2 = self.org_client.find_org(ORG2) self.assertEqual(org2_id, org2._id) roles = self.org_client.find_org_roles(org2_id) self.assertEqual(len(roles), 2) self.assertItemsEqual([r.name for r in roles], [MANAGER_ROLE, MEMBER_ROLE]) operator_role = IonObject(RT.UserRole, name=INSTRUMENT_OPERATOR, label='Instrument Operator', description='Instrument Operator') #First try to add the user role anonymously with self.assertRaises(Unauthorized) as cm: self.org_client.add_user_role(org2_id, operator_role) self.assertIn('org_management(add_user_role) has been denied', cm.exception.message) self.org_client.add_user_role(org2_id, operator_role, headers=self.sa_user_header) roles = self.org_client.find_org_roles(org2_id) self.assertEqual(len(roles), 3) self.assertItemsEqual([r.name for r in roles], [MANAGER_ROLE, MEMBER_ROLE, INSTRUMENT_OPERATOR]) # test requests for enrollments and roles. #First try to find user requests anonymously with self.assertRaises(Unauthorized) as cm: requests = self.org_client.find_requests(org2_id) self.assertIn('org_management(find_requests) has been denied', cm.exception.message) #Next try to find user requests as as a basic member with self.assertRaises(Unauthorized) as cm: requests = self.org_client.find_requests(org2_id, headers=user_header) self.assertIn('org_management(find_requests) has been denied', cm.exception.message) requests = self.org_client.find_requests(org2_id, headers=self.sa_user_header) self.assertEqual(len(requests), 0) # First try to request a role without being a member with self.assertRaises(BadRequest) as cm: req_id = self.org_client.request_role(org2_id, user_id, INSTRUMENT_OPERATOR, headers=user_header) self.assertIn( 'A precondition for this request has not been satisfied: is_enrolled(org_id,user_id)', cm.exception.message) requests = self.org_client.find_requests(org2_id, headers=self.sa_user_header) self.assertEqual(len(requests), 0) req_id = self.org_client.request_enroll(org2_id, user_id, headers=user_header) requests = self.org_client.find_requests(org2_id, headers=self.sa_user_header) self.assertEqual(len(requests), 1) requests = self.org_client.find_user_requests( user_id, org2_id, headers=self.sa_user_header) self.assertEqual(len(requests), 1) #User tried requesting enrollment again - this should fail with self.assertRaises(BadRequest) as cm: req_id = self.org_client.request_enroll(org2_id, user_id, headers=user_header) self.assertIn( 'A precondition for this request has not been satisfied: enroll_req_not_exist(org_id,user_id)', cm.exception.message) #Manager denies the request self.org_client.deny_request(org2_id, req_id, 'To test the deny process', headers=self.sa_user_header) requests = self.org_client.find_requests(org2_id, headers=self.sa_user_header) self.assertEqual(len(requests), 1) self.assertEqual(requests[0].status, REQUEST_DENIED) #Manager approves request self.org_client.approve_request(org2_id, req_id, headers=self.sa_user_header) users = self.org_client.find_enrolled_users( org2_id, headers=self.sa_user_header) self.assertEqual(len(users), 0) #User Accepts request self.org_client.accept_request(org2_id, req_id, headers=user_header) users = self.org_client.find_enrolled_users( org2_id, headers=self.sa_user_header) self.assertEqual(len(users), 1) #User tried requesting enrollment again - this should fail with self.assertRaises(BadRequest) as cm: req_id = self.org_client.request_enroll(org2_id, user_id, headers=user_header) self.assertIn( 'A precondition for this request has not been satisfied: is_not_enrolled(org_id,user_id)', cm.exception.message) req_id = self.org_client.request_role(org2_id, user_id, INSTRUMENT_OPERATOR, headers=user_header) requests = self.org_client.find_requests(org2_id, headers=self.sa_user_header) self.assertEqual(len(requests), 2) requests = self.org_client.find_requests(org2_id, request_status='Open', headers=self.sa_user_header) self.assertEqual(len(requests), 1) requests = self.org_client.find_user_requests(user_id, org2_id, headers=user_header) self.assertEqual(len(requests), 2) requests = self.org_client.find_user_requests( user_id, org2_id, request_type=RT.RoleRequest, headers=user_header) self.assertEqual(len(requests), 1) requests = self.org_client.find_user_requests(user_id, org2_id, request_status="Open", headers=user_header) self.assertEqual(len(requests), 1) ia_list, _ = self.rr_client.find_resources(restype=RT.InstrumentAgent) self.assertEqual(len(ia_list), 0) ia_obj = IonObject(RT.InstrumentAgent, name='Instrument Agent1', description='The first Instrument Agent') with self.assertRaises(Unauthorized) as cm: self.ims_client.create_instrument_agent(ia_obj) self.assertIn( 'instrument_management(create_instrument_agent) has been denied', cm.exception.message) with self.assertRaises(Unauthorized) as cm: self.ims_client.create_instrument_agent(ia_obj, headers=user_header) self.assertIn( 'instrument_management(create_instrument_agent) has been denied', cm.exception.message) #Manager approves request self.org_client.approve_request(org2_id, req_id, headers=self.sa_user_header) requests = self.org_client.find_user_requests(user_id, org2_id, request_status="Open", headers=user_header) self.assertEqual(len(requests), 0) #User accepts request self.org_client.accept_request(org2_id, req_id, headers=user_header) #Refresh headers with new role user_roles = get_role_message_headers( self.org_client.find_all_roles_by_user(user_id)) user_header = {'ion-actor-id': user_id, 'ion-actor-roles': user_roles} self.ims_client.create_instrument_agent(ia_obj, headers=user_header) ia_obj = IonObject(RT.InstrumentAgent, name='Instrument Agent2', description='The second Instrument Agent') self.ims_client.create_instrument_agent(ia_obj, headers=user_header) ia_list, _ = self.rr_client.find_resources(restype=RT.InstrumentAgent) self.assertEqual(len(ia_list), 2) #First make a acquire resource request with an non-enrolled user. with self.assertRaises(BadRequest) as cm: req_id = self.org_client.request_acquire_resource( org2_id, self.system_actor._id, ia_list[0]._id, headers=self.sa_user_header) self.assertIn( 'A precondition for this request has not been satisfied: is_enrolled(org_id,user_id)', cm.exception.message) req_id = self.org_client.request_acquire_resource(org2_id, user_id, ia_list[0]._id, headers=user_header) requests = self.org_client.find_requests(org2_id, headers=self.sa_user_header) self.assertEqual(len(requests), 3) requests = self.org_client.find_user_requests(user_id, org2_id, headers=user_header) self.assertEqual(len(requests), 3) requests = self.org_client.find_user_requests( user_id, org2_id, request_type=RT.ResourceRequest, headers=user_header) self.assertEqual(len(requests), 1) requests = self.org_client.find_user_requests(user_id, org2_id, request_status="Open", headers=user_header) self.assertEqual(len(requests), 1) self.assertEqual(requests[0]._id, req_id) #Manager approves Instrument request self.org_client.approve_request(org2_id, req_id, headers=self.sa_user_header) requests = self.org_client.find_user_requests(user_id, org2_id, request_status="Open", headers=user_header) self.assertEqual(len(requests), 0) #User accepts request self.org_client.accept_request(org2_id, req_id, headers=user_header) #Check commitments commitments, _ = self.rr_client.find_objects(ia_list[0]._id, PRED.hasCommitment, RT.ResourceCommitment) self.assertEqual(len(commitments), 1) commitments, _ = self.rr_client.find_objects(user_id, PRED.hasCommitment, RT.ResourceCommitment) self.assertEqual(len(commitments), 1) #Release the resource self.org_client.release_resource( org2_id, user_id, ia_list[0]._id, headers=self.sa_user_header, timeout=15) #TODO - Refactor release_resource #Check commitments commitments, _ = self.rr_client.find_objects(ia_list[0]._id, PRED.hasCommitment, RT.ResourceCommitment) self.assertEqual(len(commitments), 0) commitments, _ = self.rr_client.find_objects(user_id, PRED.hasCommitment, RT.ResourceCommitment) self.assertEqual(len(commitments), 0)
class TestGovernanceInt(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() #Load a deploy file self.container.start_rel_from_url('res/deploy/r2deploy.yml') #Instantiate a process to represent the test process=GovernanceTestProcess() #Load system policies after container has started all of the services LoadSystemPolicy.op_load_system_policies(process) self.rr_client = ResourceRegistryServiceProcessClient(node=self.container.node, process=process) self.id_client = IdentityManagementServiceProcessClient(node=self.container.node, process=process) self.pol_client = PolicyManagementServiceProcessClient(node=self.container.node, process=process) self.org_client = OrgManagementServiceProcessClient(node=self.container.node, process=process) self.ims_client = InstrumentManagementServiceProcessClient(node=self.container.node, process=process) self.ems_client = ExchangeManagementServiceProcessClient(node=self.container.node, process=process) self.ion_org = self.org_client.find_org() self.system_actor = self.id_client.find_actor_identity_by_name(name=CFG.system.system_actor) log.debug('system actor:' + self.system_actor._id) sa_header_roles = get_role_message_headers(self.org_client.find_all_roles_by_user(self.system_actor._id)) self.sa_user_header = {'ion-actor-id': self.system_actor._id, 'ion-actor-roles': sa_header_roles } @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_basic_policy(self): #Make sure that the system policies have been loaded policy_list,_ = self.rr_client.find_resources(restype=RT.Policy) self.assertNotEqual(len(policy_list),0,"The system policies have not been loaded into the Resource Registry") #Attempt to access an operation in service which does not have specific policies set es_obj = IonObject(RT.ExchangeSpace, description= 'ION test XS', name='ioncore2' ) with self.assertRaises(Unauthorized) as cm: self.ems_client.create_exchange_space(es_obj) self.assertIn( 'exchange_management(create_exchange_space) has been denied',cm.exception.message) #Add a new policy to allow the the above service call. policy_obj = IonObject(RT.Policy, name='Exchange_Management_Test_Policy', definition_type="Service", rule=TEST_POLICY_TEXT, description='Allow specific operations in the Exchange Management Service for anonymous user') test_policy_id = self.pol_client.create_policy(policy_obj, headers=self.sa_user_header) self.pol_client.add_service_policy('exchange_management', test_policy_id, headers=self.sa_user_header) log.info('Policy created: ' + policy_obj.name) gevent.sleep(2) # Wait for events to be fired and policy updated #The previous attempt at this operations should now be allowed. es_obj = IonObject(RT.ExchangeSpace, description= 'ION test XS', name='ioncore2' ) with self.assertRaises(BadRequest) as cm: self.ems_client.create_exchange_space(es_obj) self.assertIn( 'Arguments not set',cm.exception.message) #disable the test policy to try again self.pol_client.disable_policy(test_policy_id, headers=self.sa_user_header) gevent.sleep(2) # Wait for events to be fired and policy updated #The same request that previously was allowed should not be denied es_obj = IonObject(RT.ExchangeSpace, description= 'ION test XS', name='ioncore2' ) with self.assertRaises(Unauthorized) as cm: self.ems_client.create_exchange_space(es_obj) self.assertIn( 'exchange_management(create_exchange_space) has been denied',cm.exception.message) #now enable the test policy to try again self.pol_client.enable_policy(test_policy_id, headers=self.sa_user_header) gevent.sleep(2) # Wait for events to be fired and policy updated #The previous attempt at this operations should now be allowed. es_obj = IonObject(RT.ExchangeSpace, description= 'ION test XS', name='ioncore2' ) with self.assertRaises(BadRequest) as cm: self.ems_client.create_exchange_space(es_obj) self.assertIn( 'Arguments not set',cm.exception.message) self.pol_client.remove_service_policy('exchange_management', test_policy_id, headers=self.sa_user_header) self.pol_client.delete_policy(test_policy_id, headers=self.sa_user_header) gevent.sleep(2) # Wait for events to be fired and policy updated #The same request that previously was allowed should not be denied es_obj = IonObject(RT.ExchangeSpace, description= 'ION test XS', name='ioncore2' ) with self.assertRaises(Unauthorized) as cm: self.ems_client.create_exchange_space(es_obj) self.assertIn( 'exchange_management(create_exchange_space) has been denied',cm.exception.message) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_org_policy(self): #Make sure that the system policies have been loaded policy_list,_ = self.rr_client.find_resources(restype=RT.Policy) self.assertNotEqual(len(policy_list),0,"The system policies have not been loaded into the Resource Registry") with self.assertRaises(BadRequest) as cm: myorg = self.org_client.read_org() self.assertTrue(cm.exception.message == 'The org_id parameter is missing') user_id, valid_until, registered = self.id_client.signon(USER1_CERTIFICATE, True) log.debug( "user id=" + user_id) user_roles = get_role_message_headers(self.org_client.find_all_roles_by_user(user_id)) user_header = {'ion-actor-id': user_id, 'ion-actor-roles': user_roles } #Attempt to enroll a user anonymously - should not be allowed with self.assertRaises(Unauthorized) as cm: self.org_client.enroll_member(self.ion_org._id,user_id) self.assertIn( 'org_management(enroll_member) has been denied',cm.exception.message) #Attempt to let a user enroll themselves - should not be allowed with self.assertRaises(Unauthorized) as cm: self.org_client.enroll_member(self.ion_org._id,user_id, headers=user_header) self.assertIn( 'org_management(enroll_member) has been denied',cm.exception.message) #Attept to enroll the user in the ION Root org as a manager - should not be allowed since #registration with the system implies membership in the ROOT Org. with self.assertRaises(BadRequest) as cm: self.org_client.enroll_member(self.ion_org._id,user_id, headers=self.sa_user_header) self.assertTrue(cm.exception.message == 'A request to enroll in the root ION Org is not allowed') with self.assertRaises(Unauthorized) as cm: users = self.org_client.find_enrolled_users(self.ion_org._id) self.assertIn('org_management(find_enrolled_users) has been denied',cm.exception.message) with self.assertRaises(Unauthorized) as cm: users = self.org_client.find_enrolled_users(self.ion_org._id, headers=user_header) self.assertIn( 'org_management(find_enrolled_users) has been denied',cm.exception.message) users = self.org_client.find_enrolled_users(self.ion_org._id, headers=self.sa_user_header) self.assertEqual(len(users),2) ## test_org_roles and policies roles = self.org_client.find_org_roles(self.ion_org._id) self.assertEqual(len(roles),3) self.assertItemsEqual([r.name for r in roles], [MANAGER_ROLE, MEMBER_ROLE, ION_MANAGER]) roles = self.org_client.find_roles_by_user(self.ion_org._id, self.system_actor._id, headers=self.sa_user_header) self.assertEqual(len(roles),3) self.assertItemsEqual([r.name for r in roles], [MEMBER_ROLE, MANAGER_ROLE, ION_MANAGER]) roles = self.org_client.find_roles_by_user(self.ion_org._id, user_id, headers=self.sa_user_header) self.assertEqual(len(roles),1) self.assertItemsEqual([r.name for r in roles], [MEMBER_ROLE]) with self.assertRaises(NotFound) as nf: org2 = self.org_client.find_org(ORG2) self.assertIn('The Org with name Org2 does not exist',nf.exception.message) org2 = IonObject(RT.Org, name=ORG2, description='A second Org') org2_id = self.org_client.create_org(org2, headers=self.sa_user_header) org2 = self.org_client.find_org(ORG2) self.assertEqual(org2_id, org2._id) roles = self.org_client.find_org_roles(org2_id) self.assertEqual(len(roles),2) self.assertItemsEqual([r.name for r in roles], [MANAGER_ROLE, MEMBER_ROLE]) operator_role = IonObject(RT.UserRole, name=INSTRUMENT_OPERATOR,label='Instrument Operator', description='Instrument Operator') #First try to add the user role anonymously with self.assertRaises(Unauthorized) as cm: self.org_client.add_user_role(org2_id, operator_role) self.assertIn('org_management(add_user_role) has been denied',cm.exception.message) self.org_client.add_user_role(org2_id, operator_role, headers=self.sa_user_header) roles = self.org_client.find_org_roles(org2_id) self.assertEqual(len(roles),3) self.assertItemsEqual([r.name for r in roles], [MANAGER_ROLE, MEMBER_ROLE, INSTRUMENT_OPERATOR]) # test requests for enrollments and roles. #First try to find user requests anonymously with self.assertRaises(Unauthorized) as cm: requests = self.org_client.find_requests(org2_id) self.assertIn('org_management(find_requests) has been denied',cm.exception.message) #Next try to find user requests as as a basic member with self.assertRaises(Unauthorized) as cm: requests = self.org_client.find_requests(org2_id, headers=user_header) self.assertIn('org_management(find_requests) has been denied',cm.exception.message) requests = self.org_client.find_requests(org2_id, headers=self.sa_user_header) self.assertEqual(len(requests),0) # First try to request a role without being a member with self.assertRaises(BadRequest) as cm: req_id = self.org_client.request_role(org2_id, user_id, INSTRUMENT_OPERATOR, headers=user_header ) self.assertIn('A precondition for this request has not been satisfied: is_enrolled(org_id,user_id)',cm.exception.message) requests = self.org_client.find_requests(org2_id, headers=self.sa_user_header) self.assertEqual(len(requests),0) req_id = self.org_client.request_enroll(org2_id, user_id, headers=user_header ) requests = self.org_client.find_requests(org2_id, headers=self.sa_user_header) self.assertEqual(len(requests),1) requests = self.org_client.find_user_requests(user_id, org2_id, headers=self.sa_user_header) self.assertEqual(len(requests),1) #User tried requesting enrollment again - this should fail with self.assertRaises(BadRequest) as cm: req_id = self.org_client.request_enroll(org2_id, user_id, headers=user_header ) self.assertIn('A precondition for this request has not been satisfied: enroll_req_not_exist(org_id,user_id)',cm.exception.message) #Manager denies the request self.org_client.deny_request(org2_id,req_id,'To test the deny process', headers=self.sa_user_header) requests = self.org_client.find_requests(org2_id, headers=self.sa_user_header) self.assertEqual(len(requests),1) self.assertEqual(requests[0].status, REQUEST_DENIED) #Manager approves request self.org_client.approve_request(org2_id,req_id, headers=self.sa_user_header) users = self.org_client.find_enrolled_users(org2_id, headers=self.sa_user_header) self.assertEqual(len(users),0) #User Accepts request self.org_client.accept_request(org2_id,req_id, headers=user_header) users = self.org_client.find_enrolled_users(org2_id, headers=self.sa_user_header) self.assertEqual(len(users),1) #User tried requesting enrollment again - this should fail with self.assertRaises(BadRequest) as cm: req_id = self.org_client.request_enroll(org2_id, user_id, headers=user_header ) self.assertIn('A precondition for this request has not been satisfied: is_not_enrolled(org_id,user_id)',cm.exception.message) req_id = self.org_client.request_role(org2_id, user_id, INSTRUMENT_OPERATOR, headers=user_header ) requests = self.org_client.find_requests(org2_id, headers=self.sa_user_header) self.assertEqual(len(requests),2) requests = self.org_client.find_requests(org2_id,request_status='Open', headers=self.sa_user_header) self.assertEqual(len(requests),1) requests = self.org_client.find_user_requests(user_id, org2_id, headers=user_header) self.assertEqual(len(requests),2) requests = self.org_client.find_user_requests(user_id, org2_id, request_type=RT.RoleRequest, headers=user_header) self.assertEqual(len(requests),1) requests = self.org_client.find_user_requests(user_id, org2_id, request_status="Open", headers=user_header) self.assertEqual(len(requests),1) ia_list,_ = self.rr_client.find_resources(restype=RT.InstrumentAgent) self.assertEqual(len(ia_list),0) ia_obj = IonObject(RT.InstrumentAgent, name='Instrument Agent1', description='The first Instrument Agent') with self.assertRaises(Unauthorized) as cm: self.ims_client.create_instrument_agent(ia_obj) self.assertIn('instrument_management(create_instrument_agent) has been denied',cm.exception.message) with self.assertRaises(Unauthorized) as cm: self.ims_client.create_instrument_agent(ia_obj, headers=user_header) self.assertIn('instrument_management(create_instrument_agent) has been denied',cm.exception.message) #Manager approves request self.org_client.approve_request(org2_id,req_id, headers=self.sa_user_header) requests = self.org_client.find_user_requests(user_id, org2_id, request_status="Open", headers=user_header) self.assertEqual(len(requests),0) #User accepts request self.org_client.accept_request(org2_id, req_id, headers=user_header) #Refresh headers with new role user_roles = get_role_message_headers(self.org_client.find_all_roles_by_user(user_id)) user_header = {'ion-actor-id': user_id, 'ion-actor-roles': user_roles } self.ims_client.create_instrument_agent(ia_obj, headers=user_header) ia_obj = IonObject(RT.InstrumentAgent, name='Instrument Agent2', description='The second Instrument Agent') self.ims_client.create_instrument_agent(ia_obj, headers=user_header) ia_list,_ = self.rr_client.find_resources(restype=RT.InstrumentAgent) self.assertEqual(len(ia_list),2) #First make a acquire resource request with an non-enrolled user. with self.assertRaises(BadRequest) as cm: req_id = self.org_client.request_acquire_resource(org2_id,self.system_actor._id,ia_list[0]._id , headers=self.sa_user_header) self.assertIn('A precondition for this request has not been satisfied: is_enrolled(org_id,user_id)',cm.exception.message) req_id = self.org_client.request_acquire_resource(org2_id,user_id,ia_list[0]._id , headers=user_header) requests = self.org_client.find_requests(org2_id, headers=self.sa_user_header) self.assertEqual(len(requests),3) requests = self.org_client.find_user_requests(user_id, org2_id, headers=user_header) self.assertEqual(len(requests),3) requests = self.org_client.find_user_requests(user_id, org2_id, request_type=RT.ResourceRequest, headers=user_header) self.assertEqual(len(requests),1) requests = self.org_client.find_user_requests(user_id, org2_id, request_status="Open", headers=user_header) self.assertEqual(len(requests),1) self.assertEqual(requests[0]._id, req_id) #Manager approves Instrument request self.org_client.approve_request(org2_id,req_id, headers=self.sa_user_header) requests = self.org_client.find_user_requests(user_id, org2_id, request_status="Open", headers=user_header) self.assertEqual(len(requests),0) #User accepts request self.org_client.accept_request(org2_id,req_id, headers=user_header) #Check commitments commitments, _ = self.rr_client.find_objects(ia_list[0]._id,PRED.hasCommitment, RT.ResourceCommitment) self.assertEqual(len(commitments),1) commitments, _ = self.rr_client.find_objects(user_id,PRED.hasCommitment, RT.ResourceCommitment) self.assertEqual(len(commitments),1) #Release the resource self.org_client.release_resource(org2_id,user_id ,ia_list[0]._id, headers=self.sa_user_header,timeout=15) #TODO - Refactor release_resource #Check commitments commitments, _ = self.rr_client.find_objects(ia_list[0]._id,PRED.hasCommitment, RT.ResourceCommitment) self.assertEqual(len(commitments),0) commitments, _ = self.rr_client.find_objects(user_id,PRED.hasCommitment, RT.ResourceCommitment) self.assertEqual(len(commitments),0)
class ExchangeManager(object): """ Manager object for the CC to manage Exchange related resources. """ def __init__(self, container): log.debug("ExchangeManager initializing ...") self.container = container # Define the callables that can be added to Container public API # @TODO: remove self.container_api = [self.create_xs, self.create_xp, self.create_xn_service, self.create_xn_process, self.create_xn_queue] # Add the public callables to Container for call in self.container_api: setattr(self.container, call.__name__, call) self.default_xs = ExchangeSpace(self, ION_ROOT_XS) self._xs_cache = {} # caching of xs names to RR objects self._default_xs_obj = None # default XS registry object self.org_id = None # mappings self.xs_by_name = { ION_ROOT_XS: self.default_xs } # friendly named XS to XSO self.xn_by_name = {} # friendly named XN to XNO # xn by xs is a property self._chan = None # @TODO specify our own to_name here so we don't get auto-behavior - tricky chicken/egg self._ems_client = ExchangeManagementServiceProcessClient(process=self.container) self._rr_client = ResourceRegistryServiceProcessClient(process=self.container) # TODO: Do more initializing here, not in container @property def xn_by_xs(self): """ Get a list of XNs associated by XS (friendly name). """ ret = {} for xnname, xn in self.xn_by_name.iteritems(): xsn = xn._xs._exchange if not xsn in ret: ret[xsn] = [] ret[xsn].append(xn) return ret def _get_xs_obj(self, name=ION_ROOT_XS): """ Gets a resource-registry represented XS, either via cache or RR request. """ if name in self._xs_cache: return self._xs_cache[name] xs_objs, _ = self._rr_client.find_resources(RT.ExchangeSpace, name=name) if not len(xs_objs) == 1: log.warn("Could not find RR XS object with name: %s", name) return None self._xs_cache[name] = xs_objs[0] return xs_objs[0] def start(self): log.debug("ExchangeManager starting ...") # Establish connection to broker # @TODO: raise error if sux node, ioloop = messaging.make_node() self._transport = AMQPTransport.get_instance() self._client = self._get_channel(node) # Declare root exchange #self.default_xs.ensure_exists(self._get_channel()) return node, ioloop def _ems_available(self): """ Returns True if the EMS is (likely) available and the auto_register CFG entry is True. Has the side effect of bootstrapping the org_id and default_xs's id/rev from the RR. Therefore, cannot be a property. """ if CFG.container.get('exchange', {}).get('auto_register', False): # ok now make sure it's in the directory svc_de = self.container.directory.lookup('/Services/exchange_management') if svc_de is not None: if not self.org_id: # find the default Org org_ids = self._rr_client.find_resources(RT.Org, id_only=True) if not (len(org_ids) and len(org_ids[0]) == 1): log.warn("EMS available but could not find Org") return False self.org_id = org_ids[0][0] log.debug("Bootstrapped Container exchange manager with org id: %s", self.org_id) return True return False def _get_channel(self, node): """ Get a raw channel to be used by all the ensure_exists methods. """ assert self.container # @TODO: needs lock, but so do all these methods if not self._chan: self._chan = blocking_cb(node.client.channel, 'on_open_callback') return self._chan def create_xs(self, name, use_ems=True, exchange_type='topic', durable=False, auto_delete=True): log.debug("ExchangeManager.create_xs: %s", name) xs = ExchangeSpace(self, name, exchange_type=exchange_type, durable=durable, auto_delete=auto_delete) self.xs_by_name[name] = xs if use_ems and self._ems_available(): log.debug("Using EMS to create_xs") # create a RR object xso = ResExchangeSpace(name=name) xso_id = self._ems_client.create_exchange_space(xso, self.org_id) log.debug("Created RR XS object, id: %s", xso_id) else: xs.declare() return xs def delete_xs(self, xs, use_ems=True): """ @type xs ExchangeSpace """ log.debug("ExchangeManager.delete_xs: %s", xs) name = xs._exchange # @TODO this feels wrong del self.xs_by_name[name] if use_ems and self._ems_available(): log.debug("Using EMS to delete_xs") xso = self._get_xs_obj(name) self._ems_client.delete_exchange_space(xso._id) del self._xs_cache[name] else: xs.delete() def create_xp(self, name, xs=None, use_ems=True, **kwargs): log.debug("ExchangeManager.create_xp: %s", name) xs = xs or self.default_xs xp = ExchangePoint(self, name, xs, **kwargs) # put in xn_by_name anyway self.xn_by_name[name] = xp if use_ems and self._ems_available(): log.debug("Using EMS to create_xp") # create an RR object xpo = ResExchangePoint(name=name, topology_type=xp._xptype) xpo_id = self._ems_client.create_exchange_point(xpo, self._get_xs_obj(xs._exchange)._id) # @TODO: _exchange is wrong else: xp.declare() return xp def delete_xp(self, xp, use_ems=True): log.debug("ExchangeManager.delete_xp: name=%s", 'TODO') #xp.build_xname()) name = xp._exchange # @TODO: not right del self.xn_by_name[name] if use_ems and self._ems_available(): log.debug("Using EMS to delete_xp") # find the XP object via RR xpo_ids = self._rr_client.find_resources(RT.ExchangePoint, name=name, id_only=True) if not (len(xpo_ids) and len(xpo_ids[0]) == 1): log.warn("Could not find XP in RR with name of %s", name) xpo_id = xpo_ids[0][0] self._ems_client.delete_exchange_point(xpo_id) else: xp.delete() def _create_xn(self, xn_type, name, xs=None, use_ems=True, **kwargs): xs = xs or self.default_xs log.debug("ExchangeManager._create_xn: type: %s, name=%s, xs=%s, kwargs=%s", xn_type, name, xs, kwargs) if xn_type == "service": xn = ExchangeNameService(self, name, xs, **kwargs) elif xn_type == "process": xn = ExchangeNameProcess(self, name, xs, **kwargs) elif xn_type == "queue": xn = ExchangeNameQueue(self, name, xs, **kwargs) else: raise StandardError("Unknown XN type: %s" % xn_type) self.xn_by_name[name] = xn if use_ems and self._ems_available(): log.debug("Using EMS to create_xn") xno = ResExchangeName(name=name, xn_type=xn.xn_type) self._ems_client.declare_exchange_name(xno, self._get_xs_obj(xs._exchange)._id) # @TODO: exchange is wrong else: xn.declare() return xn def create_xn_service(self, name, xs=None, **kwargs): return self._create_xn('service', name, xs=xs, **kwargs) def create_xn_process(self, name, xs=None, **kwargs): return self._create_xn('process', name, xs=xs, **kwargs) def create_xn_queue(self, name, xs=None, **kwargs): return self._create_xn('queue', name, xs=xs, **kwargs) def delete_xn(self, xn, use_ems=False): log.debug("ExchangeManager.delete_xn: name=%s", "TODO") #xn.build_xlname()) name = xn._queue # @TODO feels wrong del self.xn_by_name[name] if use_ems and self._ems_available(): log.debug("Using EMS to delete_xn") # find the XN object via RR? xno_ids = self._rr_client.find_resources(RT.ExchangeName, name=name, id_only=True) if not (len(xno_ids) and len(xno_ids[0]) == 1): log.warn("Could not find XN in RR with name of %s", name) xno_id = xno_ids[0][0] self._ems_client.undeclare_exchange_name(xno_id) # "canonical name" currently understood to be RR id else: xn.delete() def stop(self, *args, **kwargs): log.debug("ExchangeManager stopping ...") # transport implementations - XOTransport objects call here def declare_exchange(self, exchange, exchange_type='topic', durable=False, auto_delete=True): log.info("ExchangeManager.declare_exchange") self._transport.declare_exchange_impl(self._client, exchange, exchange_type=exchange_type, durable=durable, auto_delete=auto_delete) def delete_exchange(self, exchange, **kwargs): log.info("ExchangeManager.delete_exchange") self._transport.delete_exchange_impl(self._client, exchange, **kwargs) def declare_queue(self, queue, durable=False, auto_delete=True): log.info("ExchangeManager.declare_queue") return self._transport.declare_queue_impl(self._client, queue, durable=durable, auto_delete=auto_delete) def delete_queue(self, queue, **kwargs): log.info("ExchangeManager.delete_queue") self._transport.delete_queue_impl(self._client, queue, **kwargs) def bind(self, exchange, queue, binding): log.info("ExchangeManager.bind") self._transport.bind_impl(self._client, exchange, queue, binding) def unbind(self, exchange, queue, binding): log.info("ExchangeManager.unbind") self._transport.unbind_impl(self._client, exchange, queue, binding)
class ExchangeManager(object): """ Manager object for the CC to manage Exchange related resources. """ def __init__(self, container): log.debug("ExchangeManager initializing ...") self.container = container # Define the callables that can be added to Container public API # @TODO: remove self.container_api = [self.create_xs, self.create_xp, self.create_xn_service, self.create_xn_process, self.create_xn_queue] # Add the public callables to Container for call in self.container_api: setattr(self.container, call.__name__, call) self.default_xs = ExchangeSpace(self, ION_ROOT_XS) self._xs_cache = {} # caching of xs names to RR objects self._default_xs_obj = None # default XS registry object self.org_id = None # mappings self.xs_by_name = { ION_ROOT_XS: self.default_xs } # friendly named XS to XSO self.xn_by_name = {} # friendly named XN to XNO # xn by xs is a property self._chan = None # @TODO specify our own to_name here so we don't get auto-behavior - tricky chicken/egg self._ems_client = ExchangeManagementServiceProcessClient(process=self.container) self._rr_client = ResourceRegistryServiceProcessClient(process=self.container) # mapping of node/ioloop runner by connection name (in config, named via container.messaging.server keys) self._nodes = {} self._ioloops = {} self._client = None self._transport = None self._default_xs_declared = False def start(self): log.debug("ExchangeManager.start") total_count = 0 def handle_failure(name, node): log.warn("Node %s could not be started", name) node.ready.set() # let it fall out below # Establish connection(s) to broker for name, cfgkey in CFG.container.messaging.server.iteritems(): if not cfgkey: continue if cfgkey not in CFG.server: raise ExchangeManagerError("Config key %s (name: %s) (from CFG.container.messaging.server) not in CFG.server" % (cfgkey, name)) total_count += 1 log.debug("Starting connection: %s", name) # start it with a zero timeout so it comes right back to us try: node, ioloop = messaging.make_node(CFG.server[cfgkey], name, 0) # install a finished handler directly on the ioloop just for this startup period fail_handle = lambda _: handle_failure(name, node) ioloop.link(fail_handle) # wait for the node ready event, with a large timeout just in case node_ready = node.ready.wait(timeout=15) # remove the finished handler, we don't care about it here ioloop.unlink(fail_handle) # only add to our list if we started successfully if not node.running: ioloop.kill() # make sure ioloop dead else: self._nodes[name] = node self._ioloops[name] = ioloop except socket.error as e: log.warn("Could not start connection %s due to socket error, continuing", name) fail_count = total_count - len(self._nodes) if fail_count > 0 or total_count == 0: if fail_count == total_count: raise ExchangeManagerError("No node connection was able to start (%d nodes attempted, %d nodes failed)" % (total_count, fail_count)) log.warn("Some nodes could not be started, ignoring for now") # @TODO change when ready self._transport = AMQPTransport.get_instance() self._client = self._get_channel(self._nodes.get('priviledged', self._nodes.values()[0])) # @TODO log.debug("Started %d connections (%s)", len(self._nodes), ",".join(self._nodes.iterkeys())) def stop(self, *args, **kwargs): # ############## # HACK HACK HACK # # It appears during shutdown that when a channel is closed, it's not FULLY closed by the pika connection # until the next round of _handle_events. We have to yield here to let that happen, in order to have close # work fine without blowing up. # ############## time.sleep(0.1) # ############## # /HACK # ############## log.debug("ExchangeManager.stopping (%d connections)", len(self._nodes)) for name in self._nodes: self._nodes[name].stop_node() self._ioloops[name].kill() self._nodes[name].client.ioloop.start() # loop until connection closes # @TODO undeclare root xs?? need to know if last container #self.default_xs.delete() @property def default_node(self): """ Returns the default node connection. """ if 'primary' in self._nodes: return self._nodes['primary'] elif len(self._nodes): log.warn("No primary connection, returning first available") return self._nodes.values()[0] return None @property def xn_by_xs(self): """ Get a list of XNs associated by XS (friendly name). """ ret = {} for xnname, xn in self.xn_by_name.iteritems(): xsn = xn._xs._exchange if not xsn in ret: ret[xsn] = [] ret[xsn].append(xn) return ret def _get_xs_obj(self, name=ION_ROOT_XS): """ Gets a resource-registry represented XS, either via cache or RR request. """ if name in self._xs_cache: return self._xs_cache[name] xs_objs, _ = self._rr_client.find_resources(RT.ExchangeSpace, name=name) if not len(xs_objs) == 1: log.warn("Could not find RR XS object with name: %s", name) return None self._xs_cache[name] = xs_objs[0] return xs_objs[0] def _ems_available(self): """ Returns True if the EMS is (likely) available and the auto_register CFG entry is True. Has the side effect of bootstrapping the org_id and default_xs's id/rev from the RR. Therefore, cannot be a property. """ if CFG.container.get('exchange', {}).get('auto_register', False): # ok now make sure it's in the directory svc_de = self.container.directory.lookup('/Services/exchange_management') if svc_de is not None: if not self.org_id: # find the default Org org_ids = self._rr_client.find_resources(RT.Org, id_only=True) if not (len(org_ids) and len(org_ids[0]) == 1): log.warn("EMS available but could not find Org") return False self.org_id = org_ids[0][0] log.debug("Bootstrapped Container exchange manager with org id: %s", self.org_id) return True return False def _get_channel(self, node): """ Get a raw channel to be used by all the ensure_exists methods. """ assert self.container # @TODO: needs lock, but so do all these methods if not self._chan: self._chan = blocking_cb(node.client.channel, 'on_open_callback') return self._chan def create_xs(self, name, use_ems=True, exchange_type='topic', durable=False, auto_delete=True): log.debug("ExchangeManager.create_xs: %s", name) xs = ExchangeSpace(self, name, exchange_type=exchange_type, durable=durable, auto_delete=auto_delete) self.xs_by_name[name] = xs if use_ems and self._ems_available(): log.debug("Using EMS to create_xs") # create a RR object xso = ResExchangeSpace(name=name) xso_id = self._ems_client.create_exchange_space(xso, self.org_id) log.debug("Created RR XS object, id: %s", xso_id) else: xs.declare() return xs def delete_xs(self, xs, use_ems=True): """ @type xs ExchangeSpace """ log.debug("ExchangeManager.delete_xs: %s", xs) name = xs._exchange # @TODO this feels wrong del self.xs_by_name[name] if use_ems and self._ems_available(): log.debug("Using EMS to delete_xs") xso = self._get_xs_obj(name) self._ems_client.delete_exchange_space(xso._id) del self._xs_cache[name] else: xs.delete() def create_xp(self, name, xs=None, use_ems=True, **kwargs): log.debug("ExchangeManager.create_xp: %s", name) xs = xs or self.default_xs xp = ExchangePoint(self, name, xs, **kwargs) # put in xn_by_name anyway self.xn_by_name[name] = xp if use_ems and self._ems_available(): log.debug("Using EMS to create_xp") # create an RR object xpo = ResExchangePoint(name=name, topology_type=xp._xptype) xpo_id = self._ems_client.create_exchange_point(xpo, self._get_xs_obj(xs._exchange)._id) # @TODO: _exchange is wrong else: xp.declare() return xp def delete_xp(self, xp, use_ems=True): log.debug("ExchangeManager.delete_xp: name=%s", 'TODO') #xp.build_xname()) name = xp._exchange # @TODO: not right del self.xn_by_name[name] if use_ems and self._ems_available(): log.debug("Using EMS to delete_xp") # find the XP object via RR xpo_ids = self._rr_client.find_resources(RT.ExchangePoint, name=name, id_only=True) if not (len(xpo_ids) and len(xpo_ids[0]) == 1): log.warn("Could not find XP in RR with name of %s", name) xpo_id = xpo_ids[0][0] self._ems_client.delete_exchange_point(xpo_id) else: xp.delete() def _create_xn(self, xn_type, name, xs=None, use_ems=True, **kwargs): xs = xs or self.default_xs log.debug("ExchangeManager._create_xn: type: %s, name=%s, xs=%s, kwargs=%s", xn_type, name, xs, kwargs) if xn_type == "service": xn = ExchangeNameService(self, name, xs, **kwargs) elif xn_type == "process": xn = ExchangeNameProcess(self, name, xs, **kwargs) elif xn_type == "queue": xn = ExchangeNameQueue(self, name, xs, **kwargs) else: raise StandardError("Unknown XN type: %s" % xn_type) self.xn_by_name[name] = xn if use_ems and self._ems_available(): log.debug("Using EMS to create_xn") xno = ResExchangeName(name=name, xn_type=xn.xn_type) self._ems_client.declare_exchange_name(xno, self._get_xs_obj(xs._exchange)._id) # @TODO: exchange is wrong else: xn.declare() return xn def create_xn_service(self, name, xs=None, **kwargs): return self._create_xn('service', name, xs=xs, **kwargs) def create_xn_process(self, name, xs=None, **kwargs): return self._create_xn('process', name, xs=xs, **kwargs) def create_xn_queue(self, name, xs=None, **kwargs): return self._create_xn('queue', name, xs=xs, **kwargs) def delete_xn(self, xn, use_ems=False): log.debug("ExchangeManager.delete_xn: name=%s", "TODO") #xn.build_xlname()) name = xn._queue # @TODO feels wrong del self.xn_by_name[name] if use_ems and self._ems_available(): log.debug("Using EMS to delete_xn") # find the XN object via RR? xno_ids = self._rr_client.find_resources(RT.ExchangeName, name=name, id_only=True) if not (len(xno_ids) and len(xno_ids[0]) == 1): log.warn("Could not find XN in RR with name of %s", name) xno_id = xno_ids[0][0] self._ems_client.undeclare_exchange_name(xno_id) # "canonical name" currently understood to be RR id else: xn.delete() def _ensure_default_declared(self): """ Ensures we declared the default exchange space. Needed by most exchange object calls, so each one calls here. """ if not self._default_xs_declared: log.debug("ExchangeManager._ensure_default_declared, declaring default xs") self._default_xs_declared = True self.default_xs.declare() # transport implementations - XOTransport objects call here def declare_exchange(self, exchange, exchange_type='topic', durable=False, auto_delete=True): log.info("ExchangeManager.declare_exchange") self._ensure_default_declared() self._transport.declare_exchange_impl(self._client, exchange, exchange_type=exchange_type, durable=durable, auto_delete=auto_delete) def delete_exchange(self, exchange, **kwargs): log.info("ExchangeManager.delete_exchange") self._ensure_default_declared() self._transport.delete_exchange_impl(self._client, exchange, **kwargs) def declare_queue(self, queue, durable=False, auto_delete=False): log.info("ExchangeManager.declare_queue (queue %s, durable %s, AD %s)", queue, durable, auto_delete) self._ensure_default_declared() return self._transport.declare_queue_impl(self._client, queue, durable=durable, auto_delete=auto_delete) def delete_queue(self, queue, **kwargs): log.info("ExchangeManager.delete_queue") self._ensure_default_declared() self._transport.delete_queue_impl(self._client, queue, **kwargs) def bind(self, exchange, queue, binding): log.info("ExchangeManager.bind") self._ensure_default_declared() self._transport.bind_impl(self._client, exchange, queue, binding) def unbind(self, exchange, queue, binding): log.info("ExchangeManager.unbind") self._ensure_default_declared() self._transport.unbind_impl(self._client, exchange, queue, binding) def get_stats(self, queue): log.info("ExchangeManager.get_stats") self._ensure_default_declared() return self._transport.get_stats(self._client, queue) def purge(self, queue): log.info("ExchangeManager.purge") self._ensure_default_declared() self._transport.purge(self._client, queue)
class ExchangeManager(object): """ Manager object for the CC to manage Exchange related resources. """ def __init__(self, container): log.debug("ExchangeManager initializing ...") self.container = container # Define the callables that can be added to Container public API # @TODO: remove self.container_api = [self.create_xs, self.create_xp, self.create_xn_service, self.create_xn_process, self.create_xn_queue] # Add the public callables to Container for call in self.container_api: setattr(self.container, call.__name__, call) self.default_xs = None self._xs_cache = {} # caching of xs names to RR objects self._default_xs_obj = None # default XS registry object self.org_id = None self._default_xs_declared = False # mappings self.xs_by_name = {} # friendly named XS to XSO self.xn_by_name = {} # friendly named XN to XNO # xn by xs is a property # @TODO specify our own to_name here so we don't get auto-behavior - tricky chicken/egg self._ems_client = ExchangeManagementServiceProcessClient(process=self.container) self._rr_client = ResourceRegistryServiceProcessClient(process=self.container) # mapping of node/ioloop runner by connection name (in config, named via container.messaging.server keys) self._nodes = {} self._ioloops = {} # public toggle switch for if EMS should be used by default self.use_ems = True def start(self): log.debug("ExchangeManager.start") total_count = 0 def handle_failure(name, node): log.warn("Node %s could not be started", name) node.ready.set() # let it fall out below # Establish connection(s) to broker for name, cfgkey in CFG.container.messaging.server.iteritems(): if not cfgkey: continue if cfgkey not in CFG.server: raise ExchangeManagerError("Config key %s (name: %s) (from CFG.container.messaging.server) not in CFG.server" % (cfgkey, name)) total_count += 1 log.debug("Starting connection: %s", name) # start it with a zero timeout so it comes right back to us try: cfg_params = CFG.server[cfgkey] if cfg_params['type'] == 'local': node, ioloop = messaging.make_local_node(0, self.container.local_router) else: node, ioloop = messaging.make_node(cfg_params, name, 0) # install a finished handler directly on the ioloop just for this startup period fail_handle = lambda _: handle_failure(name, node) ioloop.link(fail_handle) # wait for the node ready event, with a large timeout just in case node_ready = node.ready.wait(timeout=15) # remove the finished handler, we don't care about it here ioloop.unlink(fail_handle) # only add to our list if we started successfully if not node.running: ioloop.kill() # make sure ioloop dead else: self._nodes[name] = node self._ioloops[name] = ioloop except socket.error as e: log.warn("Could not start connection %s due to socket error, continuing", name) fail_count = total_count - len(self._nodes) if fail_count > 0 or total_count == 0: if fail_count == total_count: raise ExchangeManagerError("No node connection was able to start (%d nodes attempted, %d nodes failed)" % (total_count, fail_count)) log.warn("Some nodes could not be started, ignoring for now") # @TODO change when ready # load interceptors into each map(lambda x: x.setup_interceptors(CFG.interceptor), self._nodes.itervalues()) # prepare priviledged transport self._priviledged_transport = self.get_transport(self._nodes.get('priviledged', self._nodes.get('primary'))) self._priviledged_transport.lock = True # prevent any attempt to close self.default_xs = ExchangeSpace(self, self._priviledged_transport, ION_ROOT_XS) self.xs_by_name[ION_ROOT_XS] = self.default_xs log.debug("Started %d connections (%s)", len(self._nodes), ",".join(self._nodes.iterkeys())) def stop(self, *args, **kwargs): # ############## # HACK HACK HACK # # It appears during shutdown that when a channel is closed, it's not FULLY closed by the pika connection # until the next round of _handle_events. We have to yield here to let that happen, in order to have close # work fine without blowing up. # ############## time.sleep(0.1) # ############## # /HACK # ############## log.debug("ExchangeManager.stopping (%d connections)", len(self._nodes)) for name in self._nodes: self._nodes[name].stop_node() self._ioloops[name].kill() #self._nodes[name].client.ioloop.start() # loop until connection closes self._priviledged_transport.lock = False self._priviledged_transport.close() # @TODO undeclare root xs?? need to know if last container #self.default_xs.delete() @property def default_node(self): """ Returns the default node connection. """ if 'primary' in self._nodes: return self._nodes['primary'] elif len(self._nodes): log.warn("No primary connection, returning first available") return self._nodes.values()[0] return None @property def xn_by_xs(self): """ Get a list of XNs associated by XS (friendly name). """ ret = {} for xnname, xn in self.xn_by_name.iteritems(): xsn = xn._xs._exchange if not xsn in ret: ret[xsn] = [] ret[xsn].append(xn) return ret def cleanup_xos(self): """ Iterates the list of Exchange Objects and deletes them. Typically used for test cleanup. """ xns = self.xn_by_name.values() # copy as we're removing as we go for xn in xns: if isinstance(xn, ExchangePoint): # @TODO ugh self.delete_xp(xn) else: self.delete_xn(xn) xss = self.xs_by_name.values() for xs in xss: if not (xs == self.default_xs and not self._default_xs_declared): self.delete_xs(xs) # reset xs map to initial state self._default_xs_declared = False self.xs_by_name = { ION_ROOT_XS: self.default_xs } # friendly named XS to XSO def _get_xs_obj(self, name=ION_ROOT_XS): """ Gets a resource-registry represented XS, either via cache or RR request. """ if name in self._xs_cache: return self._xs_cache[name] xs_objs, _ = self._rr.find_resources(RT.ExchangeSpace, name=name) if not len(xs_objs) == 1: log.warn("Could not find RR XS object with name: %s", name) return None self._xs_cache[name] = xs_objs[0] return xs_objs[0] def _ems_available(self): """ Returns True if the EMS is (likely) available and the auto_register CFG entry is True. @TODO: make a property """ if CFG.get_safe('container.exchange.auto_register', False) and self.use_ems: # ok now make sure it's in the directory exchange_service = get_service_registry().is_service_available('exchange_management') if exchange_service: return True return False def _bootstrap_default_org(self): """ Finds an Org resource to be used by create_xs. @TODO: create_xs is being removed, so this will not be needed """ if not self.org_id: # find the default Org root_orgname = CFG.get_safe("system.root_org", "ION") org_ids,_ = self._rr.find_resources(RT.Org, name=root_orgname, id_only=True) if not org_ids or len(org_ids) != 1: log.warn("EMS available but could not find ION root Org") return None self.org_id = org_ids[0] log.debug("Bootstrapped Container exchange manager with org id: %s", self.org_id) return self.org_id @property def _rr(self): """ Returns the active resource registry instance or client. Used to directly contact the resource registry via the container if available, otherwise the messaging client to the RR service is returned. """ if self.container.has_capability('RESOURCE_REGISTRY'): return self.container.resource_registry return self._rr_client def get_transport(self, node): """ Get a transport to be used by operations here. """ assert self.container with node._lock: transport = node._new_transport() return transport def _build_security_headers(self): """ Builds additional security headers to be passed through to EMS. """ # pull context from container ctx = self.container.context.get_context() if isinstance(ctx, dict): return ProcessEndpointUnitMixin.build_security_headers(ctx) return None def create_xs(self, name, use_ems=True, exchange_type='topic', durable=False, auto_delete=True): log.debug("ExchangeManager.create_xs: %s", name) xs = ExchangeSpace(self, self._priviledged_transport, name, exchange_type=exchange_type, durable=durable, auto_delete=auto_delete) if use_ems and self._ems_available(): log.debug("Using EMS to create_xs") org_id = self._bootstrap_default_org() if org_id is None: raise ExchangeManagerError("Could not find org, refusing to create_xs") # create a RR object xso = ResExchangeSpace(name=name) xso_id = self._ems_client.create_exchange_space(xso, org_id, headers=self._build_security_headers()) log.debug("Created RR XS object, id: %s", xso_id) else: self._ensure_default_declared() xs.declare() self.xs_by_name[name] = xs return xs def delete_xs(self, xs, use_ems=True): """ @type xs ExchangeSpace """ log.debug("ExchangeManager.delete_xs: %s", xs) name = xs._exchange # @TODO this feels wrong self.xs_by_name.pop(name, None) # EMS may be running on the same container, which touches this same dict # so delete in the safest way possible # @TODO: does this mean we need to sync xs_by_name and friends in the datastore? if use_ems and self._ems_available(): log.debug("Using EMS to delete_xs") xso = self._get_xs_obj(name) self._ems_client.delete_exchange_space(xso._id, headers=self._build_security_headers()) del self._xs_cache[name] else: try: xs.delete() except TransportError as ex: log.warn("Could not delete XS (%s): %s", name, ex) def create_xp(self, name, xs=None, use_ems=True, **kwargs): log.debug("ExchangeManager.create_xp: %s", name) xs = xs or self.default_xs xp = ExchangePoint(self, self._priviledged_transport, name, xs, **kwargs) # put in xn_by_name anyway self.xn_by_name[name] = xp if use_ems and self._ems_available(): log.debug("Using EMS to create_xp") # create an RR object xpo = ResExchangePoint(name=name, topology_type=xp._xptype) xpo_id = self._ems_client.create_exchange_point(xpo, self._get_xs_obj(xs._exchange)._id, headers=self._build_security_headers()) # @TODO: _exchange is wrong else: self._ensure_default_declared() xp.declare() return xp def delete_xp(self, xp, use_ems=True): log.debug("ExchangeManager.delete_xp: name=%s", 'TODO') # xp.build_xname()) name = xp._exchange # @TODO: not right self.xn_by_name.pop(name, None) # EMS may be running on the same container, which touches this same dict # so delete in the safest way possible # @TODO: does this mean we need to sync xs_by_name and friends in the datastore? if use_ems and self._ems_available(): log.debug("Using EMS to delete_xp") # find the XP object via RR xpo_ids = self._rr.find_resources(RT.ExchangePoint, name=name, id_only=True) if not (len(xpo_ids) and len(xpo_ids[0]) == 1): log.warn("Could not find XP in RR with name of %s", name) xpo_id = xpo_ids[0][0] self._ems_client.delete_exchange_point(xpo_id, headers=self._build_security_headers()) else: try: xp.delete() except TransportError as ex: log.warn("Could not delete XP (%s): %s", name, ex) def _create_xn(self, xn_type, name, xs=None, use_ems=True, **kwargs): xs = xs or self.default_xs log.info("ExchangeManager._create_xn: type: %s, name=%s, xs=%s, kwargs=%s", xn_type, name, xs, kwargs) if xn_type == "service": xn = ExchangeNameService(self, self._priviledged_transport, name, xs, **kwargs) elif xn_type == "process": xn = ExchangeNameProcess(self, self._priviledged_transport, name, xs, **kwargs) elif xn_type == "queue": xn = ExchangeNameQueue(self, self._priviledged_transport, name, xs, **kwargs) else: raise StandardError("Unknown XN type: %s" % xn_type) self.xn_by_name[name] = xn if use_ems and self._ems_available(): log.debug("Using EMS to create_xn") xno = ResExchangeName(name=name, xn_type=xn.xn_type) self._ems_client.declare_exchange_name(xno, self._get_xs_obj(xs._exchange)._id, headers=self._build_security_headers()) # @TODO: exchange is wrong else: self._ensure_default_declared() xn.declare() return xn def create_xn_service(self, name, xs=None, **kwargs): return self._create_xn('service', name, xs=xs, **kwargs) def create_xn_process(self, name, xs=None, **kwargs): return self._create_xn('process', name, xs=xs, **kwargs) def create_xn_queue(self, name, xs=None, **kwargs): return self._create_xn('queue', name, xs=xs, **kwargs) def delete_xn(self, xn, use_ems=False): log.debug("ExchangeManager.delete_xn: name=%s", "TODO") # xn.build_xlname()) name = xn._queue # @TODO feels wrong self.xn_by_name.pop(name, None) # EMS may be running on the same container, which touches this same dict # so delete in the safest way possible # @TODO: does this mean we need to sync xs_by_name and friends in the datastore? if use_ems and self._ems_available(): log.debug("Using EMS to delete_xn") # find the XN object via RR? xno_ids = self._rr.find_resources(RT.ExchangeName, name=name, id_only=True) if not (len(xno_ids) and len(xno_ids[0]) == 1): log.warn("Could not find XN in RR with name of %s", name) xno_id = xno_ids[0][0] self._ems_client.undeclare_exchange_name(xno_id, headers=self._build_security_headers()) # "canonical name" currently understood to be RR id else: try: xn.delete() except TransportError as ex: log.warn("Could not delete XN (%s): %s", name, ex) def _ensure_default_declared(self): """ Ensures we declared the default exchange space. Needed by most exchange object calls, so each one calls here. """ if not self._default_xs_declared: log.debug("ExchangeManager._ensure_default_declared, declaring default xs") self._default_xs_declared = True self.default_xs.declare() def get_definitions(self): """ Rabbit HTTP management API call to get all defined objects on a broker. Returns users, vhosts, queues, exchanges, bindings, rabbit_version, and permissions. """ url = self._get_management_url("definitions") raw_defs = self._call_management(url) return raw_defs def list_nodes(self): """ Rabbit HTTP management API call to get all nodes in a cluster. """ url = self._get_management_url("nodes") nodes = self._call_management(url) return nodes def list_connections(self): """ Rabbit HTTP management API call to get all connections to a broker. """ url = self._get_management_url("connections") conns = self._call_management(url) return conns def list_channels(self): """ Rabbit HTTP management API call to get channels opened on the broker. """ url = self._get_management_url("channels") chans = self._call_management(url) return chans def list_exchanges(self): """ Rabbit HTTP management API call to list exchanges on the broker. Returns a list of exchange names. If you want the full set of properties for each, use _list_exchanges. """ raw_exchanges = self._list_exchanges() exchanges = [x['name'] for x in raw_exchanges] return exchanges def _list_exchanges(self): """ Rabbit HTTP management API call to list exchanges with full properties. This is used by list_exchanges to get a list of names, but does not filter anything. """ url = self._get_management_url("exchanges", "%2f") raw_exchanges = self._call_management(url) return raw_exchanges def list_queues(self, name=None): """ Rabbit HTTP management API call to list names of queues on the broker. Returns a list of queue names. If you want the full properties for each, use _list_queues. @param name If set, filters the list by only including queues with name in them. """ raw_queues = self._list_queues() nl = lambda x: (name is None) or (name is not None and name in x) queues = [x['name'] for x in raw_queues if nl(x['name'])] return queues def _list_queues(self): """ Rabbit HTTP management API call to list queues with full properties. This is used by list_queues to get a list of names, but does not filter anything. """ url = self._get_management_url("queues", "%2f") raw_queues = self._call_management(url) return raw_queues def get_queue_info(self, queue): """ Rabbit HTTP management API call to get full properties of a single queue. """ url = self._get_management_url("queues", "%2f", queue) queue_info = self._call_management(url) return queue_info def list_bindings(self, exchange=None, queue=None): """ Rabbit HTTP management API call to list bindings. Returns a list of tuples formatted as (exchange, queue, routing_key, properties_key aka binding id). This method can optionally filter queues or exchanges (or both) by specifying strings to exchange/queue keyword arguments. If you want the full list of properties unfiltered, call _list_bindings instead. The properties_key is used to delete a binding. If you want to get the bindings on a specific queue or exchange, don't use the filters here, but call the specific list_bindings_for_queue or list_bindings_for_exchange, as they will not result in a large result from the management API. @param exchange If set, filters the list by only including bindings with exchanges that have the passed value in them. @param queue If set, filters the list by only including bindings with queues that have the passed value in them. """ raw_binds = self._list_bindings() ql = lambda x: (queue is None) or (queue is not None and queue in x) el = lambda x: (exchange is None) or (exchange is not None and exchange in x) binds = [(x['source'], x['destination'], x['routing_key'], x['properties_key']) for x in raw_binds if x['destination_type'] == 'queue' and x['source'] != '' and ql(x['destination']) and el(x['source'])] return binds def _list_bindings(self): """ Rabbit HTTP management API call to list bindings with full properties. This is used by list_bindings to get a list of binding tuples, but does not filter anything. """ url = self._get_management_url("bindings", "%2f") raw_binds = self._call_management(url) return raw_binds def list_bindings_for_queue(self, queue): """ Rabbit HTTP management API call to list bindings for a queue. Returns a list of tuples formatted as (exchange, queue, routing_key, properties_key aka binding id). If you want the full list of properties for all the bindings, call _list_bindings_for_queue instead. This method is much more efficient than calling list_bindings with a filter. @param queue The name of the queue you want bindings for. """ raw_binds = self._list_bindings_for_queue(queue) binds = [(x['source'], x['destination'], x['routing_key'], x['properties_key']) for x in raw_binds if x['source'] != ''] return binds def _list_bindings_for_queue(self, queue): """ Rabbit HTTP management API call to list bindings on a queue with full properties. This is used by list_bindings_for_queue to get a list of binding tuples, but does not filter anything. """ url = self._get_management_url("queues", "%2f", queue, "bindings") raw_binds = self._call_management(url) return raw_binds def list_bindings_for_exchange(self, exchange): """ Rabbit HTTP management API call to list bindings for an exchange. Returns a list of tuples formatted as (exchange, queue, routing_key, properties_key aka binding id). If you want the full list of properties for all the bindings, call _list_bindings_for_exchange instead. This method is much more efficient than calling list_bindings with a filter. @param exchange The name of the exchange you want bindings for. """ raw_binds = self._list_bindings_for_exchange(exchange) binds = [(x['source'], x['destination'], x['routing_key'], x['properties_key']) for x in raw_binds if x['source'] != ''] return binds def _list_bindings_for_exchange(self, exchange): """ Rabbit HTTP management API call to list bindings for an exchange with full properties. This is used by list_bindings_for_exchange to get a list of binding tuples, but does not filter anything. """ url = self._get_management_url("exchanges", "%2f", exchange, "bindings", "source") raw_binds = self._call_management(url) return raw_binds def delete_binding(self, exchange, queue, binding_prop_key): """ Rabbit HTTP management API call to delete a binding. You may also use delete_binding_tuple to directly pass the tuples returned by any of the list binding calls. """ # have to urlencode the %, even though it is already urlencoded - rabbit needs this. url = self._get_management_url("bindings", "%2f", "e", exchange, "q", queue, binding_prop_key.replace("%", "%25")) self._call_management_delete(url) return True def delete_binding_tuple(self, binding_tuple): """ Rabbit HTTP management API call to delete a binding using a tuple from our list binding methods. """ return self.delete_binding(binding_tuple[0], binding_tuple[1], binding_tuple[3]) def delete_queue(self, queue): """ Rabbit HTTP management API call to delete a queue. """ url = self._get_management_url("queues", "%2f", queue) self._call_management_delete(url) def purge_queue(self, queue): """ Rabbit HTTP management API call to purge a queue. """ url = self._get_management_url("queues", "%2f", queue, "contents") self._call_management_delete(url) return True def _get_management_url(self, *feats): """ Builds a URL to be used with the Rabbit HTTP management API. """ node = self._nodes.get('priviledged', self._nodes.values()[0]) host = node.client.parameters.host url = "http://%s:%s/api/%s" % (host, CFG.get_safe("container.exchange.management.port", "55672"), "/".join(feats)) return url def _call_management(self, url): """ Makes a GET HTTP request to the Rabbit HTTP management API. This method will raise an exception if a non-200 HTTP status code is returned. @param url A URL to be used, build one with _get_management_url. """ return self._make_management_call(url) def _call_management_delete(self, url): """ Makes an HTTP DELETE request to the Rabbit HTTP management API. This method will raise an exception if a non-200 HTTP status code is returned. @param url A URL to be used, build one with _get_management_url. """ return self._make_management_call(url, method="delete") def _make_management_call(self, url, use_ems=True, method="get", data=None): """ Makes a call to the Rabbit HTTP management API using the passed in HTTP method. """ log.debug("Calling rabbit API management (%s): %s", method, url) if use_ems and self._ems_available(): log.debug("Directing call to EMS") content = self._ems_client.call_management(url, method, headers=self._build_security_headers()) else: meth = getattr(requests, method) try: username = CFG.get_safe("container.exchange.management.username", "guest") password = CFG.get_safe("container.exchange.management.password", "guest") with gevent.timeout.Timeout(10): r = meth(url, auth=(username, password), data=data) r.raise_for_status() if not r.content == "": content = json.loads(r.content) else: content = None except gevent.timeout.Timeout as ex: raise Timeout(str(ex)) except requests.exceptions.Timeout as ex: raise Timeout(str(ex)) except (requests.exceptions.ConnectionError, socket.error) as ex: raise ServiceUnavailable(str(ex)) except requests.exceptions.RequestException as ex: # the generic base exception all requests' exceptions inherit from, raise our # general server error too. raise ServerError(str(ex)) return content
class ExchangeManager(object): """ Manager object for the CC to manage Exchange related resources. """ def __init__(self, container): log.debug("ExchangeManager initializing ...") self.container = container # Define the callables that can be added to Container public API # @TODO: remove self.container_api = [ self.create_xs, self.create_xp, self.create_xn_service, self.create_xn_process, self.create_xn_queue ] # Add the public callables to Container for call in self.container_api: setattr(self.container, call.__name__, call) self.default_xs = None self._xs_cache = {} # caching of xs names to RR objects self._default_xs_obj = None # default XS registry object self.org_id = None self._default_xs_declared = False # mappings self.xs_by_name = {} # friendly named XS to XSO self.xn_by_name = {} # friendly named XN to XNO # xn by xs is a property # @TODO specify our own to_name here so we don't get auto-behavior - tricky chicken/egg self._ems_client = ExchangeManagementServiceProcessClient( process=self.container) self._rr_client = ResourceRegistryServiceProcessClient( process=self.container) # mapping of node/ioloop runner by connection name (in config, named via container.messaging.server keys) self._nodes = {} self._ioloops = {} # public toggle switch for if EMS should be used by default self.use_ems = True def start(self): log.debug("ExchangeManager.start") total_count = 0 def handle_failure(name, node): log.warn("Node %s could not be started", name) node.ready.set() # let it fall out below # Establish connection(s) to broker for name, cfgkey in CFG.container.messaging.server.iteritems(): if not cfgkey: continue if cfgkey not in CFG.server: raise ExchangeManagerError( "Config key %s (name: %s) (from CFG.container.messaging.server) not in CFG.server" % (cfgkey, name)) total_count += 1 log.debug("Starting connection: %s", name) # start it with a zero timeout so it comes right back to us try: cfg_params = CFG.server[cfgkey] if cfg_params['type'] == 'local': node, ioloop = messaging.make_local_node( 0, self.container.local_router) else: node, ioloop = messaging.make_node(cfg_params, name, 0) # install a finished handler directly on the ioloop just for this startup period fail_handle = lambda _: handle_failure(name, node) ioloop.link(fail_handle) # wait for the node ready event, with a large timeout just in case node_ready = node.ready.wait(timeout=15) # remove the finished handler, we don't care about it here ioloop.unlink(fail_handle) # only add to our list if we started successfully if not node.running: ioloop.kill() # make sure ioloop dead else: self._nodes[name] = node self._ioloops[name] = ioloop except socket.error as e: log.warn( "Could not start connection %s due to socket error, continuing", name) fail_count = total_count - len(self._nodes) if fail_count > 0 or total_count == 0: if fail_count == total_count: raise ExchangeManagerError( "No node connection was able to start (%d nodes attempted, %d nodes failed)" % (total_count, fail_count)) log.warn("Some nodes could not be started, ignoring for now" ) # @TODO change when ready # load interceptors into each map(lambda x: x.setup_interceptors(CFG.interceptor), self._nodes.itervalues()) # prepare priviledged transport self._priviledged_transport = self.get_transport( self._nodes.get('priviledged', self._nodes.get('primary'))) self._priviledged_transport.lock = True # prevent any attempt to close self.default_xs = ExchangeSpace(self, self._priviledged_transport, ION_ROOT_XS) self.xs_by_name[ION_ROOT_XS] = self.default_xs log.debug("Started %d connections (%s)", len(self._nodes), ",".join(self._nodes.iterkeys())) def stop(self, *args, **kwargs): # ############## # HACK HACK HACK # # It appears during shutdown that when a channel is closed, it's not FULLY closed by the pika connection # until the next round of _handle_events. We have to yield here to let that happen, in order to have close # work fine without blowing up. # ############## time.sleep(0.1) # ############## # /HACK # ############## log.debug("ExchangeManager.stopping (%d connections)", len(self._nodes)) for name in self._nodes: self._nodes[name].stop_node() self._ioloops[name].kill() #self._nodes[name].client.ioloop.start() # loop until connection closes self._priviledged_transport.lock = False self._priviledged_transport.close() # @TODO undeclare root xs?? need to know if last container #self.default_xs.delete() @property def default_node(self): """ Returns the default node connection. """ if 'primary' in self._nodes: return self._nodes['primary'] elif len(self._nodes): log.warn("No primary connection, returning first available") return self._nodes.values()[0] return None @property def xn_by_xs(self): """ Get a list of XNs associated by XS (friendly name). """ ret = {} for xnname, xn in self.xn_by_name.iteritems(): xsn = xn._xs._exchange if not xsn in ret: ret[xsn] = [] ret[xsn].append(xn) return ret def cleanup_xos(self): """ Iterates the list of Exchange Objects and deletes them. Typically used for test cleanup. """ xns = self.xn_by_name.values() # copy as we're removing as we go for xn in xns: if isinstance(xn, ExchangePoint): # @TODO ugh self.delete_xp(xn) else: self.delete_xn(xn) xss = self.xs_by_name.values() for xs in xss: if not (xs == self.default_xs and not self._default_xs_declared): self.delete_xs(xs) # reset xs map to initial state self._default_xs_declared = False self.xs_by_name = { ION_ROOT_XS: self.default_xs } # friendly named XS to XSO def _get_xs_obj(self, name=ION_ROOT_XS): """ Gets a resource-registry represented XS, either via cache or RR request. """ if name in self._xs_cache: return self._xs_cache[name] xs_objs, _ = self._rr.find_resources(RT.ExchangeSpace, name=name) if not len(xs_objs) == 1: log.warn("Could not find RR XS object with name: %s", name) return None self._xs_cache[name] = xs_objs[0] return xs_objs[0] def _ems_available(self): """ Returns True if the EMS is (likely) available and the auto_register CFG entry is True. @TODO: make a property """ if CFG.get_safe('container.exchange.auto_register', False) and self.use_ems: # ok now make sure it's in the directory exchange_service = get_service_registry().is_service_available( 'exchange_management') if exchange_service: return True return False def _bootstrap_default_org(self): """ Finds an Org resource to be used by create_xs. @TODO: create_xs is being removed, so this will not be needed """ if not self.org_id: # find the default Org root_orgname = CFG.get_safe("system.root_org", "ION") org_ids, _ = self._rr.find_resources(RT.Org, name=root_orgname, id_only=True) if not org_ids or len(org_ids) != 1: log.warn("EMS available but could not find ION root Org") return None self.org_id = org_ids[0] log.debug( "Bootstrapped Container exchange manager with org id: %s", self.org_id) return self.org_id @property def _rr(self): """ Returns the active resource registry instance or client. Used to directly contact the resource registry via the container if available, otherwise the messaging client to the RR service is returned. """ if self.container.has_capability('RESOURCE_REGISTRY'): return self.container.resource_registry return self._rr_client def get_transport(self, node): """ Get a transport to be used by operations here. """ assert self.container with node._lock: transport = node._new_transport() return transport def _build_security_headers(self): """ Builds additional security headers to be passed through to EMS. """ # pull context from container ctx = self.container.context.get_context() if isinstance(ctx, dict): return ProcessEndpointUnitMixin.build_security_headers(ctx) return None def create_xs(self, name, use_ems=True, exchange_type='topic', durable=False, auto_delete=True): log.debug("ExchangeManager.create_xs: %s", name) xs = ExchangeSpace(self, self._priviledged_transport, name, exchange_type=exchange_type, durable=durable, auto_delete=auto_delete) if use_ems and self._ems_available(): log.debug("Using EMS to create_xs") org_id = self._bootstrap_default_org() if org_id is None: raise ExchangeManagerError( "Could not find org, refusing to create_xs") # create a RR object xso = ResExchangeSpace(name=name) xso_id = self._ems_client.create_exchange_space( xso, org_id, headers=self._build_security_headers()) log.debug("Created RR XS object, id: %s", xso_id) else: self._ensure_default_declared() xs.declare() self.xs_by_name[name] = xs return xs def delete_xs(self, xs, use_ems=True): """ @type xs ExchangeSpace """ log.debug("ExchangeManager.delete_xs: %s", xs) name = xs._exchange # @TODO this feels wrong self.xs_by_name.pop( name, None ) # EMS may be running on the same container, which touches this same dict # so delete in the safest way possible # @TODO: does this mean we need to sync xs_by_name and friends in the datastore? if use_ems and self._ems_available(): log.debug("Using EMS to delete_xs") xso = self._get_xs_obj(name) self._ems_client.delete_exchange_space( xso._id, headers=self._build_security_headers()) del self._xs_cache[name] else: try: xs.delete() except TransportError as ex: log.warn("Could not delete XS (%s): %s", name, ex) def create_xp(self, name, xs=None, use_ems=True, **kwargs): log.debug("ExchangeManager.create_xp: %s", name) xs = xs or self.default_xs xp = ExchangePoint(self, self._priviledged_transport, name, xs, **kwargs) # put in xn_by_name anyway self.xn_by_name[name] = xp if use_ems and self._ems_available(): log.debug("Using EMS to create_xp") # create an RR object xpo = ResExchangePoint(name=name, topology_type=xp._xptype) xpo_id = self._ems_client.create_exchange_point( xpo, self._get_xs_obj(xs._exchange)._id, headers=self._build_security_headers( )) # @TODO: _exchange is wrong else: self._ensure_default_declared() xp.declare() return xp def delete_xp(self, xp, use_ems=True): log.debug("ExchangeManager.delete_xp: name=%s", 'TODO') # xp.build_xname()) name = xp._exchange # @TODO: not right self.xn_by_name.pop( name, None ) # EMS may be running on the same container, which touches this same dict # so delete in the safest way possible # @TODO: does this mean we need to sync xs_by_name and friends in the datastore? if use_ems and self._ems_available(): log.debug("Using EMS to delete_xp") # find the XP object via RR xpo_ids = self._rr.find_resources(RT.ExchangePoint, name=name, id_only=True) if not (len(xpo_ids) and len(xpo_ids[0]) == 1): log.warn("Could not find XP in RR with name of %s", name) xpo_id = xpo_ids[0][0] self._ems_client.delete_exchange_point( xpo_id, headers=self._build_security_headers()) else: try: xp.delete() except TransportError as ex: log.warn("Could not delete XP (%s): %s", name, ex) def _create_xn(self, xn_type, name, xs=None, use_ems=True, **kwargs): xs = xs or self.default_xs log.info( "ExchangeManager._create_xn: type: %s, name=%s, xs=%s, kwargs=%s", xn_type, name, xs, kwargs) if xn_type == "service": xn = ExchangeNameService(self, self._priviledged_transport, name, xs, **kwargs) elif xn_type == "process": xn = ExchangeNameProcess(self, self._priviledged_transport, name, xs, **kwargs) elif xn_type == "queue": xn = ExchangeNameQueue(self, self._priviledged_transport, name, xs, **kwargs) else: raise StandardError("Unknown XN type: %s" % xn_type) self.xn_by_name[name] = xn if use_ems and self._ems_available(): log.debug("Using EMS to create_xn") xno = ResExchangeName(name=name, xn_type=xn.xn_type) self._ems_client.declare_exchange_name( xno, self._get_xs_obj(xs._exchange)._id, headers=self._build_security_headers( )) # @TODO: exchange is wrong else: self._ensure_default_declared() xn.declare() return xn def create_xn_service(self, name, xs=None, **kwargs): return self._create_xn('service', name, xs=xs, **kwargs) def create_xn_process(self, name, xs=None, **kwargs): return self._create_xn('process', name, xs=xs, **kwargs) def create_xn_queue(self, name, xs=None, **kwargs): return self._create_xn('queue', name, xs=xs, **kwargs) def delete_xn(self, xn, use_ems=False): log.debug("ExchangeManager.delete_xn: name=%s", "TODO") # xn.build_xlname()) name = xn._queue # @TODO feels wrong self.xn_by_name.pop( name, None ) # EMS may be running on the same container, which touches this same dict # so delete in the safest way possible # @TODO: does this mean we need to sync xs_by_name and friends in the datastore? if use_ems and self._ems_available(): log.debug("Using EMS to delete_xn") # find the XN object via RR? xno_ids = self._rr.find_resources(RT.ExchangeName, name=name, id_only=True) if not (len(xno_ids) and len(xno_ids[0]) == 1): log.warn("Could not find XN in RR with name of %s", name) xno_id = xno_ids[0][0] self._ems_client.undeclare_exchange_name( xno_id, headers=self._build_security_headers( )) # "canonical name" currently understood to be RR id else: try: xn.delete() except TransportError as ex: log.warn("Could not delete XN (%s): %s", name, ex) def _ensure_default_declared(self): """ Ensures we declared the default exchange space. Needed by most exchange object calls, so each one calls here. """ if not self._default_xs_declared: log.debug( "ExchangeManager._ensure_default_declared, declaring default xs" ) self._default_xs_declared = True self.default_xs.declare() def get_definitions(self): """ Rabbit HTTP management API call to get all defined objects on a broker. Returns users, vhosts, queues, exchanges, bindings, rabbit_version, and permissions. """ url = self._get_management_url("definitions") raw_defs = self._call_management(url) return raw_defs def list_nodes(self): """ Rabbit HTTP management API call to get all nodes in a cluster. """ url = self._get_management_url("nodes") nodes = self._call_management(url) return nodes def list_connections(self): """ Rabbit HTTP management API call to get all connections to a broker. """ url = self._get_management_url("connections") conns = self._call_management(url) return conns def list_channels(self): """ Rabbit HTTP management API call to get channels opened on the broker. """ url = self._get_management_url("channels") chans = self._call_management(url) return chans def list_exchanges(self): """ Rabbit HTTP management API call to list exchanges on the broker. Returns a list of exchange names. If you want the full set of properties for each, use _list_exchanges. """ raw_exchanges = self._list_exchanges() exchanges = [x['name'] for x in raw_exchanges] return exchanges def _list_exchanges(self): """ Rabbit HTTP management API call to list exchanges with full properties. This is used by list_exchanges to get a list of names, but does not filter anything. """ url = self._get_management_url("exchanges", "%2f") raw_exchanges = self._call_management(url) return raw_exchanges def list_queues(self, name=None): """ Rabbit HTTP management API call to list names of queues on the broker. Returns a list of queue names. If you want the full properties for each, use _list_queues. @param name If set, filters the list by only including queues with name in them. """ raw_queues = self._list_queues() nl = lambda x: (name is None) or (name is not None and name in x) queues = [x['name'] for x in raw_queues if nl(x['name'])] return queues def _list_queues(self): """ Rabbit HTTP management API call to list queues with full properties. This is used by list_queues to get a list of names, but does not filter anything. """ url = self._get_management_url("queues", "%2f") raw_queues = self._call_management(url) return raw_queues def get_queue_info(self, queue): """ Rabbit HTTP management API call to get full properties of a single queue. """ url = self._get_management_url("queues", "%2f", queue) queue_info = self._call_management(url) return queue_info def list_bindings(self, exchange=None, queue=None): """ Rabbit HTTP management API call to list bindings. Returns a list of tuples formatted as (exchange, queue, routing_key, properties_key aka binding id). This method can optionally filter queues or exchanges (or both) by specifying strings to exchange/queue keyword arguments. If you want the full list of properties unfiltered, call _list_bindings instead. The properties_key is used to delete a binding. If you want to get the bindings on a specific queue or exchange, don't use the filters here, but call the specific list_bindings_for_queue or list_bindings_for_exchange, as they will not result in a large result from the management API. @param exchange If set, filters the list by only including bindings with exchanges that have the passed value in them. @param queue If set, filters the list by only including bindings with queues that have the passed value in them. """ raw_binds = self._list_bindings() ql = lambda x: (queue is None) or (queue is not None and queue in x) el = lambda x: (exchange is None) or (exchange is not None and exchange in x) binds = [(x['source'], x['destination'], x['routing_key'], x['properties_key']) for x in raw_binds if x['destination_type'] == 'queue' and x['source'] != '' and ql(x['destination']) and el(x['source'])] return binds def _list_bindings(self): """ Rabbit HTTP management API call to list bindings with full properties. This is used by list_bindings to get a list of binding tuples, but does not filter anything. """ url = self._get_management_url("bindings", "%2f") raw_binds = self._call_management(url) return raw_binds def list_bindings_for_queue(self, queue): """ Rabbit HTTP management API call to list bindings for a queue. Returns a list of tuples formatted as (exchange, queue, routing_key, properties_key aka binding id). If you want the full list of properties for all the bindings, call _list_bindings_for_queue instead. This method is much more efficient than calling list_bindings with a filter. @param queue The name of the queue you want bindings for. """ raw_binds = self._list_bindings_for_queue(queue) binds = [(x['source'], x['destination'], x['routing_key'], x['properties_key']) for x in raw_binds if x['source'] != ''] return binds def _list_bindings_for_queue(self, queue): """ Rabbit HTTP management API call to list bindings on a queue with full properties. This is used by list_bindings_for_queue to get a list of binding tuples, but does not filter anything. """ url = self._get_management_url("queues", "%2f", queue, "bindings") raw_binds = self._call_management(url) return raw_binds def list_bindings_for_exchange(self, exchange): """ Rabbit HTTP management API call to list bindings for an exchange. Returns a list of tuples formatted as (exchange, queue, routing_key, properties_key aka binding id). If you want the full list of properties for all the bindings, call _list_bindings_for_exchange instead. This method is much more efficient than calling list_bindings with a filter. @param exchange The name of the exchange you want bindings for. """ raw_binds = self._list_bindings_for_exchange(exchange) binds = [(x['source'], x['destination'], x['routing_key'], x['properties_key']) for x in raw_binds if x['source'] != ''] return binds def _list_bindings_for_exchange(self, exchange): """ Rabbit HTTP management API call to list bindings for an exchange with full properties. This is used by list_bindings_for_exchange to get a list of binding tuples, but does not filter anything. """ url = self._get_management_url("exchanges", "%2f", exchange, "bindings", "source") raw_binds = self._call_management(url) return raw_binds def delete_binding(self, exchange, queue, binding_prop_key): """ Rabbit HTTP management API call to delete a binding. You may also use delete_binding_tuple to directly pass the tuples returned by any of the list binding calls. """ # have to urlencode the %, even though it is already urlencoded - rabbit needs this. url = self._get_management_url("bindings", "%2f", "e", exchange, "q", queue, binding_prop_key.replace("%", "%25")) self._call_management_delete(url) return True def delete_binding_tuple(self, binding_tuple): """ Rabbit HTTP management API call to delete a binding using a tuple from our list binding methods. """ return self.delete_binding(binding_tuple[0], binding_tuple[1], binding_tuple[3]) def delete_queue(self, queue): """ Rabbit HTTP management API call to delete a queue. """ url = self._get_management_url("queues", "%2f", queue) self._call_management_delete(url) def purge_queue(self, queue): """ Rabbit HTTP management API call to purge a queue. """ url = self._get_management_url("queues", "%2f", queue, "contents") self._call_management_delete(url) return True def _get_management_url(self, *feats): """ Builds a URL to be used with the Rabbit HTTP management API. """ node = self._nodes.get('priviledged', self._nodes.values()[0]) host = node.client.parameters.host url = "http://%s:%s/api/%s" % ( host, CFG.get_safe("container.exchange.management.port", "55672"), "/".join(feats)) return url def _call_management(self, url): """ Makes a GET HTTP request to the Rabbit HTTP management API. This method will raise an exception if a non-200 HTTP status code is returned. @param url A URL to be used, build one with _get_management_url. """ return self._make_management_call(url) def _call_management_delete(self, url): """ Makes an HTTP DELETE request to the Rabbit HTTP management API. This method will raise an exception if a non-200 HTTP status code is returned. @param url A URL to be used, build one with _get_management_url. """ return self._make_management_call(url, method="delete") def _make_management_call(self, url, use_ems=True, method="get", data=None): """ Makes a call to the Rabbit HTTP management API using the passed in HTTP method. """ log.debug("Calling rabbit API management (%s): %s", method, url) if use_ems and self._ems_available(): log.debug("Directing call to EMS") content = self._ems_client.call_management( url, method, headers=self._build_security_headers()) else: meth = getattr(requests, method) try: username = CFG.get_safe( "container.exchange.management.username", "guest") password = CFG.get_safe( "container.exchange.management.password", "guest") with gevent.timeout.Timeout(10): r = meth(url, auth=(username, password), data=data) r.raise_for_status() if not r.content == "": content = json.loads(r.content) else: content = None except gevent.timeout.Timeout as ex: raise Timeout(str(ex)) except requests.exceptions.Timeout as ex: raise Timeout(str(ex)) except (requests.exceptions.ConnectionError, socket.error) as ex: raise ServiceUnavailable(str(ex)) except requests.exceptions.RequestException as ex: # the generic base exception all requests' exceptions inherit from, raise our # general server error too. raise ServerError(str(ex)) return content