Example #1
0
    def test_process_engines_no_mapping(self):
        registry = EngineRegistry.from_config(ENGINE_CONF1, default="engine1")

        definition = dict(executable={"module": "some.module", "class": "SomeClass"})
        self.assertIsNone(registry.get_process_definition_engine_id(definition))

        registry = EngineRegistry.from_config(ENGINE_CONF1, default="engine1", process_engines={})
        self.assertIsNone(registry.get_process_definition_engine_id(definition))
Example #2
0
    def test_process_engines(self):

        process_engines = {
            "a.b.c.D": "engine1",
            "a.b.c": "engine2",
            "a": "engine3",
            "e.f": "engine3",
            "e.f.G": "engine1",
        }
        registry = EngineRegistry.from_config(ENGINE_CONF1, default="engine1", process_engines=process_engines)

        definition = dict(executable={"module": "a.b.c", "class": "D"})
        self.assertEqual(registry.get_process_definition_engine_id(definition), "engine1")

        definition = dict(executable={"module": "a.b.c", "class": "E"})
        self.assertEqual(registry.get_process_definition_engine_id(definition), "engine2")

        definition = dict(executable={"module": "a.b.x", "class": "D"})
        self.assertEqual(registry.get_process_definition_engine_id(definition), "engine3")

        definition = dict(executable={"module": "e", "class": "B"})
        self.assertEqual(registry.get_process_definition_engine_id(definition), None)

        definition = dict(executable={"module": "e.f", "class": "B"})
        self.assertEqual(registry.get_process_definition_engine_id(definition), "engine3")

        definition = dict(executable={"module": "e.f", "class": "G"})
        self.assertEqual(registry.get_process_definition_engine_id(definition), "engine1")
Example #3
0
 def setUp(self):
     self.store = self.get_store()
     self.registry = EngineRegistry.from_config(self.engine_conf)
     self.resource_client = Mock()
     self.notifier = MockNotifier()
     self.core = ProcessDispatcherCore(self.store, self.registry,
         self.resource_client, self.notifier)
Example #4
0
    def setUp(self):
        self.store = self.setup_store()
        self.registry = EngineRegistry.from_config(self.engine_conf)
        self.resource_client = MockResourceClient()
        self.notifier = MockNotifier()
        self.core = ProcessDispatcherCore(self.store, self.registry,
            self.resource_client, self.notifier)
        self.doctor = PDDoctor(self.core, self.store)

        self.docthread = None

        self.monitor = None
    def __init__(self, conf, service):
        engine_conf = conf.get('engines', {})
        self.store = ProcessDispatcherStore()
        self.registry = EngineRegistry.from_config(engine_conf)

        # The Process Dispatcher communicates with EE Agents over ION messaging
        # but it still uses dashi to talk to the EPU Management Service, until
        # it also is fronted with an ION interface.

        dashi_name = get_pd_dashi_name()

        # grab config parameters used to connect to dashi
        try:
            uri = conf.dashi_uri
            exchange = conf.dashi_exchange
        except AttributeError, e:
            log.warn("Needed Process Dispatcher config not found: %s", e)
            raise
Example #6
0
    def test_engine_config(self):
        self.mm.cancel()
        maximum_vms = 10
        base_iaas_allocation = 'm1.small'
        engine_iaas_allocation = 'm1.medium'
        self.base_domain_config = {
            'engine_conf': {
                'iaas_allocation': base_iaas_allocation
            }
        }

        engine_conf = {
            'engine1': {
                'slots': 1,
                'iaas_allocation': engine_iaas_allocation,
                'maximum_vms': maximum_vms
            },
            'engine2': {
                'slots': 2,
                'iaas_allocation': engine_iaas_allocation,
                'maximum_vms': maximum_vms
            },
        }

        self.registry = EngineRegistry.from_config(engine_conf, default='engine1')
        self.mm = PDMatchmaker(self.core, self.store, self.resource_client,
            self.registry, self.epum_client, self.notifier, self.service_name,
            self.definition_id, self.base_domain_config, self.run_type,
            self.restart_throttling_config)

        self.mm.initialize()
        self.assertEqual(len(self.epum_client.domains), len(engine_conf.keys()))

        engine1_domain_conf = self.epum_client.domains['pd_domain_engine1']['engine_conf']
        engine2_domain_conf = self.epum_client.domains['pd_domain_engine2']['engine_conf']

        self.assertEqual(engine1_domain_conf['iaas_allocation'], engine_iaas_allocation)
        self.assertEqual(engine2_domain_conf['iaas_allocation'], engine_iaas_allocation)

        self.assertEqual(engine1_domain_conf['maximum_vms'], maximum_vms)
        self.assertEqual(engine2_domain_conf['maximum_vms'], maximum_vms)
Example #7
0
    def setUp(self):
        self.store = self.setup_store()
        self.resource_client = MockResourceClient()
        self.epum_client = MockEPUMClient()
        self.registry = EngineRegistry.from_config(self.engine_conf, default='engine1')
        self.notifier = MockNotifier()
        self.service_name = "some_pd"
        self.definition_id = "pd_definition"
        self.definition = get_definition()
        self.base_domain_config = get_domain_config()
        self.run_type = "fake_run_type"
        self.restart_throttling_config = {}

        self.core = ProcessDispatcherCore(self.store, self.registry,
            self.resource_client, self.notifier)

        self.epum_client.add_domain_definition(self.definition_id, self.definition)
        self.mm = PDMatchmaker(self.core, self.store, self.resource_client,
            self.registry, self.epum_client, self.notifier, self.service_name,
            self.definition_id, self.base_domain_config, self.run_type,
            self.restart_throttling_config)

        self.mmthread = None
Example #8
0
 def test_bad_default(self):
     with self.assertRaises(KeyError):
         EngineRegistry.from_config(ENGINE_CONF1, default="engineX")
Example #9
0
    def __init__(self, amqp_uri=None, topic="process_dispatcher", registry=None,
                 store=None, epum_client=None, notifier=None, definition_id=None,
                 domain_config=None, sysname=None):

        configs = ["service", "processdispatcher"]
        config_files = get_config_paths(configs)
        self.CFG = bootstrap.configure(config_files)
        self.topic = self.CFG.processdispatcher.get('service_name', topic)

        self.dashi = bootstrap.dashi_connect(self.topic, self.CFG,
                                             amqp_uri=amqp_uri, sysname=sysname)

        engine_conf = self.CFG.processdispatcher.get('engines', {})
        default_engine = self.CFG.processdispatcher.get('default_engine')
        process_engines = self.CFG.processdispatcher.get('process_engines')
        if default_engine is None and len(engine_conf.keys()) == 1:
            default_engine = engine_conf.keys()[0]
        self.store = store or get_processdispatcher_store(self.CFG)
        self.store.initialize()
        self.registry = registry or EngineRegistry.from_config(engine_conf,
            default=default_engine, process_engines=process_engines)
        self.eeagent_client = EEAgentClient(self.dashi)

        domain_definition_id = None
        base_domain_config = None
        # allow disabling communication with EPUM for epuharness case
        if epum_client:
            self.epum_client = epum_client
            domain_definition_id = definition_id
            base_domain_config = domain_config
        elif not self.CFG.processdispatcher.get('static_resources'):
            domain_definition_id = definition_id or self.CFG.processdispatcher.get('definition_id')
            base_domain_config = domain_config or self.CFG.processdispatcher.get('domain_config')
            epum_service_name = self.CFG.processdispatcher.get('epum_service_name',
                    'epu_management_service')
            self.epum_client = EPUManagementClient(self.dashi, epum_service_name)

        else:
            self.epum_client = None

        if notifier:
            self.notifier = notifier
        else:
            self.notifier = SubscriberNotifier(self.dashi)

        self.core = ProcessDispatcherCore(self.store,
                                          self.registry,
                                          self.eeagent_client,
                                          self.notifier)

        launch_type = self.CFG.processdispatcher.get('launch_type', 'supd')
        restart_throttling_config = self.CFG.processdispatcher.get('restart_throttling_config', {})
        dispatch_retry_seconds = self.CFG.processdispatcher.get('dispatch_retry_seconds')

        self.matchmaker = PDMatchmaker(self.core, self.store, self.eeagent_client,
            self.registry, self.epum_client, self.notifier, self.topic,
            domain_definition_id, base_domain_config, launch_type,
            restart_throttling_config, dispatch_retry_seconds)

        self.doctor = PDDoctor(self.core, self.store, config=self.CFG)
        self.ready_event = threading.Event()