def test_process_engines_no_mapping(self):
registry = EngineRegistry.from_config(ENGINE_CONF1, default="engine1")
definition = dict(executable={"module": "some.module", "class": "SomeClass"})
self.assertIsNone(registry.get_process_definition_engine_id(definition))
registry = EngineRegistry.from_config(ENGINE_CONF1, default="engine1", process_engines={})
self.assertIsNone(registry.get_process_definition_engine_id(definition))
def test_process_engines(self):
process_engines = {
"a.b.c.D": "engine1",
"a.b.c": "engine2",
"a": "engine3",
"e.f": "engine3",
"e.f.G": "engine1",
}
registry = EngineRegistry.from_config(ENGINE_CONF1, default="engine1", process_engines=process_engines)
definition = dict(executable={"module": "a.b.c", "class": "D"})
self.assertEqual(registry.get_process_definition_engine_id(definition), "engine1")
definition = dict(executable={"module": "a.b.c", "class": "E"})
self.assertEqual(registry.get_process_definition_engine_id(definition), "engine2")
definition = dict(executable={"module": "a.b.x", "class": "D"})
self.assertEqual(registry.get_process_definition_engine_id(definition), "engine3")
definition = dict(executable={"module": "e", "class": "B"})
self.assertEqual(registry.get_process_definition_engine_id(definition), None)
definition = dict(executable={"module": "e.f", "class": "B"})
self.assertEqual(registry.get_process_definition_engine_id(definition), "engine3")
definition = dict(executable={"module": "e.f", "class": "G"})
self.assertEqual(registry.get_process_definition_engine_id(definition), "engine1")
def setUp(self):
self.store = self.get_store()
self.registry = EngineRegistry.from_config(self.engine_conf)
self.resource_client = Mock()
self.notifier = MockNotifier()
self.core = ProcessDispatcherCore(self.store, self.registry,
self.resource_client, self.notifier)
def setUp(self):
self.store = self.setup_store()
self.registry = EngineRegistry.from_config(self.engine_conf)
self.resource_client = MockResourceClient()
self.notifier = MockNotifier()
self.core = ProcessDispatcherCore(self.store, self.registry,
self.resource_client, self.notifier)
self.doctor = PDDoctor(self.core, self.store)
self.docthread = None
self.monitor = None
def __init__(self, conf, service):
engine_conf = conf.get('engines', {})
self.store = ProcessDispatcherStore()
self.registry = EngineRegistry.from_config(engine_conf)
# The Process Dispatcher communicates with EE Agents over ION messaging
# but it still uses dashi to talk to the EPU Management Service, until
# it also is fronted with an ION interface.
dashi_name = get_pd_dashi_name()
# grab config parameters used to connect to dashi
try:
uri = conf.dashi_uri
exchange = conf.dashi_exchange
except AttributeError, e:
log.warn("Needed Process Dispatcher config not found: %s", e)
raise
def test_engine_config(self):
self.mm.cancel()
maximum_vms = 10
base_iaas_allocation = 'm1.small'
engine_iaas_allocation = 'm1.medium'
self.base_domain_config = {
'engine_conf': {
'iaas_allocation': base_iaas_allocation
}
}
engine_conf = {
'engine1': {
'slots': 1,
'iaas_allocation': engine_iaas_allocation,
'maximum_vms': maximum_vms
},
'engine2': {
'slots': 2,
'iaas_allocation': engine_iaas_allocation,
'maximum_vms': maximum_vms
},
}
self.registry = EngineRegistry.from_config(engine_conf, default='engine1')
self.mm = PDMatchmaker(self.core, self.store, self.resource_client,
self.registry, self.epum_client, self.notifier, self.service_name,
self.definition_id, self.base_domain_config, self.run_type,
self.restart_throttling_config)
self.mm.initialize()
self.assertEqual(len(self.epum_client.domains), len(engine_conf.keys()))
engine1_domain_conf = self.epum_client.domains['pd_domain_engine1']['engine_conf']
engine2_domain_conf = self.epum_client.domains['pd_domain_engine2']['engine_conf']
self.assertEqual(engine1_domain_conf['iaas_allocation'], engine_iaas_allocation)
self.assertEqual(engine2_domain_conf['iaas_allocation'], engine_iaas_allocation)
self.assertEqual(engine1_domain_conf['maximum_vms'], maximum_vms)
self.assertEqual(engine2_domain_conf['maximum_vms'], maximum_vms)
def setUp(self):
self.store = self.setup_store()
self.resource_client = MockResourceClient()
self.epum_client = MockEPUMClient()
self.registry = EngineRegistry.from_config(self.engine_conf, default='engine1')
self.notifier = MockNotifier()
self.service_name = "some_pd"
self.definition_id = "pd_definition"
self.definition = get_definition()
self.base_domain_config = get_domain_config()
self.run_type = "fake_run_type"
self.restart_throttling_config = {}
self.core = ProcessDispatcherCore(self.store, self.registry,
self.resource_client, self.notifier)
self.epum_client.add_domain_definition(self.definition_id, self.definition)
self.mm = PDMatchmaker(self.core, self.store, self.resource_client,
self.registry, self.epum_client, self.notifier, self.service_name,
self.definition_id, self.base_domain_config, self.run_type,
self.restart_throttling_config)
self.mmthread = None
def test_bad_default(self):
with self.assertRaises(KeyError):
EngineRegistry.from_config(ENGINE_CONF1, default="engineX")
def __init__(self, amqp_uri=None, topic="process_dispatcher", registry=None,
store=None, epum_client=None, notifier=None, definition_id=None,
domain_config=None, sysname=None):
configs = ["service", "processdispatcher"]
config_files = get_config_paths(configs)
self.CFG = bootstrap.configure(config_files)
self.topic = self.CFG.processdispatcher.get('service_name', topic)
self.dashi = bootstrap.dashi_connect(self.topic, self.CFG,
amqp_uri=amqp_uri, sysname=sysname)
engine_conf = self.CFG.processdispatcher.get('engines', {})
default_engine = self.CFG.processdispatcher.get('default_engine')
process_engines = self.CFG.processdispatcher.get('process_engines')
if default_engine is None and len(engine_conf.keys()) == 1:
default_engine = engine_conf.keys()[0]
self.store = store or get_processdispatcher_store(self.CFG)
self.store.initialize()
self.registry = registry or EngineRegistry.from_config(engine_conf,
default=default_engine, process_engines=process_engines)
self.eeagent_client = EEAgentClient(self.dashi)
domain_definition_id = None
base_domain_config = None
# allow disabling communication with EPUM for epuharness case
if epum_client:
self.epum_client = epum_client
domain_definition_id = definition_id
base_domain_config = domain_config
elif not self.CFG.processdispatcher.get('static_resources'):
domain_definition_id = definition_id or self.CFG.processdispatcher.get('definition_id')
base_domain_config = domain_config or self.CFG.processdispatcher.get('domain_config')
epum_service_name = self.CFG.processdispatcher.get('epum_service_name',
'epu_management_service')
self.epum_client = EPUManagementClient(self.dashi, epum_service_name)
else:
self.epum_client = None
if notifier:
self.notifier = notifier
else:
self.notifier = SubscriberNotifier(self.dashi)
self.core = ProcessDispatcherCore(self.store,
self.registry,
self.eeagent_client,
self.notifier)
launch_type = self.CFG.processdispatcher.get('launch_type', 'supd')
restart_throttling_config = self.CFG.processdispatcher.get('restart_throttling_config', {})
dispatch_retry_seconds = self.CFG.processdispatcher.get('dispatch_retry_seconds')
self.matchmaker = PDMatchmaker(self.core, self.store, self.eeagent_client,
self.registry, self.epum_client, self.notifier, self.topic,
domain_definition_id, base_domain_config, launch_type,
restart_throttling_config, dispatch_retry_seconds)
self.doctor = PDDoctor(self.core, self.store, config=self.CFG)
self.ready_event = threading.Event()