def test_get_process_constraints(self):
"""test_get_process_constraints
ensure that order of precedence of engine ids is correct. Should be:
1. process target - when a process is scheduled, an execution_engine_id
can be specified in the request's ProcessTarget object. If specified,
this EE is used.
2. process/engine mappings - the CEI Launch YML file contains a
process_engines mapping of process packages to EE names. If the process'
module matches an entry in this configuration, the associated EE is
chosen. This format is described below.
3. default execution engine - the CEI Launch YML file also must specify
a default_execution_engine value. This is used as a last resort.
"""
self.registry.set_process_engine_mapping("my", "engine4")
self.registry.default = "engine1"
process_definition = {"executable": {"module": "my.test", "class": "MyClass"}}
process_constraints = {"engine": "mostimportantengine"}
p1 = ProcessRecord.new(None, "proc1", {}, ProcessState.PENDING)
constraints = self.core.get_process_constraints(p1)
self.assertEqual(constraints["engine"], self.registry.default)
p3 = ProcessRecord.new(None, "proc3", process_definition, ProcessState.PENDING, constraints=process_constraints)
constraints = self.core.get_process_constraints(p3)
self.assertEqual(constraints["engine"], "mostimportantengine")
def test_initialized_system_boot_with_procs(self):
# tests the case where just the doctor dies in the middle of system boot
# but after a doctor has already declared the system initialized. In this
# case we have processes in the UNSCHEDULED_PENDING state that should be
# rescheduled once system boot ends.
self.store.set_system_boot(True)
self.store.set_initialized()
self.store.set_pd_state(ProcessDispatcherState.SYSTEM_BOOTING)
p0 = ProcessRecord.new(None, "proc0", {}, ProcessState.UNSCHEDULED_PENDING)
self.store.add_process(p0)
p1 = ProcessRecord.new(None, "proc1", {}, ProcessState.UNSCHEDULED_PENDING)
self.store.add_process(p1)
p2 = ProcessRecord.new(None, "proc2", {}, ProcessState.UNSCHEDULED_PENDING)
self.store.add_process(p2)
restartable_procs = ["proc0", "proc1", "proc2"]
self._run_in_thread()
# now end system boot
self.store.set_system_boot(False)
wait(lambda: self.store.get_pd_state() == ProcessDispatcherState.OK)
# check that pending processes were correctly rescheduled
self.assertEqual(len(self.store.get_queued_processes()), len(restartable_procs))
for proc in restartable_procs:
self.assertEqual(self.store.get_process(None, proc).state,
ProcessState.REQUESTED)
def test_uninitialized_not_system_boot_with_procs(self):
# tests the case where doctor arrives to an uninitialized system
# that is not doing a system boot. HOWEVER, there are procs in the
# UNSCHEDULED_PENDING state. This would likely only happen if the
# PD died during system boot and recovered after the system boot flag
# was turned off. Very small window, but possible.
p0 = ProcessRecord.new(None, "proc0", {}, ProcessState.UNSCHEDULED_PENDING)
self.store.add_process(p0)
p1 = ProcessRecord.new(None, "proc1", {}, ProcessState.UNSCHEDULED_PENDING)
self.store.add_process(p1)
p2 = ProcessRecord.new(None, "proc2", {}, ProcessState.UNSCHEDULED_PENDING)
self.store.add_process(p2)
restartable_procs = ["proc0", "proc1", "proc2"]
self._run_in_thread()
assert self.store.wait_initialized(timeout=10)
self.assertEqual(self.store.get_pd_state(),
ProcessDispatcherState.OK)
# check that pending processes were correctly rescheduled
self.assertEqual(len(self.store.get_queued_processes()), len(restartable_procs))
for proc in restartable_procs:
self.assertEqual(self.store.get_process(None, proc).state,
ProcessState.REQUESTED)
def test_match_double_queued_process(self):
props = {"engine": "engine1"}
r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
self.store.add_resource(r1)
r2 = ResourceRecord.new("r2", "n1", 1, properties=props)
self.store.add_resource(r2)
p1 = ProcessRecord.new(None, "p1", get_process_definition(),
ProcessState.REQUESTED)
p1key = p1.get_key()
self.store.add_process(p1)
p2 = ProcessRecord.new(None, "p2", get_process_definition(),
ProcessState.REQUESTED)
p2key = p2.get_key()
self.store.add_process(p2)
# enqueue p1 repeatedly. make sure that doesn't bomb anything
self.store.enqueue_process(*p1key)
self.store.enqueue_process(*p1key)
self.store.enqueue_process(*p2key)
self.store.enqueue_process(*p1key)
self._run_in_thread()
self.wait_process(p1.owner, p1.upid,
lambda p: p.state == ProcessState.PENDING)
self.wait_process(p2.owner, p2.upid,
lambda p: p.state == ProcessState.PENDING)
def test_stale_procs(self):
"""test that the matchmaker doesn't try to schedule stale procs
A stale proc is one that the matchmaker has attempted to scale before
while the state have the resources hasn't changed.
"""
if not os.environ.get('INT'):
raise SkipTest("Skip slow integration test")
self.mm.initialize()
p1 = ProcessRecord.new(None, "p1", get_process_definition(),
ProcessState.REQUESTED)
p1key = p1.get_key()
self.store.add_process(p1)
self.store.enqueue_process(*p1key)
# sneak into MM and force it to update this info from the store
self.mm._get_queued_processes()
self.mm._get_resource_set()
self.assertTrue(self.mm.needs_matchmaking)
self.mm.matchmake()
self.assertFalse(self.mm.needs_matchmaking)
self.assertTrue(len(self.mm.stale_processes) > 0)
self.mm._get_queued_processes()
self.mm._get_resource_set()
self.assertFalse(self.mm.needs_matchmaking)
self.assertTrue(len(self.mm.stale_processes) > 0)
p2 = ProcessRecord.new(None, "p2", get_process_definition(),
ProcessState.REQUESTED)
p2key = p2.get_key()
self.store.add_process(p2)
self.store.enqueue_process(*p2key)
self.mm._get_queued_processes()
self.mm._get_resource_set()
self.assertTrue(self.mm.needs_matchmaking)
self.assertTrue(len(self.mm.stale_processes) > 0)
self.assertTrue(len(self.mm.queued_processes) > len(self.mm.stale_processes))
self.mm.matchmake()
self.assertFalse(self.mm.needs_matchmaking)
self.assertTrue(len(self.mm.queued_processes) == len(self.mm.stale_processes))
# Add a resource, and ensure that stale procs get dumped
props = {"engine": "engine1"}
r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
self.store.add_resource(r1)
self.mm._get_queued_processes()
self.mm._get_resources()
self.mm._get_resource_set()
self.assertTrue(len(self.mm.stale_processes) == 0)
def test_rejected_procs(self):
parameters = {"preserve_n": 2, "execution_engine_id": "some_ee", "node_exclusive": "hats"}
self.policy.parameters = parameters
upids = []
all_procs = {"pd0": []}
self.policy.apply_policy(all_procs, upids)
schedule_call = call(ANY, ANY, configuration=None, execution_engine_id="some_ee", node_exclusive="hats")
self.assertEqual(self.mock_schedule.call_args_list, [schedule_call, schedule_call])
self.assertEqual(self.mock_terminate.call_count, 0)
self.mock_schedule.reset_mock()
self.mock_terminate.reset_mock()
owner = "fred"
upids = ["myupid0", "myupid1"]
definition = None
state_running = ProcessState.RUNNING
state_rejected = ProcessState.REJECTED
all_procs = {
"pd0": [
ProcessRecord.new(owner, upids[0], definition, state_running),
ProcessRecord.new(owner, upids[1], definition, state_rejected),
]
}
self.policy.apply_policy(all_procs, upids)
self.assertEqual(self.mock_schedule.call_count, 1)
self.assertEqual(self.mock_terminate.call_count, 0)
self.mock_schedule.reset_mock()
self.mock_terminate.reset_mock()
upids = ["myupid0", "myupid1", "myupid2"]
all_procs = {
"pd0": [
ProcessRecord.new(owner, upids[0], definition, state_running),
ProcessRecord.new(owner, upids[1], definition, state_rejected),
ProcessRecord.new(owner, upids[2], definition, state_running),
]
}
self.policy.apply_policy(all_procs, upids)
self.assertEqual(self.mock_schedule.call_count, 0)
self.assertEqual(self.mock_terminate.call_count, 0)
self.mock_schedule.reset_mock()
self.mock_terminate.reset_mock()
parameters["preserve_n"] = 1
self.policy.parameters = parameters
self.policy.apply_policy(all_procs, upids)
self.assertEqual(self.mock_terminate.call_count, 1)
self.assertEqual(self.mock_schedule.call_count, 0)
def test_node_exclusive_bug(self):
"""test_node_exclusive_bug
If two processes with the same node exclusive attribute where scheduled
in the same matchmaking cycle, they could be scheduled to the same
resource, due to a caching issue. This test tests the fix.
"""
self.mm.initialize()
n1 = NodeRecord.new("n1", "d1")
self.store.add_node(n1)
props = {"engine": "engine1"}
r1 = ResourceRecord.new("r1", "n1", 2, properties=props)
self.store.add_resource(r1)
n2 = NodeRecord.new("n2", "d1")
self.store.add_node(n2)
props = {"engine": "engine1"}
r2 = ResourceRecord.new("r2", "n2", 2, properties=props)
self.store.add_resource(r2)
xattr_1 = "port5000"
constraints = {}
p1 = ProcessRecord.new(None, "p1", get_process_definition(),
ProcessState.REQUESTED, constraints=constraints,
node_exclusive=xattr_1)
p1key = p1.get_key()
self.store.add_process(p1)
self.store.enqueue_process(*p1key)
p2 = ProcessRecord.new(None, "p2", get_process_definition(),
ProcessState.REQUESTED, constraints=constraints,
node_exclusive=xattr_1)
p2key = p2.get_key()
self.store.add_process(p2)
self.store.enqueue_process(*p2key)
# sneak into MM and force it to update this info from the store
self.mm._get_queued_processes()
self.mm._get_resource_set()
self.mm.matchmake()
# Ensure these processes are pending and scheduled to different nodes
p1 = self.store.get_process(None, "p1")
p2 = self.store.get_process(None, "p2")
self.assertNotEqual(p1.assigned, p2.assigned)
def test_match_writeconflict(self):
self.mm.initialize()
props = {"engine": "engine1"}
r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
self.store.add_resource(r1)
p1 = ProcessRecord.new(None, "p1", get_process_definition(),
ProcessState.REQUESTED)
p1key = p1.get_key()
self.store.add_process(p1)
self.store.enqueue_process(*p1key)
# sneak into MM and force it to update this info from the store
self.mm._get_queued_processes()
self.mm._get_resource_set()
# now update the resource record so the matchmake() attempt to write will conflict
r1.assigned = ["hats"]
self.store.update_resource(r1)
# this should bail out without resetting the needs_matchmaking flag
# or registering any need
self.assertTrue(self.mm.needs_matchmaking)
self.mm.matchmake()
self.assertFalse(self.epum_client.reconfigures)
self.assertTrue(self.mm.needs_matchmaking)
r1copy = self.store.get_resource(r1.resource_id)
self.assertRecordVersions(r1, r1copy)
def test_process_already_assigned(self):
# this is a recovery situation, probably. The process is assigned
# to a resource already at the start of the matchmaker run, but
# the process record hasn't been updated to reflect that. The
# situation should be detected and handled by matchmaker.
p1 = ProcessRecord.new(None, "p1", get_process_definition(),
ProcessState.REQUESTED)
self.store.add_process(p1)
self.store.enqueue_process(*p1.key)
props = {"engine": "engine1"}
r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
r1.assigned.append(p1.key)
self.store.add_resource(r1)
self._run_in_thread()
self.wait_process(None, "p1", lambda p: p.state == ProcessState.PENDING)
r1 = self.store.get_resource("r1")
self.assertEqual(len(r1.assigned), 1)
self.assertTrue(r1.is_assigned(p1.owner, p1.upid, p1.round))
self.assertEqual(r1.available_slots, 0)
def test_match_process_terminated(self):
self.mm.initialize()
props = {"engine": "engine1"}
r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
self.store.add_resource(r1)
p1 = ProcessRecord.new(None, "p1", get_process_definition(),
ProcessState.REQUESTED)
p1key = p1.get_key()
self.store.add_process(p1)
self.store.enqueue_process(*p1key)
# sneak into MM and force it to update this info from the store
self.mm._get_queued_processes()
self.mm._get_resource_set()
# now update the process record to be TERMINATED so that
# MM should bail out of matching this process
p1.state = ProcessState.TERMINATED
self.store.update_process(p1)
self.store.remove_queued_process(*p1key)
self.mm.matchmake()
p1 = self.store.get_process(None, "p1")
self.assertEqual(p1.state, ProcessState.TERMINATED)
def test_engine_types(self):
self._run_in_thread()
props = {"engine": "engine1"}
r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
self.store.add_resource(r1)
constraints = {"engine": "engine2"}
p1 = ProcessRecord.new(None, "p1", get_process_definition(),
ProcessState.REQUESTED, constraints=constraints)
p1key = p1.get_key()
self.store.add_process(p1)
self.store.enqueue_process(*p1key)
# We don't have a resource that can run this yet
timed_out = False
try:
self.wait_resource(r1.resource_id, lambda r: list(p1key) in r.assigned,
timeout=2)
except Exception:
timed_out = True
assert timed_out
props = {"engine": "engine2"}
r2 = ResourceRecord.new("r2", "n2", 1, properties=props)
self.store.add_resource(r2)
self.wait_resource(r2.resource_id, lambda r: list(p1key) in r.assigned)
time.sleep(0.05)
self.resource_client.check_process_launched(p1, r2.resource_id)
self.wait_process(p1.owner, p1.upid,
lambda p: p.assigned == r2.resource_id and
p.state == ProcessState.PENDING)
def test_terminate_assigned(self):
p1 = ProcessRecord.new(None, "proc1", {}, ProcessState.ASSIGNED)
p1.assigned = "hats"
self.store.add_process(p1)
self.core.terminate_process(None, "proc1")
self.resource_client.terminate_process.assert_called_once_with("hats", "proc1", 0)
self.notifier.assert_process_state("proc1", ProcessState.TERMINATING)
def test_terminate_retry(self):
# try to kill a process that is already terminating
p1 = ProcessRecord.new(None, "proc1", {}, ProcessState.TERMINATING)
p1.assigned = "hats"
self.store.add_process(p1)
self.core.terminate_process(None, "proc1")
self.resource_client.terminate_process.assert_called_once_with("hats", "proc1", 0)
self.notifier.assert_no_process_state()
def test_get_hostnames(self):
owner = "fred"
upids = ["myupid0", "myupid1", "myupid2"]
hostnames = ["my.hostname0.tld", "my.hostname1.tld", "my.hostname2.tld"]
definition = None
state = None
all_procs = {
"pd0": [ProcessRecord.new(owner, upids[0], definition, state, hostname=hostnames[0])],
"pd1": [
ProcessRecord.new(owner, upids[1], definition, state, hostname=hostnames[1]),
ProcessRecord.new(owner, upids[2], definition, state, hostname=hostnames[2]),
],
}
got = self.policy._get_hostnames(all_procs, upids)
self.assertEqual(sorted(got), sorted(hostnames))
def test_add_remove_node_with_resource_and_processes(self):
self.core.node_state("node1", domain_id_from_engine("engine1"),
InstanceState.RUNNING)
resource_id = "eeagent_1"
self.core.ee_heartbeat(resource_id, make_beat("node1"))
# set up a few of processes on the resource
p1 = ProcessRecord.new(None, "proc1", {}, ProcessState.RUNNING,
assigned=resource_id)
self.store.add_process(p1)
p2 = ProcessRecord.new(None, "proc2", {}, ProcessState.PENDING,
assigned=resource_id)
self.store.add_process(p2)
p3 = ProcessRecord.new(None, "proc3", {}, ProcessState.TERMINATING,
assigned=resource_id)
self.store.add_process(p3)
resource = self.store.get_resource(resource_id)
resource.assigned = [p1.key, p2.key, p3.key]
self.store.update_resource(resource)
# now send a terminated state for the node. resource should be removed.
self.core.node_state("node1", domain_id_from_engine("engine1"),
InstanceState.TERMINATED)
self.assertTrue(self.store.get_resource(resource_id) is None)
self.assertTrue(self.store.get_node("node1") is None)
queued_processes = set(self.store.get_queued_processes())
# these two should have been rescheduled
for procname in ("proc1", "proc2"):
proc = self.store.get_process(None, procname)
self.assertEqual(proc.state, ProcessState.DIED_REQUESTED)
self.assertEqual(proc.round, 1)
self.assertIn(proc.key, queued_processes)
self.notifier.assert_process_state(procname, ProcessState.DIED_REQUESTED)
# this one should be terminated
proc3 = self.store.get_process(None, "proc3")
self.assertEqual(proc3.state, ProcessState.TERMINATED)
self.assertEqual(proc3.round, 0)
self.assertNotIn(proc3.key, queued_processes)
self.notifier.assert_process_state("proc3", ProcessState.TERMINATED)
def create_n_pending_processes(self, n_processes, engine_id):
pkeys = []
for pid in range(n_processes):
upid = uuid.uuid4().hex
p = ProcessRecord.new(None, upid, get_process_definition(),
ProcessState.UNSCHEDULED_PENDING, constraints={'engine': engine_id})
self.store.add_process(p)
pkey = p.get_key()
pkeys.append(pkey)
return pkeys
def test_get_hostnames(self):
owner = 'fred'
upids = ['myupid0', 'myupid1', 'myupid2']
hostnames = ['my.hostname0.tld', 'my.hostname1.tld', 'my.hostname2.tld']
definition = None
state = None
all_procs = {
'pd0': [
ProcessRecord.new(owner, upids[0], definition, state, hostname=hostnames[0]),
],
'pd1': [
ProcessRecord.new(owner, upids[1], definition, state, hostname=hostnames[1]),
ProcessRecord.new(owner, upids[2], definition, state, hostname=hostnames[2]),
]
}
got = self.policy._get_hostnames(all_procs, upids)
self.assertEqual(sorted(got), sorted(hostnames))
def test_apply_policy(self):
parameters = {
'preserve_n': 2,
'execution_engine_id': 'some_ee',
'node_exclusive': 'hats'
}
self.policy.parameters = parameters
upids = []
all_procs = {'pd0': []}
self.policy.apply_policy(all_procs, upids)
schedule_call = call(ANY, ANY, configuration=None,
execution_engine_id='some_ee', node_exclusive='hats')
self.assertEqual(self.mock_schedule.call_args_list, [schedule_call, schedule_call])
self.assertEqual(self.mock_terminate.call_count, 0)
self.mock_schedule.reset_mock()
self.mock_terminate.reset_mock()
owner = 'fred'
upids = ['myupid0', 'myupid1']
definition = None
state = None
all_procs = {
'pd0': [
ProcessRecord.new(owner, upids[0], definition, state),
ProcessRecord.new(owner, upids[1], definition, state),
],
}
parameters['preserve_n'] = 1
self.policy.parameters = parameters
self.policy.apply_policy(all_procs, upids)
self.assertEqual(self.mock_terminate.call_count, 1)
self.assertEqual(self.mock_schedule.call_count, 0)
def enqueue_n_processes(self, n_processes, engine_id):
pkeys = []
for pid in range(n_processes):
upid = uuid.uuid4().hex
p = ProcessRecord.new(None, upid, get_process_definition(),
ProcessState.REQUESTED, constraints={'engine': engine_id})
pkey = p.get_key()
pkeys.append(pkey)
self.store.add_process(p)
self.store.enqueue_process(*pkey)
self.mm.queued_processes.append(pkey)
return pkeys
def test_queueing_order(self):
self._run_in_thread()
procnames = []
# queue 10 processes
for i in range(10):
proc = ProcessRecord.new(None, "proc" + str(i), get_process_definition(),
ProcessState.REQUESTED)
prockey = proc.key
self.store.add_process(proc)
self.store.enqueue_process(*prockey)
self.epum_client.clear()
self.wait_process(proc.owner, proc.upid,
lambda p: p.state == ProcessState.WAITING)
procnames.append(proc.upid)
# potentially retry a few times to account for race between process
# state updates and need reconfigures
for i in range(5):
try:
self.assert_one_reconfigure(preserve_n=i + 1, retirees=[])
break
except AssertionError:
time.sleep(0.01)
self.epum_client.clear()
# now add 10 resources each with 1 slot. processes should start in order
for i in range(10):
props = {"engine": "engine1"}
res = ResourceRecord.new("res" + str(i), "node" + str(i), 1,
properties=props)
self.store.add_resource(res)
self.wait_process(None, procnames[i],
lambda p: p.state >= ProcessState.PENDING and
p.assigned == res.resource_id)
# finally doublecheck that launch requests happened in order too
for i in range(5):
try:
self.assertEqual(self.resource_client.launch_count, 10)
for i, launch in enumerate(self.resource_client.launches):
self.assertEqual(launch[0], "res" + str(i))
self.assertEqual(launch[1], "proc" + str(i))
break
except AssertionError:
time.sleep(0.01)
def test_process_terminated(self):
self._run_in_thread()
event = threading.Event()
# we set up a resource and a matching process that should be assigned
# to it. we will simulate marking the process TERMINATED out-of-band
# and ensure that is recognized before the dispatch.
# when the matchmaker attempts to update the process, sneak in an update
# first so the matchmaker request conflicts
original_update_process = self.store.update_process
def patched_update_process(process):
original = self.store.get_process(process.owner, process.upid)
original.state = ProcessState.TERMINATED
original_update_process(original)
try:
original_update_process(process)
finally:
event.set()
self.store.update_process = patched_update_process
p1 = ProcessRecord.new(None, "p1", get_process_definition(),
ProcessState.REQUESTED)
self.store.add_process(p1)
self.store.enqueue_process(*p1.key)
# now give it a resource. it should be matched but in the meantime
# the process will be terminated
props = {"engine": "engine1"}
r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
self.store.add_resource(r1)
# wait for MM to hit our update conflict, kill it, and check that it
# appropriately backed out the allocation
assert event.wait(5)
self.mm.cancel()
self.mmthread.join()
self.mmthread = None
resource = self.store.get_resource("r1")
self.assertEqual(len(resource.assigned), 0)
self.assertEqual(self.resource_client.launch_count, 0)
def test_terminate_unassigned_process(self):
p1 = ProcessRecord.new(None, "proc1", {}, ProcessState.WAITING)
self.store.add_process(p1)
self.store.enqueue_process(*p1.key)
gotproc = self.core.terminate_process(None, "proc1")
self.assertEqual(gotproc.upid, "proc1")
self.assertEqual(gotproc.state, ProcessState.TERMINATED)
p1 = self.store.get_process(None, "proc1")
self.assertEqual(p1.state, ProcessState.TERMINATED)
self.notifier.assert_process_state("proc1", ProcessState.TERMINATED)
# should be gone from queue too
self.assertFalse(self.store.get_queued_processes())
self.assertEqual(self.resource_client.call_count, 0)
def create_process(self, owner, upid, definition_id, name=None):
"""Create a new process in the system
@param upid: unique process identifier
@param definition_id: process definition to start
@param name: a (hopefully) human recognizable name for the process
Retry
=====
If a call to this operation times out without a reply, it can safely
be retried. The upid and other parameters will be used to ensure that
nothing is repeated. If the service fields an operation request that
it thinks has already been acknowledged, it will return the current
state of the process.
"""
validate_owner_upid(owner, upid)
validate_definition_id(definition_id)
# if not a real def, a NotFoundError will bubble up to caller
definition = self.store.get_definition(definition_id)
if definition is None:
raise NotFoundError("Couldn't find process definition %s in store" % definition_id)
process = ProcessRecord.new(owner, upid, definition,
ProcessState.UNSCHEDULED, name=name)
try:
self.store.add_process(process)
except WriteConflictError:
process = self.store.get_process(owner, upid)
if not process:
raise BadRequestError("process exists but cannot be found")
# check parameters from call against process record in store.
# if they don't match, the caller probably has a coding error
# and is reusing upids or something.
if name is not None and process.name != name:
raise BadRequestError("process %s exists with different name '%s'" %
(upid, process.name))
if definition_id != process.definition['definition_id']:
raise BadRequestError("process %s exists with different definition_id '%s'" %
(upid, process.definition['definition_id']))
return process
def test_node_filo(self):
"""test_node_filo
We prioritize shutting down the newest VMs as a workaround for OOI
Testing strategy
"""
self.mm.initialize()
n1 = NodeRecord.new("n1", "d1")
self.store.add_node(n1)
props = {"engine": "engine4"}
r1 = ResourceRecord.new("r1", "n1", 2, properties=props)
self.store.add_resource(r1)
n2 = NodeRecord.new("n2", "d1")
self.store.add_node(n2)
props = {"engine": "engine4"}
r2 = ResourceRecord.new("r2", "n2", 2, properties=props)
self.store.add_resource(r2)
constraints = {"engine": "engine4"}
p1 = ProcessRecord.new(None, "p1", get_process_definition(),
ProcessState.REQUESTED, constraints=constraints)
p1key = p1.get_key()
self.store.add_process(p1)
self.store.enqueue_process(*p1key)
# sneak into MM and force it to update this info from the store
self.mm._get_queued_processes()
self.mm._get_resource_set()
self.mm.register_needs()
self.epum_client.clear()
self.mm.queued_processes = []
self.mm.register_needs()
conf = self.epum_client.reconfigures['pd_domain_engine4'][0]
retired_nodes = conf['engine_conf']['retirable_nodes']
assert len(retired_nodes) == 1
# This should be the second node we started
assert retired_nodes[0] == "n2"
def test_match_copy_hostname(self):
self._run_in_thread()
props = {"engine": "engine1", "hostname": "vm123"}
r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
self.store.add_resource(r1)
p1 = ProcessRecord.new(None, "p1", get_process_definition(),
ProcessState.REQUESTED)
p1key = p1.get_key()
self.store.add_process(p1)
self.store.enqueue_process(*p1key)
self.wait_process(p1.owner, p1.upid,
lambda p: p.assigned == r1.resource_id and
p.state == ProcessState.PENDING)
p1 = self.store.get_process(None, "p1")
self.assertEqual(p1.hostname, "vm123")
def test_terminate_raciness(self):
# ensure process is TERMINATING before resource client is called
p1 = ProcessRecord.new(None, "proc1", {}, ProcessState.RUNNING)
p1.assigned = "hats"
self.store.add_process(p1)
def assert_process_terminating(resource_id, upid, round):
self.assertEqual(resource_id, "hats")
self.assertEqual(upid, "proc1")
process = self.store.get_process(None, upid)
self.assertEqual(process.state, ProcessState.TERMINATING)
self.resource_client.terminate_process.side_effect = assert_process_terminating
self.core.terminate_process(None, "proc1")
self.resource_client.terminate_process.assert_called_once_with("hats", "proc1", 0)
self.notifier.assert_process_state("proc1", ProcessState.TERMINATING)
def test_match1(self):
self._run_in_thread()
props = {"engine": "engine1"}
r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
self.store.add_resource(r1)
p1 = ProcessRecord.new(None, "p1", get_process_definition(),
ProcessState.REQUESTED)
p1key = p1.get_key()
self.store.add_process(p1)
self.store.enqueue_process(*p1key)
self.wait_resource(r1.resource_id, lambda r: list(p1key) in r.assigned)
time.sleep(0.05)
self.resource_client.check_process_launched(p1, r1.resource_id)
self.wait_process(p1.owner, p1.upid,
lambda p: p.assigned == r1.resource_id and
p.state == ProcessState.PENDING)
def test_disabled_resource(self):
self._run_in_thread()
props = {"engine": "engine1"}
r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
r1.state = ExecutionResourceState.DISABLED
self.store.add_resource(r1)
self.wait_resource("r1", lambda r: r.resource_id == "r1")
p1 = ProcessRecord.new(None, "p1", get_process_definition(),
ProcessState.REQUESTED)
p1key = p1.key
self.store.add_process(p1)
self.store.enqueue_process(*p1key)
# the resource matches but it is disabled, process should
# remain in the queue
self.wait_process(p1.owner, p1.upid,
lambda p: p.state == ProcessState.WAITING)
def test_waiting(self):
self._run_in_thread()
# not-immediate process enqueued while there are no resources
p1 = ProcessRecord.new(None, "p1", get_process_definition(),
ProcessState.REQUESTED)
p1key = p1.get_key()
self.store.add_process(p1)
self.store.enqueue_process(*p1key)
self.wait_process(None, "p1", lambda p: p.state == ProcessState.WAITING)
# now give it a resource. it should be scheduled
props = {"engine": "engine1"}
r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
self.store.add_resource(r1)
self.wait_resource(r1.resource_id, lambda r: list(p1key) in r.assigned)
time.sleep(0.05)
self.resource_client.check_process_launched(p1, r1.resource_id)
def test_terminate_terminal_process(self):
# processes which are already in a terminal state shouldn't change
p1 = ProcessRecord.new(None, "proc1", {}, ProcessState.UNSCHEDULED)
p2 = ProcessRecord.new(None, "proc2", {}, ProcessState.UNSCHEDULED_PENDING)
p3 = ProcessRecord.new(None, "proc3", {}, ProcessState.TERMINATED)
p4 = ProcessRecord.new(None, "proc4", {}, ProcessState.EXITED)
p5 = ProcessRecord.new(None, "proc5", {}, ProcessState.FAILED)
p6 = ProcessRecord.new(None, "proc6", {}, ProcessState.REJECTED)
for p in (p1, p2, p3, p4, p5, p6):
self.store.add_process(p)
for p in (p1, p2, p3, p4, p5, p6):
gotproc = self.core.terminate_process(None, p.upid)
self.assertEqual(gotproc.upid, p.upid)
self.assertEqual(gotproc.state, p.state)
p1 = self.store.get_process(None, p.upid)
self.assertEqual(p1.state, p.state)
self.assertEqual(self.resource_client.call_count, 0)
