def test_labels(self):
"""Test scheduling with labels."""
cell = scheduler.Cell('top')
left = scheduler.Bucket('left', traits=0)
right = scheduler.Bucket('right', traits=0)
srv_a = scheduler.Server('a_xx', [10, 10], valid_until=500, label='xx')
srv_b = scheduler.Server('b', [10, 10], valid_until=500)
srv_y = scheduler.Server('y_xx', [10, 10], valid_until=500, label='xx')
srv_z = scheduler.Server('z', [10, 10], valid_until=500)
cell.add_node(left)
cell.add_node(right)
left.add_node(srv_a)
left.add_node(srv_b)
right.add_node(srv_y)
right.add_node(srv_z)
app1 = scheduler.Application('app1', 4, [1, 1], 'app')
app2 = scheduler.Application('app2', 3, [2, 2], 'app')
app3 = scheduler.Application('app_xx_3', 2, [3, 3], 'app')
app4 = scheduler.Application('app_xx_4', 1, [4, 4], 'app')
cell.partitions[None].allocation.add(app1)
cell.partitions[None].allocation.add(app2)
cell.partitions['xx'].allocation.add(app3)
cell.partitions['xx'].allocation.add(app4)
cell.schedule()
self.assertIn(app1.server, ['b', 'z'])
self.assertIn(app2.server, ['b', 'z'])
self.assertIn(app3.server, ['a_xx', 'y_xx'])
self.assertIn(app4.server, ['a_xx', 'y_xx'])
def test_size_and_members(self):
"""Tests recursive size calculation."""
top = scheduler.Bucket('top', traits=_traits2int(['top']))
left = scheduler.Bucket('left', traits=_traits2int(['left']))
right = scheduler.Bucket('right', traits=_traits2int(['right']))
srv_a = scheduler.Server('a', [1, 1], traits=_traits2int(['a', '0']),
valid_until=500)
srv_b = scheduler.Server('b', [1, 1], traits=_traits2int(['b', '0']),
valid_until=500)
srv_y = scheduler.Server('y', [1, 1], traits=_traits2int(['y', '1']),
valid_until=500)
srv_z = scheduler.Server('z', [1, 1], traits=_traits2int(['z', '1']),
valid_until=500)
top.add_node(left)
top.add_node(right)
left.add_node(srv_a)
left.add_node(srv_b)
right.add_node(srv_y)
right.add_node(srv_z)
# pylint: disable=W0212
self.assertTrue(scheduler._all_isclose(srv_a.size(None), [1, 1]))
self.assertTrue(scheduler._all_isclose(left.size(None), [2, 2]))
self.assertTrue(scheduler._all_isclose(top.size(None), [4, 4]))
self.assertEqual({'a': srv_a,
'b': srv_b,
'y': srv_y,
'z': srv_z}, top.members())
def test_serialization(self):
"""Tests cell serialization."""
# Disable pylint's too many statements warning.
#
# pylint: disable=R0915
cell = scheduler.Cell('top')
left = scheduler.Bucket('left', traits=0)
right = scheduler.Bucket('right', traits=0)
srv_a = scheduler.Server('a', [10, 10], traits=0, valid_until=500)
srv_b = scheduler.Server('b', [10, 10], traits=0, valid_until=500)
srv_y = scheduler.Server('y', [10, 10], traits=0, valid_until=500)
srv_z = scheduler.Server('z', [10, 10], traits=0, valid_until=500)
cell.add_node(left)
cell.add_node(right)
left.add_node(srv_a)
left.add_node(srv_b)
right.add_node(srv_y)
right.add_node(srv_z)
left.level = 'rack'
right.level = 'rack'
apps = app_list(10, 'app', 50, [1, 1],
affinity_limits={'server': 1, 'rack': 1})
cell.add_app(cell.partitions[None].allocation, apps[0])
cell.add_app(cell.partitions[None].allocation, apps[1])
cell.add_app(cell.partitions[None].allocation, apps[2])
cell.add_app(cell.partitions[None].allocation, apps[3])
cell.schedule()
def test_valid_times(self):
"""Tests node valid_until calculation."""
top = scheduler.Bucket('top', traits=_traits2int(['top']))
left = scheduler.Bucket('left', traits=_traits2int(['left']))
right = scheduler.Bucket('right', traits=_traits2int(['right']))
srv_a = scheduler.Server('a', [10, 10], traits=_traits2int(['a', '0']),
valid_until=1)
srv_b = scheduler.Server('b', [10, 10], traits=_traits2int(['b', '0']),
valid_until=2)
srv_y = scheduler.Server('y', [10, 10], traits=_traits2int(['y', '1']),
valid_until=3)
srv_z = scheduler.Server('z', [10, 10], traits=_traits2int(['z', '1']),
valid_until=4)
top.add_node(left)
top.add_node(right)
left.add_node(srv_a)
left.add_node(srv_b)
right.add_node(srv_y)
right.add_node(srv_z)
self.assertEqual(top.valid_until, 4)
self.assertEqual(left.valid_until, 2)
self.assertEqual(right.valid_until, 4)
left.remove_node_by_name('a')
self.assertEqual(top.valid_until, 4)
self.assertEqual(left.valid_until, 2)
self.assertEqual(right.valid_until, 4)
right.remove_node_by_name('z')
self.assertEqual(top.valid_until, 3)
self.assertEqual(left.valid_until, 2)
self.assertEqual(right.valid_until, 3)
def _construct_cell():
"""Constructs a test cell."""
cell = scheduler.Cell('top')
rack1 = scheduler.Bucket('rack:rack1', traits=0, level='rack')
rack2 = scheduler.Bucket('rack:rack2', traits=0, level='rack')
cell.add_node(rack1)
cell.add_node(rack2)
srv1 = scheduler.Server('srv1', [10, 20, 30], traits=3, valid_until=1000)
srv2 = scheduler.Server('srv2', [10, 20, 30], traits=7, valid_until=2000)
srv3 = scheduler.Server('srv3', [10, 20, 30], traits=0, valid_until=3000)
srv4 = scheduler.Server('srv4', [10, 20, 30], traits=0, valid_until=4000)
rack1.add_node(srv1)
rack1.add_node(srv2)
rack2.add_node(srv3)
rack2.add_node(srv4)
tenant1 = scheduler.Allocation()
tenant2 = scheduler.Allocation()
tenant3 = scheduler.Allocation()
alloc1 = scheduler.Allocation([10, 10, 10], rank=100, traits=0)
alloc2 = scheduler.Allocation([10, 10, 10], rank=100, traits=3)
cell.partitions[None].allocation.add_sub_alloc('t1', tenant1)
cell.partitions[None].allocation.add_sub_alloc('t2', tenant2)
tenant1.add_sub_alloc('t3', tenant3)
tenant2.add_sub_alloc('a1', alloc1)
tenant3.add_sub_alloc('a2', alloc2)
return cell
def test_affinity_counters(self):
"""Tests affinity counters."""
top = scheduler.Bucket('top', traits=_traits2int(['top']))
left = scheduler.Bucket('left', traits=_traits2int(['left']))
right = scheduler.Bucket('right', traits=_traits2int(['right']))
srv_a = scheduler.Server('a', [10, 10], traits=0, valid_until=500)
srv_b = scheduler.Server('b', [10, 10], traits=0, valid_until=500)
srv_y = scheduler.Server('y', [10, 10], traits=0, valid_until=500)
srv_z = scheduler.Server('z', [10, 10], traits=0, valid_until=500)
top.add_node(left)
top.add_node(right)
left.add_node(srv_a)
left.add_node(srv_b)
right.add_node(srv_y)
right.add_node(srv_z)
apps_a = app_list(10, 'app_a', 50, [1, 1])
self.assertTrue(srv_a.put(apps_a[0]))
self.assertEqual(1, srv_a.affinity_counters['app_a'])
self.assertEqual(1, left.affinity_counters['app_a'])
self.assertEqual(1, top.affinity_counters['app_a'])
srv_z.put(apps_a[0])
self.assertEqual(1, srv_z.affinity_counters['app_a'])
self.assertEqual(1, left.affinity_counters['app_a'])
self.assertEqual(2, top.affinity_counters['app_a'])
srv_a.remove(apps_a[0].name)
self.assertEqual(0, srv_a.affinity_counters['app_a'])
self.assertEqual(0, left.affinity_counters['app_a'])
self.assertEqual(1, top.affinity_counters['app_a'])
def test_bucket_placement(self):
"""Tests placement strategies."""
top = scheduler.Bucket('top')
a_bucket = scheduler.Bucket('a_bucket')
top.add_node(a_bucket)
b_bucket = scheduler.Bucket('b_bucket')
top.add_node(b_bucket)
a1_srv = scheduler.Server('a1_srv', [10, 10], valid_until=500)
a_bucket.add_node(a1_srv)
a2_srv = scheduler.Server('a2_srv', [10, 10], valid_until=500)
a_bucket.add_node(a2_srv)
b1_srv = scheduler.Server('b1_srv', [10, 10], valid_until=500)
b_bucket.add_node(b1_srv)
b2_srv = scheduler.Server('b2_srv', [10, 10], valid_until=500)
b_bucket.add_node(b2_srv)
# bunch of apps with the same affinity
apps1 = app_list(10, 'app1', 50, [1, 1])
apps2 = app_list(10, 'app2', 50, [1, 1])
# Default strategy is spread, so placing 4 apps1 will result in each
# node having one app.
self.assertTrue(top.put(apps1[0]))
self.assertTrue(top.put(apps1[1]))
self.assertTrue(top.put(apps1[2]))
self.assertTrue(top.put(apps1[3]))
# from top level, it will spread between a and b buckets, so first
# two apps go to a1_srv, b1_srv respectively.
#
# 3rd app - buckets rotate, and a bucket is preferred again. Inside the
# bucket, next node is chosed. Same for 4th app.
#
# Result is the after 4 placements they are spread evenly.
#
self.assertEqual(1, len(a1_srv.apps))
self.assertEqual(1, len(a2_srv.apps))
self.assertEqual(1, len(b1_srv.apps))
self.assertEqual(1, len(b2_srv.apps))
a_bucket.set_affinity_strategy('app2', scheduler.PackStrategy)
self.assertTrue(top.put(apps2[0]))
self.assertTrue(top.put(apps2[1]))
self.assertTrue(top.put(apps2[2]))
self.assertTrue(top.put(apps2[3]))
# B bucket still uses spread strategy.
self.assertEqual(2, len(b1_srv.apps))
self.assertEqual(2, len(b2_srv.apps))
# Without predicting exact placement, apps will be placed on one of
# the servers in A bucket but not the other, as they use pack strateg.
self.assertNotEquals(len(a1_srv.apps), len(a2_srv.apps))
def test_node_features(self):
"""Tests node feature inheritance."""
top = scheduler.Bucket('top', features=['top'])
left = scheduler.Bucket('left', features=['left'])
right = scheduler.Bucket('right', features=['right'])
srv_a = scheduler.Server('a', [10, 10],
features=['a', '0'],
valid_until=500)
srv_b = scheduler.Server('b', [10, 10],
features=['b', '0'],
valid_until=500)
srv_y = scheduler.Server('y', [10, 10],
features=['y', '1'],
valid_until=500)
srv_z = scheduler.Server('z', [10, 10],
features=['z', '1'],
valid_until=500)
top.add_node(left)
top.add_node(right)
left.add_node(srv_a)
left.add_node(srv_b)
right.add_node(srv_y)
right.add_node(srv_z)
self.assertTrue(top.features.has('a'))
self.assertTrue(top.features.has('b'))
self.assertTrue(top.features.has('0'))
self.assertTrue(top.features.has('y'))
self.assertTrue(top.features.has('z'))
self.assertTrue(top.features.has('1'))
self.assertTrue(left.features.has('a'))
self.assertTrue(left.features.has('b'))
self.assertTrue(left.features.has('0'))
self.assertFalse(left.features.has('y'))
self.assertFalse(left.features.has('z'))
self.assertFalse(left.features.has('1'))
self.assertTrue(srv_a.features.has('top'))
self.assertTrue(srv_a.features.has('left'))
self.assertFalse(srv_a.features.has('right'))
left.remove_node('a')
self.assertFalse(left.features.has('a'))
self.assertTrue(left.features.has('b'))
self.assertTrue(left.features.has('0'))
self.assertFalse(top.features.has('a'))
self.assertTrue(top.features.has('b'))
self.assertTrue(top.features.has('0'))
left.remove_node('b')
self.assertFalse(left.features.has('b'))
self.assertFalse(left.features.has('0'))
self.assertFalse(top.features.has('b'))
self.assertFalse(top.features.has('0'))
def test_node_traits(self):
"""Tests node trait inheritance."""
top = scheduler.Bucket('top', traits=_traits2int(['top']))
left = scheduler.Bucket('left', traits=_traits2int(['left']))
right = scheduler.Bucket('right', traits=_traits2int(['right']))
srv_a = scheduler.Server('a', [10, 10],
traits=_traits2int(['a', '0']),
valid_until=500)
srv_b = scheduler.Server('b', [10, 10],
traits=_traits2int(['b', '0']),
valid_until=500)
srv_y = scheduler.Server('y', [10, 10],
traits=_traits2int(['y', '1']),
valid_until=500)
srv_z = scheduler.Server('z', [10, 10],
traits=_traits2int(['z', '1']),
valid_until=500)
top.add_node(left)
top.add_node(right)
left.add_node(srv_a)
left.add_node(srv_b)
right.add_node(srv_y)
right.add_node(srv_z)
self.assertTrue(top.traits.has(_trait2int('a')))
self.assertTrue(top.traits.has(_trait2int('b')))
self.assertTrue(top.traits.has(_trait2int('0')))
self.assertTrue(top.traits.has(_trait2int('y')))
self.assertTrue(top.traits.has(_trait2int('z')))
self.assertTrue(top.traits.has(_trait2int('1')))
self.assertTrue(left.traits.has(_trait2int('a')))
self.assertTrue(left.traits.has(_trait2int('b')))
self.assertTrue(left.traits.has(_trait2int('0')))
self.assertFalse(left.traits.has(_trait2int('y')))
self.assertFalse(left.traits.has(_trait2int('z')))
self.assertFalse(left.traits.has(_trait2int('1')))
left.remove_node_by_name('a')
self.assertFalse(left.traits.has(_trait2int('a')))
self.assertTrue(left.traits.has(_trait2int('b')))
self.assertTrue(left.traits.has(_trait2int('0')))
self.assertFalse(top.traits.has(_trait2int('a')))
self.assertTrue(top.traits.has(_trait2int('b')))
self.assertTrue(top.traits.has(_trait2int('0')))
left.remove_node_by_name('b')
self.assertFalse(left.traits.has(_trait2int('b')))
self.assertFalse(left.traits.has(_trait2int('0')))
self.assertFalse(top.traits.has(_trait2int('b')))
self.assertFalse(top.traits.has(_trait2int('0')))
def test_emtpy(self):
"""Simple test to test empty bucket"""
cell = scheduler.Cell('top')
empty = scheduler.Bucket('empty', traits=0)
cell.add_node(empty)
bucket = scheduler.Bucket('bucket', traits=0)
srv_a = scheduler.Server('a', [10, 10], traits=0, valid_until=500)
bucket.add_node(srv_a)
cell.add_node(bucket)
cell.schedule()
def prepareData(nodes_count, app_count, affinity):
scheduler.DIMENSION_COUNT = 3
cell = scheduler.Cell("local", labels=set([None]))
num_racks = math.ceil(nodes_count / NODES_PER_RACK)
counter_nodes = nodes_count
for i in range(0, num_racks):
rack = scheduler.Bucket('racks' + str(i), traits=0)
cell.add_node(rack)
for j in range(0, NODES_PER_RACK):
if counter_nodes is 0:
break
counter_nodes = counter_nodes - 1
rack.add_node(
scheduler.Server(
'node' + str(j),
resources({
"memory": "2G",
"disk": "20G",
"cpu": "90%"
}),
time.time() * 2))
for app_idx in range(0, app_count):
prio = random.randint(0, 5)
demand = resources({"memory": "1G", "disk": "10G", "cpu": "40%"})
name = 'app_.%s' % (app_idx)
app = scheduler.Application(name,
prio,
demand,
affinity=affinity(app_idx))
cell.partitions[None].allocation.add(app)
return cell
def _construct_cell(empty=False):
"""Constructs a test cell."""
cell = scheduler.Cell('top')
if empty:
return cell
rack1 = scheduler.Bucket('rack:rack1', traits=0, level='rack')
rack2 = scheduler.Bucket('rack:rack2', traits=0, level='rack')
cell.add_node(rack1)
cell.add_node(rack2)
srv1 = scheduler.Server('srv1', [10, 20, 30],
traits=1,
valid_until=1000,
label='part')
srv2 = scheduler.Server('srv2', [10, 20, 30],
traits=3,
valid_until=2000,
label='part')
srv3 = scheduler.Server('srv3', [10, 20, 30],
traits=0,
valid_until=3000,
label='_default')
srv4 = scheduler.Server('srv4', [10, 20, 30],
traits=0,
valid_until=4000,
label='_default')
rack1.add_node(srv1)
rack1.add_node(srv2)
rack2.add_node(srv3)
rack2.add_node(srv4)
tenant1 = scheduler.Allocation()
cell.partitions['_default'].allocation.add_sub_alloc('t1', tenant1)
tenant11 = scheduler.Allocation()
tenant1.add_sub_alloc('t11', tenant11)
alloc1 = scheduler.Allocation([10, 10, 10], rank=100, traits=0)
tenant11.add_sub_alloc('a1', alloc1)
tenant2 = scheduler.Allocation()
cell.partitions['part'].allocation.add_sub_alloc('t2', tenant2)
alloc2 = scheduler.Allocation([10, 10, 10], rank=100, traits=3)
tenant2.add_sub_alloc('a2', alloc2)
return cell
def _construct_cell():
"""Constructs a test cell."""
cell = scheduler.Cell('top')
rack1 = scheduler.Bucket('rack:rack1', features=[], level='rack')
rack2 = scheduler.Bucket('rack:rack2', features=[], level='rack')
cell.add_node(rack1)
cell.add_node(rack2)
srv1 = scheduler.Server('srv1', [10, 20, 30],
features=['aaa', 'bbb'],
valid_until=1000)
srv2 = scheduler.Server('srv2', [10, 20, 30],
features=['ccc'],
valid_until=2000)
srv3 = scheduler.Server('srv3', [10, 20, 30],
features=[],
valid_until=3000)
srv4 = scheduler.Server('srv4', [10, 20, 30],
features=[],
valid_until=4000)
rack1.add_node(srv1)
rack1.add_node(srv2)
rack2.add_node(srv3)
rack2.add_node(srv4)
tenant1 = scheduler.Allocation()
tenant2 = scheduler.Allocation()
tenant3 = scheduler.Allocation()
alloc1 = scheduler.Allocation([10, 10, 10], rank=100, features=[])
alloc2 = scheduler.Allocation([10, 10, 10], rank=100, features=['aaa'])
cell.allocation.add_sub_alloc('t1', tenant1)
cell.allocation.add_sub_alloc('t2', tenant2)
tenant1.add_sub_alloc('t3', tenant3)
tenant2.add_sub_alloc('a1', alloc1)
tenant3.add_sub_alloc('a2', alloc2)
return cell
def test_bucket_capacity(self):
"""Tests adjustment of bucket capacity up and down."""
parent = scheduler.Bucket('top')
bucket = scheduler.Bucket('b')
parent.add_node(bucket)
srv1 = scheduler.Server('n1', [10, 5], valid_until=500)
bucket.add_node(srv1)
self.assertTrue(np.array_equal(bucket.free_capacity,
np.array([10., 5.])))
self.assertTrue(np.array_equal(parent.free_capacity,
np.array([10., 5.])))
srv2 = scheduler.Server('n2', [5, 10], valid_until=500)
bucket.add_node(srv2)
self.assertTrue(np.array_equal(bucket.free_capacity,
np.array([10., 10.])))
self.assertTrue(np.array_equal(parent.free_capacity,
np.array([10., 10.])))
srv3 = scheduler.Server('n3', [3, 3], valid_until=500)
bucket.add_node(srv3)
self.assertTrue(np.array_equal(bucket.free_capacity,
np.array([10., 10.])))
self.assertTrue(np.array_equal(parent.free_capacity,
np.array([10., 10.])))
bucket.remove_node_by_name('n3')
self.assertTrue(np.array_equal(bucket.free_capacity,
np.array([10., 10.])))
self.assertTrue(np.array_equal(parent.free_capacity,
np.array([10., 10.])))
bucket.remove_node_by_name('n1')
self.assertTrue(np.array_equal(bucket.free_capacity,
np.array([5., 10.])))
self.assertTrue(np.array_equal(parent.free_capacity,
np.array([5., 10.])))
def test_app_node_placement(self):
"""Tests capacity adjustments for app placement."""
parent = scheduler.Bucket('top')
bucket = scheduler.Bucket('a_bucket')
parent.add_node(bucket)
srv1 = scheduler.Server('n1', [10, 5], valid_until=500)
bucket.add_node(srv1)
srv2 = scheduler.Server('n2', [10, 5], valid_until=500)
bucket.add_node(srv2)
self.assertTrue(np.array_equal(bucket.free_capacity,
np.array([10., 5.])))
self.assertTrue(np.array_equal(parent.free_capacity,
np.array([10., 5.])))
self.assertTrue(np.array_equal(bucket.size(None),
np.array([20., 10.])))
# Create 10 identical apps.
apps = app_list(10, 'app', 50, [1, 2])
self.assertTrue(srv1.put(apps[0]))
# Capacity of buckets should not change, other node is intact.
self.assertTrue(np.array_equal(bucket.free_capacity,
np.array([10., 5.])))
self.assertTrue(np.array_equal(parent.free_capacity,
np.array([10., 5.])))
self.assertTrue(srv1.put(apps[1]))
self.assertTrue(srv2.put(apps[2]))
self.assertTrue(np.array_equal(bucket.free_capacity,
np.array([9., 3.])))
self.assertTrue(np.array_equal(parent.free_capacity,
np.array([9., 3.])))
def load_bucket(self, bucketname):
"""Load bucket info, assume parent is already created."""
# Do not load twice.
if bucketname in self.buckets:
return self.buckets[bucketname]
_LOGGER.info('loading bucket: %s', bucketname)
data = self.backend.get_default(z.path.bucket(bucketname), default={})
level = data.get('level', bucketname.split(':')[0])
bucket = scheduler.Bucket(bucketname, level=level)
self.buckets[bucketname] = bucket
parent_name = data.get('parent')
if parent_name:
parent = self.load_bucket(parent_name)
parent.add_node(bucket)
return bucket
def test_data_retention(self):
"""Tests data retention."""
# Disable pylint's too many statements warning.
#
# pylint: disable=R0915
cell = scheduler.Cell('top')
left = scheduler.Bucket('left', traits=0)
right = scheduler.Bucket('right', traits=0)
srvs = {
'a': scheduler.Server('a', [10, 10], traits=0, valid_until=500),
'b': scheduler.Server('b', [10, 10], traits=0, valid_until=500),
'y': scheduler.Server('y', [10, 10], traits=0, valid_until=500),
'z': scheduler.Server('z', [10, 10], traits=0, valid_until=500),
}
cell.add_node(left)
cell.add_node(right)
left.add_node(srvs['a'])
left.add_node(srvs['b'])
right.add_node(srvs['y'])
right.add_node(srvs['z'])
left.level = 'rack'
right.level = 'rack'
time.time.return_value = 100
sticky_apps = app_list(10, 'sticky', 50, [1, 1],
affinity_limits={'server': 1, 'rack': 1},
data_retention_timeout=30)
unsticky_app = scheduler.Application('unsticky', 10, [1., 1.],
'unsticky',
data_retention_timeout=0)
cell.partitions[None].allocation.add(sticky_apps[0])
cell.partitions[None].allocation.add(unsticky_app)
cell.schedule()
# Both apps having different affinity, will be on same node.
first_srv = sticky_apps[0].server
self.assertEqual(sticky_apps[0].server, unsticky_app.server)
# Mark srv_a as down, unsticky app migrates right away,
# sticky stays.
srvs[first_srv].state = scheduler.State.down
cell.schedule()
self.assertEqual(sticky_apps[0].server, first_srv)
self.assertNotEquals(unsticky_app.server, first_srv)
self.assertEqual(cell.next_event_at, 130)
time.time.return_value = 110
cell.schedule()
self.assertEqual(sticky_apps[0].server, first_srv)
self.assertNotEquals(unsticky_app.server, first_srv)
self.assertEqual(cell.next_event_at, 130)
time.time.return_value = 130
cell.schedule()
self.assertNotEquals(sticky_apps[0].server, first_srv)
self.assertNotEquals(unsticky_app.server, first_srv)
self.assertEqual(cell.next_event_at, np.inf)
second_srv = sticky_apps[0].server
# Mark srv_a as up, srv_y as down.
srvs[first_srv].state = scheduler.State.up
srvs[second_srv].state = scheduler.State.down
cell.schedule()
self.assertEqual(sticky_apps[0].server, second_srv)
self.assertNotEquals(unsticky_app.server, second_srv)
self.assertEqual(cell.next_event_at, 160)
# Schedule one more sticky app. As it has rack affinity limit 1, it
# can't to to right (x,y) rack, rather will end up in left (a,b) rack.
#
# Other sticky apps will be pending.
time.time.return_value = 135
cell.partitions[None].allocation.add(sticky_apps[1])
cell.partitions[None].allocation.add(sticky_apps[2])
cell.schedule()
# Original app still on 'y', timeout did not expire
self.assertEqual(sticky_apps[0].server, second_srv)
# next sticky app is on (a,b) rack.
# self.assertIn(sticky_apps[1].server, ['a', 'b'])
# The 3rd sticky app pending, as rack affinity taken by currently
# down node y.
self.assertIsNone(sticky_apps[2].server)
srvs[second_srv].state = scheduler.State.up
cell.schedule()
# Original app still on 'y', timeout did not expire
self.assertEqual(sticky_apps[0].server, second_srv)
# next sticky app is on (a,b) rack.
# self.assertIn(sticky_apps[1].server, ['a', 'b'])
# The 3rd sticky app pending, as rack affinity taken by currently
# app[0] on node y.
self.assertIsNone(sticky_apps[2].server)
def test_affinity_limits(self):
"""Simple placement test."""
cell = scheduler.Cell('top')
left = scheduler.Bucket('left', traits=0)
right = scheduler.Bucket('right', traits=0)
srv_a = scheduler.Server('a', [10, 10], traits=0, valid_until=500)
srv_b = scheduler.Server('b', [10, 10], traits=0, valid_until=500)
srv_y = scheduler.Server('y', [10, 10], traits=0, valid_until=500)
srv_z = scheduler.Server('z', [10, 10], traits=0, valid_until=500)
cell.add_node(left)
cell.add_node(right)
left.add_node(srv_a)
left.add_node(srv_b)
right.add_node(srv_y)
right.add_node(srv_z)
left.level = 'rack'
right.level = 'rack'
apps = app_list(10, 'app', 50, [1, 1],
affinity_limits={'server': 1})
cell.add_app(cell.partitions[None].allocation, apps[0])
cell.add_app(cell.partitions[None].allocation, apps[1])
cell.add_app(cell.partitions[None].allocation, apps[2])
cell.add_app(cell.partitions[None].allocation, apps[3])
cell.add_app(cell.partitions[None].allocation, apps[4])
cell.schedule()
self.assertIsNotNone(apps[0].server)
self.assertIsNotNone(apps[1].server)
self.assertIsNotNone(apps[2].server)
self.assertIsNotNone(apps[3].server)
self.assertIsNone(apps[4].server)
for app in apps:
cell.remove_app(app.name)
apps = app_list(10, 'app', 50, [1, 1],
affinity_limits={'server': 1, 'rack': 1})
cell.add_app(cell.partitions[None].allocation, apps[0])
cell.add_app(cell.partitions[None].allocation, apps[1])
cell.add_app(cell.partitions[None].allocation, apps[2])
cell.add_app(cell.partitions[None].allocation, apps[3])
cell.schedule()
self.assertIsNotNone(apps[0].server)
self.assertIsNotNone(apps[1].server)
self.assertIsNone(apps[2].server)
self.assertIsNone(apps[3].server)
for app in apps:
cell.remove_app(app.name)
apps = app_list(10, 'app', 50, [1, 1],
affinity_limits={'server': 1, 'rack': 2, 'cell': 3})
cell.add_app(cell.partitions[None].allocation, apps[0])
cell.add_app(cell.partitions[None].allocation, apps[1])
cell.add_app(cell.partitions[None].allocation, apps[2])
cell.add_app(cell.partitions[None].allocation, apps[3])
cell.schedule()
self.assertIsNotNone(apps[0].server)
self.assertIsNotNone(apps[1].server)
self.assertIsNotNone(apps[2].server)
self.assertIsNone(apps[3].server)
def test_simple(self):
"""Simple placement test."""
# pylint - too many lines.
#
# pylint: disable=R0915
cell = scheduler.Cell('top')
left = scheduler.Bucket('left', traits=0)
right = scheduler.Bucket('right', traits=0)
srv_a = scheduler.Server('a', [10, 10], traits=0, valid_until=500)
srv_b = scheduler.Server('b', [10, 10], traits=0, valid_until=500)
srv_y = scheduler.Server('y', [10, 10], traits=0, valid_until=500)
srv_z = scheduler.Server('z', [10, 10], traits=0, valid_until=500)
cell.add_node(left)
cell.add_node(right)
left.add_node(srv_a)
left.add_node(srv_b)
right.add_node(srv_y)
right.add_node(srv_z)
app1 = scheduler.Application('app1', 4, [1, 1], 'app')
app2 = scheduler.Application('app2', 3, [2, 2], 'app')
app3 = scheduler.Application('app3', 2, [3, 3], 'app')
app4 = scheduler.Application('app4', 1, [4, 4], 'app')
cell.partitions[None].allocation.add(app1)
cell.partitions[None].allocation.add(app2)
cell.partitions[None].allocation.add(app3)
cell.partitions[None].allocation.add(app4)
cell.schedule()
self.assertEqual(
set([app1.server, app2.server, app3.server, app4.server]),
set(['a', 'y', 'b', 'z'])
)
srv1 = app1.server
srv2 = app2.server
srv3 = app3.server
srv4 = app4.server
# Add high priority app that needs entire cell
app_prio50 = scheduler.Application('prio50', 50, [10, 10], 'app')
cell.partitions[None].allocation.add(app_prio50)
cell.schedule()
# The queue is ordered by priority:
# - prio50, app1, app2, app3, app4
#
# As placement not found for prio50, app4 will be evicted first.
#
# As result, prio50 will be placed on 'z', and app4 (evicted) will be
# placed on "next" server, which is 'a'.
self.assertEqual(app_prio50.server, srv4)
self.assertEqual(app4.server, srv1)
app_prio51 = scheduler.Application('prio51', 51, [10, 10], 'app')
cell.partitions[None].allocation.add(app_prio51)
cell.schedule()
# app4 is now colocated with app1. app4 will still be evicted first,
# then app3, at which point there will be enough capacity to place
# large app.
#
# app3 will be rescheduled to run on "next" server - 'y', and app4 will
# be restored to 'a'.
self.assertEqual(app_prio51.server, srv3)
self.assertEqual(app_prio50.server, srv4)
self.assertEqual(app4.server, srv1)
app_prio49_1 = scheduler.Application('prio49_1', 49, [10, 10], 'app')
app_prio49_2 = scheduler.Application('prio49_2', 49, [9, 9], 'app')
cell.partitions[None].allocation.add(app_prio49_1)
cell.partitions[None].allocation.add(app_prio49_2)
cell.schedule()
# 50/51 not moved. from the end of the queue,
self.assertEqual(app_prio51.server, srv3)
self.assertEqual(app_prio50.server, srv4)
self.assertEqual(set([app_prio49_1.server, app_prio49_2.server]),
set([srv1, srv2]))
# Only capacity left for small [1, 1] app.
self.assertIsNotNone(app1.server)
self.assertIsNone(app2.server)
self.assertIsNone(app3.server)
self.assertIsNone(app4.server)
def test_app_trait_placement(self):
"""Tests placement of app with traits."""
top = scheduler.Bucket('top', traits=_traits2int(['top']))
left = scheduler.Bucket('left', traits=_traits2int(['left']))
right = scheduler.Bucket('right', traits=_traits2int(['right']))
srv_a = scheduler.Server('a', [10, 10], traits=_traits2int(['a', '0']),
valid_until=500)
srv_b = scheduler.Server('b', [10, 10], traits=_traits2int(['b', '0']),
valid_until=500)
srv_y = scheduler.Server('y', [10, 10], traits=_traits2int(['y', '1']),
valid_until=500)
srv_z = scheduler.Server('z', [10, 10], traits=_traits2int(['z', '1']),
valid_until=500)
top.add_node(left)
top.add_node(right)
left.add_node(srv_a)
left.add_node(srv_b)
right.add_node(srv_y)
right.add_node(srv_z)
alloc_a = scheduler.Allocation(traits=_traits2int(['a']))
apps_a = app_list(10, 'app_a', 50, [2, 2])
for app in apps_a:
alloc_a.add(app)
# srv_a is the only one with trait 'a'.
self.assertTrue(top.put(apps_a[0]))
self.assertTrue(top.put(apps_a[1]))
self.assertIn(apps_a[0].name, srv_a.apps)
self.assertIn(apps_a[1].name, srv_a.apps)
alloc_0 = scheduler.Allocation(traits=_traits2int(['0']))
apps_0 = app_list(10, 'app_0', 50, [2, 2])
for app in apps_0:
alloc_0.add(app)
# '0' trait - two servers, will spread by default.
self.assertTrue(top.put(apps_0[0]))
self.assertTrue(top.put(apps_0[1]))
self.assertIn(apps_0[0].name, srv_a.apps)
self.assertIn(apps_0[1].name, srv_b.apps)
# Prev implementation propagated traits from parent to children,
# so "right" trait propagated to leaf servers.
#
# This behavior is removed, so placing app with "right" trait will
# fail.
#
# alloc_r1 = scheduler.Allocation(traits=_traits2int(['right', '1']))
# apps_r1 = app_list(10, 'app_r1', 50, [2, 2])
# for app in apps_r1:
# alloc_r1.add(app)
# self.assertTrue(top.put(apps_r1[0]))
# self.assertTrue(top.put(apps_r1[1]))
# self.assertIn(apps_r1[0].name, srv_y.apps)
# self.assertIn(apps_r1[1].name, srv_z.apps)
apps_nothing = app_list(10, 'apps_nothing', 50, [1, 1])
# All nodes fit. Spead first between buckets, then between nodes.
# top
# left right
# a b y z
self.assertTrue(top.put(apps_nothing[0]))
self.assertTrue(top.put(apps_nothing[1]))
self.assertTrue(
(apps_nothing[0].server in ['a', 'b'] and
apps_nothing[1].server in ['y', 'z']) or
(apps_nothing[0].server in ['y', 'z'] and
apps_nothing[1].server in ['a', 'b']))
self.assertTrue(top.put(apps_nothing[2]))
self.assertTrue(top.put(apps_nothing[3]))
self.assertTrue(
(apps_nothing[2].server in ['a', 'b'] and
apps_nothing[3].server in ['y', 'z']) or
(apps_nothing[2].server in ['y', 'z'] and
apps_nothing[3].server in ['a', 'b']))
def test_app_feature_placement(self):
"""Tests placement of app with features."""
top = scheduler.Bucket('top', features=['top'])
left = scheduler.Bucket('left', features=['left'])
right = scheduler.Bucket('right', features=['right'])
srv_a = scheduler.Server('a', [10, 10],
features=['a', '0'],
valid_until=500)
srv_b = scheduler.Server('b', [10, 10],
features=['b', '0'],
valid_until=500)
srv_y = scheduler.Server('y', [10, 10],
features=['y', '1'],
valid_until=500)
srv_z = scheduler.Server('z', [10, 10],
features=['z', '1'],
valid_until=500)
top.add_node(left)
top.add_node(right)
left.add_node(srv_a)
left.add_node(srv_b)
right.add_node(srv_y)
right.add_node(srv_z)
alloc_a = scheduler.Allocation(features=['a'])
apps_a = app_list(10, 'app_a', 50, [2, 2])
for app in apps_a:
alloc_a.add(app)
# srv_a is the only one with feature 'a'.
self.assertTrue(top.put(apps_a[0]))
self.assertTrue(top.put(apps_a[1]))
self.assertIn(apps_a[0].name, srv_a.apps)
self.assertIn(apps_a[1].name, srv_a.apps)
alloc_0 = scheduler.Allocation(features=['0'])
apps_0 = app_list(10, 'app_0', 50, [2, 2])
for app in apps_0:
alloc_0.add(app)
# '0' feature - two servers, will spread by default.
self.assertTrue(top.put(apps_0[0]))
self.assertTrue(top.put(apps_0[1]))
self.assertIn(apps_0[0].name, srv_a.apps)
self.assertIn(apps_0[1].name, srv_b.apps)
alloc_r1 = scheduler.Allocation(features=['right', '1'])
apps_r1 = app_list(10, 'app_r1', 50, [2, 2])
for app in apps_r1:
alloc_r1.add(app)
self.assertTrue(top.put(apps_r1[0]))
self.assertTrue(top.put(apps_r1[1]))
self.assertIn(apps_r1[0].name, srv_y.apps)
self.assertIn(apps_r1[1].name, srv_z.apps)
apps_nothing = app_list(10, 'apps_nothing', 50, [1, 1])
self.assertTrue(top.put(apps_nothing[0]))
self.assertTrue(top.put(apps_nothing[1]))
self.assertTrue(top.put(apps_nothing[2]))
self.assertTrue(top.put(apps_nothing[3]))
# All nodes fit. Spead first between buckets, then between nodes.
# top
# left right
# a b y z
self.assertIn(apps_nothing[0].name, srv_a.apps)
self.assertIn(apps_nothing[1].name, srv_y.apps)
self.assertIn(apps_nothing[2].name, srv_b.apps)
self.assertIn(apps_nothing[3].name, srv_z.apps)
