def test_explain_queue(self):
"""Test explain queue"""
app1 = scheduler.Application('foo.xxx#1',
100,
demand=[1, 1, 1],
affinity='foo.xxx')
app2 = scheduler.Application('bar.xxx#2',
100,
demand=[1, 1, 1],
affinity='foo.xxx')
app3 = scheduler.Application('bla.xxx#3',
50,
demand=[1, 1, 1],
affinity='bla.xxx')
(self.cell.partitions['_default'].allocation.get_sub_alloc(
't1').get_sub_alloc('t3').get_sub_alloc('a2').add(app1))
(self.cell.partitions['_default'].allocation.get_sub_alloc(
't1').get_sub_alloc('t3').get_sub_alloc('a2').add(app2))
(self.cell.partitions['part'].allocation.get_sub_alloc(
't2').get_sub_alloc('a2').add(app3))
df = reports.explain_queue(self.cell, '_default')
self.assertEqual(len(df), 2 * 4) # 2 apps at 4 alloc levels
df = reports.explain_queue(self.cell, '_default', 'foo*')
self.assertEqual(len(df), 4) # 1 app at 4 alloc levels
df = reports.explain_queue(self.cell, 'part')
self.assertEqual(len(df), 3) # 1 app at 3 alloc levels
def test_placement_integrity(self):
"""Tests placement integrity."""
zk_content = {
'placement': {
'test1.xx.com': {
'xxx.app1#1234': '',
'xxx.app2#2345': '',
},
'test2.xx.com': {
'xxx.app1#1234': '',
}
},
}
self.master.cell.apps['xxx.app1#1234'] = scheduler.Application(
'xxx.app1#1234', 100, [1, 1, 1], 'app1')
self.master.cell.apps['xxx.app2#2345'] = scheduler.Application(
'xxx.app2#2345', 100, [1, 1, 1], 'app1')
self.master.cell.apps['xxx.app1#1234'].server = 'test1.xx.com'
self.make_mock_zk(zk_content)
self.master.check_placement_integrity()
treadmill.zkutils.ensure_deleted.assert_called_with(
mock.ANY, '/placement/test2.xx.com/xxx.app1#1234')
def test_labels(self):
"""Test scheduling with labels."""
cell = scheduler.Cell('top')
left = scheduler.Bucket('left', traits=0)
right = scheduler.Bucket('right', traits=0)
srv_a = scheduler.Server('a_xx', [10, 10], valid_until=500, label='xx')
srv_b = scheduler.Server('b', [10, 10], valid_until=500)
srv_y = scheduler.Server('y_xx', [10, 10], valid_until=500, label='xx')
srv_z = scheduler.Server('z', [10, 10], valid_until=500)
cell.add_node(left)
cell.add_node(right)
left.add_node(srv_a)
left.add_node(srv_b)
right.add_node(srv_y)
right.add_node(srv_z)
app1 = scheduler.Application('app1', 4, [1, 1], 'app')
app2 = scheduler.Application('app2', 3, [2, 2], 'app')
app3 = scheduler.Application('app_xx_3', 2, [3, 3], 'app')
app4 = scheduler.Application('app_xx_4', 1, [4, 4], 'app')
cell.partitions[None].allocation.add(app1)
cell.partitions[None].allocation.add(app2)
cell.partitions['xx'].allocation.add(app3)
cell.partitions['xx'].allocation.add(app4)
cell.schedule()
self.assertIn(app1.server, ['b', 'z'])
self.assertIn(app2.server, ['b', 'z'])
self.assertIn(app3.server, ['a_xx', 'y_xx'])
self.assertIn(app4.server, ['a_xx', 'y_xx'])
def test_applications(self):
"""Tests application queue report."""
app1 = scheduler.Application('foo.xxx#1',
100,
demand=[1, 1, 1],
affinity='foo.xxx')
app2 = scheduler.Application('foo.xxx#2',
100,
demand=[1, 1, 1],
affinity='foo.xxx')
app3 = scheduler.Application('bla.xxx#3',
50,
demand=[1, 1, 1],
affinity='bla.xxx')
(self.cell.partitions[None].allocation.get_sub_alloc(
't1').get_sub_alloc('t3').get_sub_alloc('a2').add(app1))
(self.cell.partitions[None].allocation.get_sub_alloc(
't1').get_sub_alloc('t3').get_sub_alloc('a2').add(app2))
(self.cell.partitions[None].allocation.get_sub_alloc(
't2').get_sub_alloc('a1').add(app3))
self.cell.schedule()
apps_df = reports.apps(self.cell)
# affinity allocation cpu data_retention_timeout disk memory \
# instance
# foo.xxx#1 foo.xxx t1/t3/a2 1 0 1 1
# foo.xxx#2 foo.xxx t1/t3/a2 1 0 1 1
# bla.xxx#3 bla.xxx t2/a1 1 0 1 1
#
# order pending rank server util
# instance
# foo.xxx#1 1.458152e+15 0 99 srv1 -0.135714
# foo.xxx#2 1.458152e+15 0 99 srv1 -0.128571
# bla.xxx#3 1.458152e+15 0 100 srv1 -0.121429
time.time.return_value = 100
self.assertEqual(apps_df.ix['foo.xxx#2']['cpu'], 1)
util0 = reports.utilization(None, apps_df)
time.time.return_value = 101
util1 = reports.utilization(util0, apps_df)
# name bla.xxx foo.xxx \
# count util disk cpu memory count util disk
# 1969-12-31 19:00:00 1 -0.121429 1 1 1 2 -0.128571 2
# 1969-12-31 19:00:01 1 -0.121429 1 1 1 2 -0.128571 2
#
# name
# cpu memory
# 1969-12-31 19:00:00 2 2
# 1969-12-31 19:00:01 2 2
time0 = pd.Timestamp(datetime.datetime.fromtimestamp(100))
time1 = pd.Timestamp(datetime.datetime.fromtimestamp(101))
self.assertEqual(util1.ix[time0]['bla.xxx']['cpu'], 1)
self.assertEqual(util1.ix[time1]['foo.xxx']['count'], 2)
def test_duplicate(self):
"""Checks behavior when adding duplicate app."""
alloc = scheduler.Allocation(None)
alloc.add(scheduler.Application('app1', 0, [1, 1], 'app1'))
self.assertEqual(
1, len(list(alloc.utilization_queue(np.array([5., 5.])))))
alloc.add(scheduler.Application('app1', 0, [1, 1], 'app1'))
self.assertEqual(
1, len(list(alloc.utilization_queue(np.array([5., 5.])))))
def test_reschedule(self):
"""Tests application placement."""
srv_1 = scheduler.Server('1', [10, 10, 10],
valid_until=1000, traits=0)
srv_2 = scheduler.Server('2', [10, 10, 10],
valid_until=1000, traits=0)
srv_3 = scheduler.Server('3', [10, 10, 10],
valid_until=1000, traits=0)
srv_4 = scheduler.Server('4', [10, 10, 10],
valid_until=1000, traits=0)
cell = self.master.cell
cell.add_node(srv_1)
cell.add_node(srv_2)
cell.add_node(srv_3)
cell.add_node(srv_4)
app1 = scheduler.Application('app1', 4, [1, 1, 1], 'app')
app2 = scheduler.Application('app2', 3, [2, 2, 2], 'app')
cell.add_app(cell.partitions[None].allocation, app1)
cell.add_app(cell.partitions[None].allocation, app2)
# At this point app1 is on server 1, app2 on server 2.
self.master.reschedule()
treadmill.zkutils.put.assert_has_calls([
mock.call(mock.ANY, '/placement/1/app1',
{'expires': 500, 'identity': None}, acl=mock.ANY),
mock.call(mock.ANY, '/placement/2/app2',
{'expires': 500, 'identity': None}, acl=mock.ANY),
], any_order=True)
treadmill.zkutils.ensure_deleted.reset_mock()
treadmill.zkutils.put.reset_mock()
srv_1.state = scheduler.State.down
self.master.reschedule()
treadmill.zkutils.ensure_deleted.assert_has_calls([
mock.call(mock.ANY, '/placement/1/app1'),
])
treadmill.zkutils.put.assert_has_calls([
mock.call(mock.ANY, '/placement/3/app1',
{'expires': 500, 'identity': None}, acl=mock.ANY),
mock.call(mock.ANY, '/placement', mock.ANY, acl=mock.ANY),
])
# Verify that placement data was properly saved as a compressed json.
args, _kwargs = treadmill.zkutils.put.call_args_list[1]
placement_data = args[2]
placement = json.loads(
zlib.decompress(placement_data).decode()
)
self.assertIn(['app1', '1', 500, '3', 500], placement)
self.assertIn(['app2', '2', 500, '2', 500], placement)
def test_reschedule(self):
"""Tests application placement."""
srv_1 = scheduler.Server('1', [10, 10, 10], valid_until=1000, traits=0)
srv_2 = scheduler.Server('2', [10, 10, 10], valid_until=1000, traits=0)
srv_3 = scheduler.Server('3', [10, 10, 10], valid_until=1000, traits=0)
srv_4 = scheduler.Server('4', [10, 10, 10], valid_until=1000, traits=0)
cell = self.master.cell
cell.add_node(srv_1)
cell.add_node(srv_2)
cell.add_node(srv_3)
cell.add_node(srv_4)
app1 = scheduler.Application('app1', 4, [1, 1, 1], 'app')
app2 = scheduler.Application('app2', 3, [2, 2, 2], 'app')
cell.add_app(cell.partitions[None].allocation, app1)
cell.add_app(cell.partitions[None].allocation, app2)
# At this point app1 is on server 1, app2 on server 2.
self.master.reschedule()
treadmill.zkutils.put.assert_has_calls([
mock.call(mock.ANY,
'/placement/1/app1', {
'expires': 500,
'identity': None
},
acl=mock.ANY),
mock.call(mock.ANY,
'/placement/2/app2', {
'expires': 500,
'identity': None
},
acl=mock.ANY),
])
srv_1.state = scheduler.State.down
self.master.reschedule()
treadmill.zkutils.ensure_deleted.assert_has_calls([
mock.call(mock.ANY, '/placement/1/app1'),
])
treadmill.zkutils.put.assert_has_calls([
mock.call(mock.ANY,
'/placement/3/app1', {
'expires': 500,
'identity': None
},
acl=mock.ANY),
mock.call(mock.ANY, '/placement', mock.ANY),
])
def test_running_order(self):
"""Test apps are ordered by status (running first) for same prio."""
alloc = scheduler.Allocation([10, 10])
alloc.add(scheduler.Application('app1', 5, [1, 1], 'app1'))
alloc.add(scheduler.Application('app2', 5, [2, 2], 'app1'))
alloc.add(scheduler.Application('app3', 5, [3, 3], 'app1'))
queue = list(alloc.utilization_queue([20., 20.]))
self.assertEqual(alloc.apps['app1'], queue[0][-1])
alloc.apps['app2'].server = 'abc'
queue = list(alloc.utilization_queue([20., 20.]))
self.assertEqual(alloc.apps['app2'], queue[0][-1])
def prepareData(nodes_count, app_count, affinity):
scheduler.DIMENSION_COUNT = 3
cell = scheduler.Cell("local", labels=set([None]))
num_racks = math.ceil(nodes_count / NODES_PER_RACK)
counter_nodes = nodes_count
for i in range(0, num_racks):
rack = scheduler.Bucket('racks' + str(i), traits=0)
cell.add_node(rack)
for j in range(0, NODES_PER_RACK):
if counter_nodes is 0:
break
counter_nodes = counter_nodes - 1
rack.add_node(
scheduler.Server(
'node' + str(j),
resources({
"memory": "2G",
"disk": "20G",
"cpu": "90%"
}),
time.time() * 2))
for app_idx in range(0, app_count):
prio = random.randint(0, 5)
demand = resources({"memory": "1G", "disk": "10G", "cpu": "40%"})
name = 'app_.%s' % (app_idx)
app = scheduler.Application(name,
prio,
demand,
affinity=affinity(app_idx))
cell.partitions[None].allocation.add(app)
return cell
def app_list(count, name, *args, **kwargs):
"""Return list of apps."""
return [
scheduler.Application(name + '-' + str(idx),
*args,
affinity=name,
**kwargs) for idx in xrange(0, count)
]
def test_reschedule_maxutil(self):
"""Tests application placement."""
srv_1 = scheduler.Server('1', [10, 10, 10], valid_until=1000, traits=0)
srv_2 = scheduler.Server('2', [10, 10, 10], valid_until=1000, traits=0)
srv_3 = scheduler.Server('3', [10, 10, 10], valid_until=1000, traits=0)
srv_4 = scheduler.Server('4', [10, 10, 10], valid_until=1000, traits=0)
cell = self.master.cell
cell.add_node(srv_1)
cell.add_node(srv_2)
cell.add_node(srv_3)
cell.add_node(srv_4)
app1 = scheduler.Application('app1', 4, [1, 1, 1], 'app')
app2 = scheduler.Application('app2', 3, [2, 2, 2], 'app')
cell.partitions[None].allocation.set_reserved([1, 1, 1])
cell.partitions[None].allocation.set_max_utilization(2)
cell.add_app(cell.partitions[None].allocation, app1)
cell.add_app(cell.partitions[None].allocation, app2)
self.master.reschedule()
treadmill.zkutils.put.assert_has_calls([
mock.call(mock.ANY,
'/placement/1/app1', {
'expires': 500,
'identity': None
},
acl=mock.ANY),
])
app2.priority = 5
self.master.reschedule()
treadmill.zkutils.ensure_deleted.assert_has_calls([
mock.call(mock.ANY, '/placement/1/app1'),
])
treadmill.zkutils.put.assert_has_calls([
mock.call(mock.ANY,
'/placement/2/app2', {
'expires': 500,
'identity': None
},
acl=mock.ANY),
])
def test_reschedule_once(self):
"""Tests application placement."""
srv_1 = scheduler.Server('1', [10, 10, 10],
valid_until=1000,
features=[])
srv_2 = scheduler.Server('2', [10, 10, 10],
valid_until=1000,
features=[])
srv_3 = scheduler.Server('3', [10, 10, 10],
valid_until=1000,
features=[])
srv_4 = scheduler.Server('4', [10, 10, 10],
valid_until=1000,
features=[])
cell = self.master.cell
cell.add_node(srv_1)
cell.add_node(srv_2)
cell.add_node(srv_3)
cell.add_node(srv_4)
app1 = scheduler.Application('app1',
4, [1, 1, 1],
'app',
schedule_once=True)
app2 = scheduler.Application('app2', 3, [2, 2, 2], 'app')
cell.add_app(cell.allocation, app1)
cell.add_app(cell.allocation, app2)
# At this point app1 is on server 1, app2 on server 2.
self.master.reschedule()
treadmill.zkutils.put.assert_has_calls([
mock.call(mock.ANY, '/placement/1/app1', None, acl=mock.ANY),
mock.call(mock.ANY, '/placement/2/app2', None, acl=mock.ANY),
])
srv_1.state = scheduler.State.down
self.master.reschedule()
treadmill.zkutils.ensure_deleted.assert_has_calls([
mock.call(mock.ANY, '/placement/1/app1'),
mock.call(mock.ANY, '/scheduled/app1'),
])
def test_utilization_max(self):
"""Tests max utilization cap on the allocation."""
alloc = scheduler.Allocation([3, 3])
alloc.add(scheduler.Application('app1', 1, [1, 1], 'app1'))
alloc.add(scheduler.Application('app2', 1, [2, 2], 'app1'))
alloc.add(scheduler.Application('app3', 1, [3, 3], 'app1'))
self.assertEqual(3, len(list(alloc.utilization_queue([20., 20.]))))
# Now set max_utilization to 1
alloc.max_utilization = 1
# XXX(boysson: Broken test. Needs upgrade to V3
# XXX:
# XXX: self.assertEqual(
# XXX: 2,
# XXX: len(list(alloc.utilization_queue([20., 20.])))
# XXX: )
alloc.set_max_utilization(None)
self.assertEqual(3, len(list(alloc.utilization_queue([20., 20.]))))
def test_utilization(self):
"""Test utilization calculation."""
alloc = scheduler.Allocation([10, 10])
alloc.add(scheduler.Application('app1', 100, [1, 1], 'app1'))
alloc.add(scheduler.Application('app2', 100, [2, 2], 'app1'))
alloc.add(scheduler.Application('app3', 100, [3, 3], 'app1'))
# First element is rank.
util_q = list(alloc.utilization_queue([20, 20]))
self.assertEqual(100, util_q[0][0])
self.assertEqual(100, util_q[1][0])
self.assertEqual(100, util_q[2][0])
# Second elememt is utilization.
self.assertEqual(-9. / (10. + 20), util_q[0][1])
self.assertEqual(-7. / (10. + 20), util_q[1][1])
self.assertEqual(-4. / (10. + 20), util_q[2][1])
# Applications are sorted by priority.
alloc = scheduler.Allocation([10, 10])
alloc.add(scheduler.Application('app1', 10, [1, 1], 'app1'))
alloc.add(scheduler.Application('app2', 50, [2, 2], 'app1'))
alloc.add(scheduler.Application('app3', 100, [3, 3], 'app1'))
util_q = list(alloc.utilization_queue([20., 20.]))
self.assertEqual(-7. / (10. + 20), util_q[0][1])
self.assertEqual(-5. / (10. + 20), util_q[1][1])
self.assertEqual(-4. / (10. + 20), util_q[2][1])
def test_explain_placement(self):
"""Test explain placement"""
app1 = scheduler.Application('foo.xxx#1',
100,
demand=[1, 1, 1],
affinity='foo.xxx')
alloc = (self.cell.partitions['_default'].allocation.get_sub_alloc(
't1').get_sub_alloc('t3').get_sub_alloc('a2'))
self.cell.add_app(alloc, app1)
df = reports.explain_placement(self.cell, app1, mode='full')
self.assertEqual(len(df), 7)
df = reports.explain_placement(self.cell, app1, mode='servers')
self.assertEqual(len(df), 4)
def load_app(self, appname):
"""Load single application data."""
manifest = self.backend.get_default(z.path.scheduled(appname))
if not manifest:
self.remove_app(appname)
return
priority, allocation = self.find_assignment(appname)
if 'priority' in manifest and int(manifest['priority']) != -1:
priority = int(manifest['priority'])
# TODO: From scheduler perspective it is theoretically
# possible to update data retention timeout.
data_retention = _get_data_retention(manifest)
lease = _get_lease(manifest)
app = self.cell.apps.get(appname, None)
if app:
app.priority = priority
app.data_retention_timeout = data_retention
else:
demand = resources(manifest)
affinity = manifest.get('affinity')
affinity_limits = manifest.get('affinity_limits', None)
identity_group = manifest.get('identity_group')
schedule_once = manifest.get('schedule_once')
traitz = manifest.get('traits', [])
app = scheduler.Application(appname,
priority,
demand,
affinity=affinity,
affinity_limits=affinity_limits,
identity_group=identity_group,
schedule_once=schedule_once,
data_retention_timeout=data_retention,
traits=traits.encode(self.trait_codes,
traitz,
use_invalid=True),
lease=lease)
app.blacklisted = self._is_blacklisted(appname)
self.cell.add_app(allocation, app)
def load_app(self, appname):
"""Load single application data."""
# TODO: need to check if app is blacklisted.
manifest = zkutils.get_default(self.zkclient,
z.path.scheduled(appname))
if not manifest:
self.cell.remove_app(appname)
return
priority, allocation = self.find_assignment(appname)
if 'priority' in manifest and int(manifest['priority']) != -1:
priority = int(manifest['priority'])
# TODO: From scheduler perspective it is theoretically
# possible to update data retention timeout.
data_retention = get_data_retention(manifest)
lease = get_lease(manifest)
app = self.cell.apps.get(appname, None)
if app:
app.priority = priority
app.data_retention_timeout = data_retention
else:
demand = resources(manifest)
affinity = manifest.get('affinity')
affinity_limits = manifest.get('affinity_limits', None)
identity_group = manifest.get('identity_group')
schedule_once = manifest.get('schedule_once')
app = scheduler.Application(appname,
priority,
demand,
affinity=affinity,
affinity_limits=affinity_limits,
identity_group=identity_group,
schedule_once=schedule_once,
data_retention_timeout=data_retention,
lease=lease)
self.cell.add_app(allocation, app)
def prepareData(nodes_count, app_count, affinity):
scheduler.DIMENSION_COUNT = 3
cell = scheduler.Cell("local", labels=set([None]))
for idx in range(0, nodes_count):
node = scheduler.Server('node' + str(idx), resources({
"memory": "2G",
"disk": "20G",
"cpu": "90%"
}), time.time() * 2)
cell.add_node(node)
for app_idx in range(0, app_count):
prio = random.randint(0, 5)
demand = resources({
"memory": "1G",
"disk": "10G",
"cpu": "40%"
})
name = 'app_.%s' % (app_idx)
app = scheduler.Application(name, prio, demand, affinity=affinity(app_idx))
cell.partitions[None].allocation.add(app)
return cell
def test_data_retention(self):
"""Tests data retention."""
# Disable pylint's too many statements warning.
#
# pylint: disable=R0915
cell = scheduler.Cell('top')
left = scheduler.Bucket('left', traits=0)
right = scheduler.Bucket('right', traits=0)
srvs = {
'a': scheduler.Server('a', [10, 10], traits=0, valid_until=500),
'b': scheduler.Server('b', [10, 10], traits=0, valid_until=500),
'y': scheduler.Server('y', [10, 10], traits=0, valid_until=500),
'z': scheduler.Server('z', [10, 10], traits=0, valid_until=500),
}
cell.add_node(left)
cell.add_node(right)
left.add_node(srvs['a'])
left.add_node(srvs['b'])
right.add_node(srvs['y'])
right.add_node(srvs['z'])
left.level = 'rack'
right.level = 'rack'
time.time.return_value = 100
sticky_apps = app_list(10, 'sticky', 50, [1, 1],
affinity_limits={'server': 1, 'rack': 1},
data_retention_timeout=30)
unsticky_app = scheduler.Application('unsticky', 10, [1., 1.],
'unsticky',
data_retention_timeout=0)
cell.partitions[None].allocation.add(sticky_apps[0])
cell.partitions[None].allocation.add(unsticky_app)
cell.schedule()
# Both apps having different affinity, will be on same node.
first_srv = sticky_apps[0].server
self.assertEqual(sticky_apps[0].server, unsticky_app.server)
# Mark srv_a as down, unsticky app migrates right away,
# sticky stays.
srvs[first_srv].state = scheduler.State.down
cell.schedule()
self.assertEqual(sticky_apps[0].server, first_srv)
self.assertNotEquals(unsticky_app.server, first_srv)
self.assertEqual(cell.next_event_at, 130)
time.time.return_value = 110
cell.schedule()
self.assertEqual(sticky_apps[0].server, first_srv)
self.assertNotEquals(unsticky_app.server, first_srv)
self.assertEqual(cell.next_event_at, 130)
time.time.return_value = 130
cell.schedule()
self.assertNotEquals(sticky_apps[0].server, first_srv)
self.assertNotEquals(unsticky_app.server, first_srv)
self.assertEqual(cell.next_event_at, np.inf)
second_srv = sticky_apps[0].server
# Mark srv_a as up, srv_y as down.
srvs[first_srv].state = scheduler.State.up
srvs[second_srv].state = scheduler.State.down
cell.schedule()
self.assertEqual(sticky_apps[0].server, second_srv)
self.assertNotEquals(unsticky_app.server, second_srv)
self.assertEqual(cell.next_event_at, 160)
# Schedule one more sticky app. As it has rack affinity limit 1, it
# can't to to right (x,y) rack, rather will end up in left (a,b) rack.
#
# Other sticky apps will be pending.
time.time.return_value = 135
cell.partitions[None].allocation.add(sticky_apps[1])
cell.partitions[None].allocation.add(sticky_apps[2])
cell.schedule()
# Original app still on 'y', timeout did not expire
self.assertEqual(sticky_apps[0].server, second_srv)
# next sticky app is on (a,b) rack.
# self.assertIn(sticky_apps[1].server, ['a', 'b'])
# The 3rd sticky app pending, as rack affinity taken by currently
# down node y.
self.assertIsNone(sticky_apps[2].server)
srvs[second_srv].state = scheduler.State.up
cell.schedule()
# Original app still on 'y', timeout did not expire
self.assertEqual(sticky_apps[0].server, second_srv)
# next sticky app is on (a,b) rack.
# self.assertIn(sticky_apps[1].server, ['a', 'b'])
# The 3rd sticky app pending, as rack affinity taken by currently
# app[0] on node y.
self.assertIsNone(sticky_apps[2].server)
def test_utilization_no_reservation(self):
"""Checks that any utilization without reservation is VERY large."""
alloc = scheduler.Allocation(None)
alloc.add(scheduler.Application('app1', 1, [1., 1.], 'app1'))
queue = list(alloc.utilization_queue(np.array([10., 10.])))
self.assertEqual(1. / (10), queue[0][1])
def test_sub_allocs(self):
"""Test utilization calculation with sub-allocs."""
alloc = scheduler.Allocation([3, 3])
self.assertEqual(3, alloc.total_reserved()[0])
alloc.add(scheduler.Application('1', 3, [2, 2], 'app1'))
alloc.add(scheduler.Application('2', 2, [1, 1], 'app1'))
alloc.add(scheduler.Application('3', 1, [3, 3], 'app1'))
queue = list(alloc.utilization_queue([20., 20.]))
sub_alloc_a = scheduler.Allocation([5, 5])
alloc.add_sub_alloc('a1/a', sub_alloc_a)
self.assertEqual(8, alloc.total_reserved()[0])
sub_alloc_a.add(scheduler.Application('1a', 3, [2, 2], 'app1'))
sub_alloc_a.add(scheduler.Application('2a', 2, [3, 3], 'app1'))
sub_alloc_a.add(scheduler.Application('3a', 1, [5, 5], 'app1'))
queue = list(alloc.utilization_queue([20., 20.]))
_rank, util, _pending, _order, app = queue[0]
self.assertEqual('1a', app.name)
self.assertEqual((2 - (5 + 3)) / (20. + (5 + 3)), util)
sub_alloc_b = scheduler.Allocation([10, 10])
alloc.add_sub_alloc('a1/b', sub_alloc_b)
sub_alloc_b.add(scheduler.Application('1b', 3, [2, 2], 'app1'))
sub_alloc_b.add(scheduler.Application('2b', 2, [3, 3], 'app1'))
sub_alloc_b.add(scheduler.Application('3b', 1, [5, 5], 'app1'))
queue = list(alloc.utilization_queue([20., 20.]))
self.assertEqual(9, len(queue))
self.assertEqual(18, alloc.total_reserved()[0])
# For each sub-alloc (and self) the least utilized app is 1.
# The sub_allloc_b is largest, so utilization smallest, 1b will be
# first.
_rank, util, _pending, _order, app = queue[0]
self.assertEqual('1b', app.name)
self.assertEqual((2. - 18) / (20. + 18), util)
# Add prio 0 app to each, make sure they all end up last.
alloc.add(scheduler.Application('1-zero', 0, [2, 2], 'app1'))
sub_alloc_b.add(scheduler.Application('b-zero', 0, [5, 5], 'app1'))
sub_alloc_a.add(scheduler.Application('a-zero', 0, [5, 5], 'app1'))
queue = list(alloc.utilization_queue([20., 20.]))
self.assertIn('1-zero', [item[-1].name for item in queue[-3:]])
self.assertIn('a-zero', [item[-1].name for item in queue[-3:]])
self.assertIn('b-zero', [item[-1].name for item in queue[-3:]])
# Check that utilization of prio 0 apps is always max float.
self.assertEqual(
[float('inf')] * 3,
[util for (_rank, util, _pending, _order, _app) in queue[-3:]] # noqa: F812,E501
)
def test_simple(self):
"""Simple placement test."""
# pylint - too many lines.
#
# pylint: disable=R0915
cell = scheduler.Cell('top')
left = scheduler.Bucket('left', traits=0)
right = scheduler.Bucket('right', traits=0)
srv_a = scheduler.Server('a', [10, 10], traits=0, valid_until=500)
srv_b = scheduler.Server('b', [10, 10], traits=0, valid_until=500)
srv_y = scheduler.Server('y', [10, 10], traits=0, valid_until=500)
srv_z = scheduler.Server('z', [10, 10], traits=0, valid_until=500)
cell.add_node(left)
cell.add_node(right)
left.add_node(srv_a)
left.add_node(srv_b)
right.add_node(srv_y)
right.add_node(srv_z)
app1 = scheduler.Application('app1', 4, [1, 1], 'app')
app2 = scheduler.Application('app2', 3, [2, 2], 'app')
app3 = scheduler.Application('app3', 2, [3, 3], 'app')
app4 = scheduler.Application('app4', 1, [4, 4], 'app')
cell.partitions[None].allocation.add(app1)
cell.partitions[None].allocation.add(app2)
cell.partitions[None].allocation.add(app3)
cell.partitions[None].allocation.add(app4)
cell.schedule()
self.assertEqual(
set([app1.server, app2.server, app3.server, app4.server]),
set(['a', 'y', 'b', 'z'])
)
srv1 = app1.server
srv2 = app2.server
srv3 = app3.server
srv4 = app4.server
# Add high priority app that needs entire cell
app_prio50 = scheduler.Application('prio50', 50, [10, 10], 'app')
cell.partitions[None].allocation.add(app_prio50)
cell.schedule()
# The queue is ordered by priority:
# - prio50, app1, app2, app3, app4
#
# As placement not found for prio50, app4 will be evicted first.
#
# As result, prio50 will be placed on 'z', and app4 (evicted) will be
# placed on "next" server, which is 'a'.
self.assertEqual(app_prio50.server, srv4)
self.assertEqual(app4.server, srv1)
app_prio51 = scheduler.Application('prio51', 51, [10, 10], 'app')
cell.partitions[None].allocation.add(app_prio51)
cell.schedule()
# app4 is now colocated with app1. app4 will still be evicted first,
# then app3, at which point there will be enough capacity to place
# large app.
#
# app3 will be rescheduled to run on "next" server - 'y', and app4 will
# be restored to 'a'.
self.assertEqual(app_prio51.server, srv3)
self.assertEqual(app_prio50.server, srv4)
self.assertEqual(app4.server, srv1)
app_prio49_1 = scheduler.Application('prio49_1', 49, [10, 10], 'app')
app_prio49_2 = scheduler.Application('prio49_2', 49, [9, 9], 'app')
cell.partitions[None].allocation.add(app_prio49_1)
cell.partitions[None].allocation.add(app_prio49_2)
cell.schedule()
# 50/51 not moved. from the end of the queue,
self.assertEqual(app_prio51.server, srv3)
self.assertEqual(app_prio50.server, srv4)
self.assertEqual(set([app_prio49_1.server, app_prio49_2.server]),
set([srv1, srv2]))
# Only capacity left for small [1, 1] app.
self.assertIsNotNone(app1.server)
self.assertIsNone(app2.server)
self.assertIsNone(app3.server)
self.assertIsNone(app4.server)
def test_applications(self):
"""Tests application queue report."""
app1 = scheduler.Application('foo.xxx#1',
100,
demand=[1, 1, 1],
affinity='foo.xxx')
app1.global_order = 1
app2 = scheduler.Application('foo.xxx#2',
100,
demand=[1, 1, 1],
affinity='foo.xxx')
app2.global_order = 2
app3 = scheduler.Application('bla.xxx#3',
50,
demand=[1, 1, 1],
affinity='bla.xxx')
app3.global_order = 3
(self.cell.partitions['_default'].allocation.get_sub_alloc(
't1').get_sub_alloc('t3').get_sub_alloc('a2').add(app1))
(self.cell.partitions['_default'].allocation.get_sub_alloc(
't1').get_sub_alloc('t3').get_sub_alloc('a2').add(app2))
(self.cell.partitions['part'].allocation.get_sub_alloc(
't2').get_sub_alloc('a1').add(app3))
self.cell.schedule()
apps_df = reports.apps(self.cell)
pd.util.testing.assert_frame_equal(
apps_df,
pd.DataFrame(
[
[
'bla.xxx#3', 't2/a1', 100, 'bla.xxx', 'part', None, -1,
3, 0, 100, 0, 0, 'srv1', -0.142857142857, -0.128571, 1,
1, 1
],
[
'foo.xxx#1', 't1/t3/a2', 100, 'foo.xxx', '_default',
None, -1, 1, 0, 100, 0, 0, 'srv3', -0.142857142857,
-0.128571, 1, 1, 1
],
[
'foo.xxx#2', 't1/t3/a2', 100, 'foo.xxx', '_default',
None, -1, 2, 0, 100, 0, 0, 'srv4', -0.128571428571,
-0.114286, 1, 1, 1
],
],
columns=[
'instance', 'allocation', 'rank', 'affinity', 'partition',
'identity_group', 'identity', 'order', 'lease', 'expires',
'data_retention', 'pending', 'server', 'util0', 'util1',
'mem', 'cpu', 'disk'
]).sort_values(by=[
'partition', 'rank', 'util0', 'util1', 'pending', 'order'
]).reset_index(drop=True))
time.time.return_value = 100
self.assertEqual(apps_df.ix[2]['cpu'], 1)
util0 = reports.utilization(None, apps_df)
time.time.return_value = 101
util1 = reports.utilization(util0, apps_df)
# name bla.xxx foo.xxx \
# count util disk cpu memory count util disk
# 1969-12-31 19:00:00 1 -0.121429 1 1 1 2 -0.128571 2
# 1969-12-31 19:00:01 1 -0.121429 1 1 1 2 -0.128571 2
#
# name
# cpu memory
# 1969-12-31 19:00:00 2 2
# 1969-12-31 19:00:01 2 2
time0 = pd.Timestamp(datetime.datetime.fromtimestamp(100))
time1 = pd.Timestamp(datetime.datetime.fromtimestamp(101))
self.assertEqual(util1.ix[time0]['bla.xxx']['cpu'], 1)
self.assertEqual(util1.ix[time1]['foo.xxx']['count'], 2)
