def test_applications(self):
"""Tests application queue report."""
app1 = scheduler.Application('foo.xxx#1',
100,
demand=[1, 1, 1],
affinity='foo.xxx')
app2 = scheduler.Application('foo.xxx#2',
100,
demand=[1, 1, 1],
affinity='foo.xxx')
app3 = scheduler.Application('bla.xxx#3',
50,
demand=[1, 1, 1],
affinity='bla.xxx')
(self.cell.partitions[None].allocation.get_sub_alloc(
't1').get_sub_alloc('t3').get_sub_alloc('a2').add(app1))
(self.cell.partitions[None].allocation.get_sub_alloc(
't1').get_sub_alloc('t3').get_sub_alloc('a2').add(app2))
(self.cell.partitions[None].allocation.get_sub_alloc(
't2').get_sub_alloc('a1').add(app3))
self.cell.schedule()
apps_df = reports.apps(self.cell)
# affinity allocation cpu data_retention_timeout disk memory \
# instance
# foo.xxx#1 foo.xxx t1/t3/a2 1 0 1 1
# foo.xxx#2 foo.xxx t1/t3/a2 1 0 1 1
# bla.xxx#3 bla.xxx t2/a1 1 0 1 1
#
# order pending rank server util
# instance
# foo.xxx#1 1.458152e+15 0 99 srv1 -0.135714
# foo.xxx#2 1.458152e+15 0 99 srv1 -0.128571
# bla.xxx#3 1.458152e+15 0 100 srv1 -0.121429
time.time.return_value = 100
self.assertEqual(apps_df.ix['foo.xxx#2']['cpu'], 1)
util0 = reports.utilization(None, apps_df)
time.time.return_value = 101
util1 = reports.utilization(util0, apps_df)
# name bla.xxx foo.xxx \
# count util disk cpu memory count util disk
# 1969-12-31 19:00:00 1 -0.121429 1 1 1 2 -0.128571 2
# 1969-12-31 19:00:01 1 -0.121429 1 1 1 2 -0.128571 2
#
# name
# cpu memory
# 1969-12-31 19:00:00 2 2
# 1969-12-31 19:00:01 2 2
time0 = pd.Timestamp(datetime.datetime.fromtimestamp(100))
time1 = pd.Timestamp(datetime.datetime.fromtimestamp(101))
self.assertEqual(util1.ix[time0]['bla.xxx']['cpu'], 1)
self.assertEqual(util1.ix[time1]['foo.xxx']['count'], 2)
def scheduler(view, reschedule, csv):
"""Manage Treadmill server configuration."""
treadmill_sched.DIMENSION_COUNT = 3
cell_master = master.Master(context.GLOBAL.zk.conn,
context.GLOBAL.cell)
cell_master.load_buckets()
cell_master.load_cell()
cell_master.load_servers(readonly=True)
cell_master.load_allocations()
cell_master.load_strategies()
cell_master.load_apps()
if reschedule:
cell_master.cell.schedule()
output = None
if view == 'servers':
output = reports.servers(cell_master.cell)
if view == 'features':
output = reports.node_features(cell_master.cell)
if view == 'allocs':
allocs = reports.allocations(cell_master.cell)
features = reports.allocation_features(cell_master.cell)
# TODO: investigate why pd.concat returns series not df.
# pylint: disable=R0204
output = pd.concat([allocs, features], axis=1)
if view == 'apps':
output = reports.apps(cell_master.cell)
if view == 'util':
apps = reports.apps(cell_master.cell)
output = reports.utilization(None, apps)
if output is not None and len(output):
if csv:
print output.to_csv()
else:
print output
def apps():
"""View apps report"""
cell_master = make_readonly_master(ctx['run_scheduler'])
output = reports.apps(cell_master.cell, cell_master.trait_codes)
# Replace integer N/As
for col in ['identity', 'expires', 'lease', 'data_retention']:
output.loc[output[col] == -1, col] = ''
# Convert to datetimes
for col in ['expires']:
output[col] = pd.to_datetime(output[col], unit='s')
# Convert to timedeltas
for col in ['lease', 'data_retention']:
output[col] = pd.to_timedelta(output[col], unit='s')
_print(output, apps_formatter)
def test_empty_cell_reports(self):
"""Tests all reports for an empty cell."""
empty_cell = _construct_cell(empty=True)
servers = reports.servers(empty_cell, self.trait_codes)
empty_servers = pd.DataFrame(columns=[
'name', 'location', 'partition', 'traits', 'state', 'valid_until',
'mem', 'cpu', 'disk', 'mem_free', 'cpu_free', 'disk_free'
]).astype({
'mem': 'int',
'cpu': 'int',
'disk': 'int',
'mem_free': 'int',
'cpu_free': 'int',
'disk_free': 'int'
}).reset_index(drop=True)
pd.util.testing.assert_frame_equal(servers, empty_servers)
allocations = reports.allocations(empty_cell, self.trait_codes)
empty_allocations = pd.DataFrame(columns=[
'partition', 'name', 'mem', 'cpu', 'disk', 'rank', 'rank_adj',
'traits', 'max_util'
]).astype({
'mem': 'int',
'cpu': 'int',
'disk': 'int'
}).reset_index(drop=True)
pd.util.testing.assert_frame_equal(allocations, empty_allocations)
apps = reports.apps(empty_cell, self.trait_codes)
empty_apps = pd.DataFrame(columns=[
'instance', 'allocation', 'rank', 'affinity', 'partition',
'identity_group', 'identity', 'order', 'lease', 'expires',
'data_retention', 'pending', 'server', 'util0', 'util1', 'mem',
'cpu', 'disk'
]).astype({
'mem': 'int',
'cpu': 'int',
'disk': 'int',
'order': 'int',
'expires': 'int',
'data_retention': 'int',
'identity': 'int'
}).reset_index(drop=True)
pd.util.testing.assert_frame_equal(apps, empty_apps)
def apps(full):
"""View apps report"""
cell_master = make_readonly_master(ctx['run_scheduler'])
output = reports.apps(cell_master.cell, cell_master.trait_codes)
# Replace integer N/As
for col in ['identity', 'expires', 'lease', 'data_retention']:
output.loc[output[col] == -1, col] = ''
# Convert to datetimes
for col in ['expires']:
output[col] = pd.to_datetime(output[col], unit='s')
# Convert to timedeltas
for col in ['lease', 'data_retention']:
output[col] = pd.to_timedelta(output[col], unit='s')
# Float precision
for col in ['util0', 'util1']:
output[col] = output[col].map(lambda x: '%0.4f' % x)
if not full:
output = output[[
'instance', 'allocation', 'partition', 'server', 'mem', 'cpu',
'disk', 'traits'
]]
_print(output)
def test_applications(self):
"""Tests application queue report."""
app1 = scheduler.Application('foo.xxx#1',
100,
demand=[1, 1, 1],
affinity='foo.xxx')
app1.global_order = 1
app2 = scheduler.Application('foo.xxx#2',
100,
demand=[1, 1, 1],
affinity='foo.xxx')
app2.global_order = 2
app3 = scheduler.Application('bla.xxx#3',
50,
demand=[1, 1, 1],
affinity='bla.xxx')
app3.global_order = 3
(self.cell.partitions['_default'].allocation.get_sub_alloc(
't1').get_sub_alloc('t3').get_sub_alloc('a2').add(app1))
(self.cell.partitions['_default'].allocation.get_sub_alloc(
't1').get_sub_alloc('t3').get_sub_alloc('a2').add(app2))
(self.cell.partitions['part'].allocation.get_sub_alloc(
't2').get_sub_alloc('a1').add(app3))
self.cell.schedule()
apps_df = reports.apps(self.cell)
pd.util.testing.assert_frame_equal(
apps_df,
pd.DataFrame(
[
[
'bla.xxx#3', 't2/a1', 100, 'bla.xxx', 'part', None, -1,
3, 0, 100, 0, 0, 'srv1', -0.142857142857, -0.128571, 1,
1, 1
],
[
'foo.xxx#1', 't1/t3/a2', 100, 'foo.xxx', '_default',
None, -1, 1, 0, 100, 0, 0, 'srv3', -0.142857142857,
-0.128571, 1, 1, 1
],
[
'foo.xxx#2', 't1/t3/a2', 100, 'foo.xxx', '_default',
None, -1, 2, 0, 100, 0, 0, 'srv4', -0.128571428571,
-0.114286, 1, 1, 1
],
],
columns=[
'instance', 'allocation', 'rank', 'affinity', 'partition',
'identity_group', 'identity', 'order', 'lease', 'expires',
'data_retention', 'pending', 'server', 'util0', 'util1',
'mem', 'cpu', 'disk'
]).sort_values(by=[
'partition', 'rank', 'util0', 'util1', 'pending', 'order'
]).reset_index(drop=True))
time.time.return_value = 100
self.assertEqual(apps_df.ix[2]['cpu'], 1)
util0 = reports.utilization(None, apps_df)
time.time.return_value = 101
util1 = reports.utilization(util0, apps_df)
# name bla.xxx foo.xxx \
# count util disk cpu memory count util disk
# 1969-12-31 19:00:00 1 -0.121429 1 1 1 2 -0.128571 2
# 1969-12-31 19:00:01 1 -0.121429 1 1 1 2 -0.128571 2
#
# name
# cpu memory
# 1969-12-31 19:00:00 2 2
# 1969-12-31 19:00:01 2 2
time0 = pd.Timestamp(datetime.datetime.fromtimestamp(100))
time1 = pd.Timestamp(datetime.datetime.fromtimestamp(101))
self.assertEqual(util1.ix[time0]['bla.xxx']['cpu'], 1)
self.assertEqual(util1.ix[time1]['foo.xxx']['count'], 2)
def queue():
"""View utilization queue"""
cell_master = _load()
apps = reports.apps(cell_master.cell)
output = reports.utilization(None, apps)
_print_frame(output)
def apps():
"""View apps report"""
cell_master = _load()
output = reports.apps(cell_master.cell)
_print_frame(output)
