def test_applications(self): """Tests application queue report.""" app1 = scheduler.Application('foo.xxx#1', 100, demand=[1, 1, 1], affinity='foo.xxx') app2 = scheduler.Application('foo.xxx#2', 100, demand=[1, 1, 1], affinity='foo.xxx') app3 = scheduler.Application('bla.xxx#3', 50, demand=[1, 1, 1], affinity='bla.xxx') (self.cell.partitions[None].allocation.get_sub_alloc( 't1').get_sub_alloc('t3').get_sub_alloc('a2').add(app1)) (self.cell.partitions[None].allocation.get_sub_alloc( 't1').get_sub_alloc('t3').get_sub_alloc('a2').add(app2)) (self.cell.partitions[None].allocation.get_sub_alloc( 't2').get_sub_alloc('a1').add(app3)) self.cell.schedule() apps_df = reports.apps(self.cell) # affinity allocation cpu data_retention_timeout disk memory \ # instance # foo.xxx#1 foo.xxx t1/t3/a2 1 0 1 1 # foo.xxx#2 foo.xxx t1/t3/a2 1 0 1 1 # bla.xxx#3 bla.xxx t2/a1 1 0 1 1 # # order pending rank server util # instance # foo.xxx#1 1.458152e+15 0 99 srv1 -0.135714 # foo.xxx#2 1.458152e+15 0 99 srv1 -0.128571 # bla.xxx#3 1.458152e+15 0 100 srv1 -0.121429 time.time.return_value = 100 self.assertEqual(apps_df.ix['foo.xxx#2']['cpu'], 1) util0 = reports.utilization(None, apps_df) time.time.return_value = 101 util1 = reports.utilization(util0, apps_df) # name bla.xxx foo.xxx \ # count util disk cpu memory count util disk # 1969-12-31 19:00:00 1 -0.121429 1 1 1 2 -0.128571 2 # 1969-12-31 19:00:01 1 -0.121429 1 1 1 2 -0.128571 2 # # name # cpu memory # 1969-12-31 19:00:00 2 2 # 1969-12-31 19:00:01 2 2 time0 = pd.Timestamp(datetime.datetime.fromtimestamp(100)) time1 = pd.Timestamp(datetime.datetime.fromtimestamp(101)) self.assertEqual(util1.ix[time0]['bla.xxx']['cpu'], 1) self.assertEqual(util1.ix[time1]['foo.xxx']['count'], 2)
def scheduler(view, reschedule, csv): """Manage Treadmill server configuration.""" treadmill_sched.DIMENSION_COUNT = 3 cell_master = master.Master(context.GLOBAL.zk.conn, context.GLOBAL.cell) cell_master.load_buckets() cell_master.load_cell() cell_master.load_servers(readonly=True) cell_master.load_allocations() cell_master.load_strategies() cell_master.load_apps() if reschedule: cell_master.cell.schedule() output = None if view == 'servers': output = reports.servers(cell_master.cell) if view == 'features': output = reports.node_features(cell_master.cell) if view == 'allocs': allocs = reports.allocations(cell_master.cell) features = reports.allocation_features(cell_master.cell) # TODO: investigate why pd.concat returns series not df. # pylint: disable=R0204 output = pd.concat([allocs, features], axis=1) if view == 'apps': output = reports.apps(cell_master.cell) if view == 'util': apps = reports.apps(cell_master.cell) output = reports.utilization(None, apps) if output is not None and len(output): if csv: print output.to_csv() else: print output
def apps(): """View apps report""" cell_master = make_readonly_master(ctx['run_scheduler']) output = reports.apps(cell_master.cell, cell_master.trait_codes) # Replace integer N/As for col in ['identity', 'expires', 'lease', 'data_retention']: output.loc[output[col] == -1, col] = '' # Convert to datetimes for col in ['expires']: output[col] = pd.to_datetime(output[col], unit='s') # Convert to timedeltas for col in ['lease', 'data_retention']: output[col] = pd.to_timedelta(output[col], unit='s') _print(output, apps_formatter)
def test_empty_cell_reports(self): """Tests all reports for an empty cell.""" empty_cell = _construct_cell(empty=True) servers = reports.servers(empty_cell, self.trait_codes) empty_servers = pd.DataFrame(columns=[ 'name', 'location', 'partition', 'traits', 'state', 'valid_until', 'mem', 'cpu', 'disk', 'mem_free', 'cpu_free', 'disk_free' ]).astype({ 'mem': 'int', 'cpu': 'int', 'disk': 'int', 'mem_free': 'int', 'cpu_free': 'int', 'disk_free': 'int' }).reset_index(drop=True) pd.util.testing.assert_frame_equal(servers, empty_servers) allocations = reports.allocations(empty_cell, self.trait_codes) empty_allocations = pd.DataFrame(columns=[ 'partition', 'name', 'mem', 'cpu', 'disk', 'rank', 'rank_adj', 'traits', 'max_util' ]).astype({ 'mem': 'int', 'cpu': 'int', 'disk': 'int' }).reset_index(drop=True) pd.util.testing.assert_frame_equal(allocations, empty_allocations) apps = reports.apps(empty_cell, self.trait_codes) empty_apps = pd.DataFrame(columns=[ 'instance', 'allocation', 'rank', 'affinity', 'partition', 'identity_group', 'identity', 'order', 'lease', 'expires', 'data_retention', 'pending', 'server', 'util0', 'util1', 'mem', 'cpu', 'disk' ]).astype({ 'mem': 'int', 'cpu': 'int', 'disk': 'int', 'order': 'int', 'expires': 'int', 'data_retention': 'int', 'identity': 'int' }).reset_index(drop=True) pd.util.testing.assert_frame_equal(apps, empty_apps)
def apps(full): """View apps report""" cell_master = make_readonly_master(ctx['run_scheduler']) output = reports.apps(cell_master.cell, cell_master.trait_codes) # Replace integer N/As for col in ['identity', 'expires', 'lease', 'data_retention']: output.loc[output[col] == -1, col] = '' # Convert to datetimes for col in ['expires']: output[col] = pd.to_datetime(output[col], unit='s') # Convert to timedeltas for col in ['lease', 'data_retention']: output[col] = pd.to_timedelta(output[col], unit='s') # Float precision for col in ['util0', 'util1']: output[col] = output[col].map(lambda x: '%0.4f' % x) if not full: output = output[[ 'instance', 'allocation', 'partition', 'server', 'mem', 'cpu', 'disk', 'traits' ]] _print(output)
def test_applications(self): """Tests application queue report.""" app1 = scheduler.Application('foo.xxx#1', 100, demand=[1, 1, 1], affinity='foo.xxx') app1.global_order = 1 app2 = scheduler.Application('foo.xxx#2', 100, demand=[1, 1, 1], affinity='foo.xxx') app2.global_order = 2 app3 = scheduler.Application('bla.xxx#3', 50, demand=[1, 1, 1], affinity='bla.xxx') app3.global_order = 3 (self.cell.partitions['_default'].allocation.get_sub_alloc( 't1').get_sub_alloc('t3').get_sub_alloc('a2').add(app1)) (self.cell.partitions['_default'].allocation.get_sub_alloc( 't1').get_sub_alloc('t3').get_sub_alloc('a2').add(app2)) (self.cell.partitions['part'].allocation.get_sub_alloc( 't2').get_sub_alloc('a1').add(app3)) self.cell.schedule() apps_df = reports.apps(self.cell) pd.util.testing.assert_frame_equal( apps_df, pd.DataFrame( [ [ 'bla.xxx#3', 't2/a1', 100, 'bla.xxx', 'part', None, -1, 3, 0, 100, 0, 0, 'srv1', -0.142857142857, -0.128571, 1, 1, 1 ], [ 'foo.xxx#1', 't1/t3/a2', 100, 'foo.xxx', '_default', None, -1, 1, 0, 100, 0, 0, 'srv3', -0.142857142857, -0.128571, 1, 1, 1 ], [ 'foo.xxx#2', 't1/t3/a2', 100, 'foo.xxx', '_default', None, -1, 2, 0, 100, 0, 0, 'srv4', -0.128571428571, -0.114286, 1, 1, 1 ], ], columns=[ 'instance', 'allocation', 'rank', 'affinity', 'partition', 'identity_group', 'identity', 'order', 'lease', 'expires', 'data_retention', 'pending', 'server', 'util0', 'util1', 'mem', 'cpu', 'disk' ]).sort_values(by=[ 'partition', 'rank', 'util0', 'util1', 'pending', 'order' ]).reset_index(drop=True)) time.time.return_value = 100 self.assertEqual(apps_df.ix[2]['cpu'], 1) util0 = reports.utilization(None, apps_df) time.time.return_value = 101 util1 = reports.utilization(util0, apps_df) # name bla.xxx foo.xxx \ # count util disk cpu memory count util disk # 1969-12-31 19:00:00 1 -0.121429 1 1 1 2 -0.128571 2 # 1969-12-31 19:00:01 1 -0.121429 1 1 1 2 -0.128571 2 # # name # cpu memory # 1969-12-31 19:00:00 2 2 # 1969-12-31 19:00:01 2 2 time0 = pd.Timestamp(datetime.datetime.fromtimestamp(100)) time1 = pd.Timestamp(datetime.datetime.fromtimestamp(101)) self.assertEqual(util1.ix[time0]['bla.xxx']['cpu'], 1) self.assertEqual(util1.ix[time1]['foo.xxx']['count'], 2)
def queue(): """View utilization queue""" cell_master = _load() apps = reports.apps(cell_master.cell) output = reports.utilization(None, apps) _print_frame(output)
def apps(): """View apps report""" cell_master = _load() output = reports.apps(cell_master.cell) _print_frame(output)