def test_get_resource_utilization_by_grouping_correctly_multi_groups():
fake_state = {
"slaves": [
{
"id": "foo1",
"resources": {"disk": 100, "cpus": 10, "mem": 50},
"attributes": {"one": "yes", "two": "yes"},
"reserved_resources": {},
},
{
"id": "bar1",
"resources": {"disk": 100, "cpus": 10, "mem": 50},
"attributes": {"one": "yes", "two": "no"},
"reserved_resources": {},
},
{
"id": "foo2",
"resources": {"disk": 100, "cpus": 10, "mem": 50},
"attributes": {"one": "no", "two": "yes"},
"reserved_resources": {},
},
{
"id": "bar2",
"resources": {"disk": 100, "cpus": 10, "mem": 50},
"attributes": {"one": "no", "two": "no"},
"reserved_resources": {},
},
],
"frameworks": [
{
"tasks": [
{
"state": "TASK_RUNNING",
"resources": {"cpus": 1, "mem": 10, "disk": 10},
"slave_id": "foo1",
},
{
"state": "TASK_RUNNING",
"resources": {"cpus": 1, "mem": 10, "disk": 10},
"slave_id": "bar1",
},
]
}
],
}
grouping_func = metastatus_lib.key_func_for_attribute_multi(["one", "two"])
resp = metastatus_lib.get_resource_utilization_by_grouping(
mesos_state=fake_state, grouping_func=grouping_func
)
# resp should have 4 keys...
assert len(resp.keys()) == 4
# Each key should be a set with 2 items...
assert len(list(resp.keys())[0]) == 2
# Each item in the set should have 2 values (original key, value)
assert len(list(list(resp.keys())[0])[0]) == 2
def utilization_table_by_grouping_from_mesos_state(
groupings: Sequence[str],
threshold: float,
humanize: bool,
mesos_state: Dict,
) -> Tuple[
List[List[str]],
bool,
]:
grouping_function = metastatus_lib.key_func_for_attribute_multi(groupings)
resource_info_dict_grouped = metastatus_lib.get_resource_utilization_by_grouping(
grouping_function,
mesos_state,
)
static_headers = [
'CPU (used/total)',
'RAM (used/total)',
'Disk (used/total)',
'GPU (used/total)',
'Agent count',
]
all_rows = [
[grouping.capitalize() for grouping in groupings] + static_headers,
]
table_rows = []
for grouping_values, resource_info_dict in resource_info_dict_grouped.items():
resource_utilizations = metastatus_lib.resource_utillizations_from_resource_info(
total=resource_info_dict['total'],
free=resource_info_dict['free'],
)
healthcheck_utilization_pairs = [
metastatus_lib.healthcheck_result_resource_utilization_pair_for_resource_utilization(
utilization,
threshold,
)
for utilization in resource_utilizations
]
healthy_exit = all(pair[0].healthy for pair in healthcheck_utilization_pairs)
table_rows.append(metastatus_lib.get_table_rows_for_resource_info_dict(
[v for g, v in grouping_values],
healthcheck_utilization_pairs,
humanize,
) + [str(resource_info_dict['slave_count'])])
table_rows = sorted(table_rows, key=lambda x: x[0:len(groupings)])
all_rows.extend(table_rows)
return all_rows, healthy_exit
def utilization_table_by_grouping_from_mesos_state(
groupings: Sequence[str],
threshold: float,
mesos_state: MesosState,
service_instance_stats: Optional[ServiceInstanceStats] = None,
) -> Tuple[Sequence[MutableSequence[str]], bool]:
grouping_function = metastatus_lib.key_func_for_attribute_multi(groupings)
resource_info_dict_grouped = metastatus_lib.get_resource_utilization_by_grouping(
grouping_function, mesos_state)
return utilization_table_by_grouping(
groupings,
grouping_function,
resource_info_dict_grouped,
threshold,
service_instance_stats,
)
def resources_utilization(request):
master = get_mesos_master()
mesos_state = block(master.state)
groupings = request.swagger_data.get('groupings', ['superregion'])
# swagger actually makes the key None if it's not set
if groupings is None:
groupings = ['superregion']
grouping_function = metastatus_lib.key_func_for_attribute_multi(groupings)
sorting_function = metastatus_lib.sort_func_for_attributes(groupings)
filters = request.swagger_data.get('filter', [])
filters = parse_filters(filters)
filter_funcs = [
metastatus_lib.make_filter_slave_func(attr, vals)
for attr, vals in filters.items()
]
resource_info_dict = metastatus_lib.get_resource_utilization_by_grouping(
grouping_func=grouping_function,
mesos_state=mesos_state,
filters=filter_funcs,
sort_func=sorting_function,
)
response_body = []
for k, v in resource_info_dict.items():
group = {'groupings': {}}
for grouping, value in k:
group['groupings'][grouping] = value
for resource, value in v['total']._asdict().items():
group[resource] = {'total': value}
for resource, value in v['free']._asdict().items():
group[resource]['free'] = value
for resource in v['free']._fields:
group[resource][
'used'] = group[resource]['total'] - group[resource]['free']
response_body.append(group)
return Response(json_body=response_body, status_code=200)
def test_get_resource_utilization_by_grouping_correctly_multi_groups():
fake_state = {
'slaves': [
{
'id': 'foo1',
'resources': {
'disk': 100,
'cpus': 10,
'mem': 50,
},
'attributes': {'one': 'yes', 'two': 'yes'},
'reserved_resources': {},
},
{
'id': 'bar1',
'resources': {
'disk': 100,
'cpus': 10,
'mem': 50,
},
'attributes': {'one': 'yes', 'two': 'no'},
'reserved_resources': {},
},
{
'id': 'foo2',
'resources': {
'disk': 100,
'cpus': 10,
'mem': 50,
},
'attributes': {'one': 'no', 'two': 'yes'},
'reserved_resources': {},
},
{
'id': 'bar2',
'resources': {
'disk': 100,
'cpus': 10,
'mem': 50,
},
'attributes': {'one': 'no', 'two': 'no'},
'reserved_resources': {},
},
],
'frameworks': [
{'tasks': [
{
'state': 'TASK_RUNNING',
'resources': {'cpus': 1, 'mem': 10, 'disk': 10},
'slave_id': 'foo1',
},
{
'state': 'TASK_RUNNING',
'resources': {'cpus': 1, 'mem': 10, 'disk': 10},
'slave_id': 'bar1',
},
]},
],
}
grouping_func = metastatus_lib.key_func_for_attribute_multi(['one', 'two'])
resp = metastatus_lib.get_resource_utilization_by_grouping(
mesos_state=fake_state,
grouping_func=grouping_func,
)
# resp should have 4 keys...
assert(len(resp.keys()) == 4)
# Each key should be a set with 2 items...
assert(len(list(resp.keys())[0]) == 2)
# Each item in the set should have 2 values (original key, value)
assert(len(list(list(resp.keys())[0])[0]) == 2)
