class ApplicationTests(unittest.TestCase):
def setUp(self) -> None:
self.app = Application("sample_app")
# client --> comp1 --> comp2 --> comp3
self.client = Component(self.app, "client", "client",
ComponentType.UNMANAGED)
self.comp1 = Component(self.app, "comp1", "comp1",
ComponentType.MANAGED)
self.comp2 = Component(self.app, "comp2", "comp2",
ComponentType.MANAGED)
self.comp3 = Component(self.app, "comp3", "comp3",
ComponentType.MANAGED)
self.client.add_dependency(self.comp1)
self.comp1.add_dependency(self.comp2)
self.comp2.add_dependency(self.comp3)
self.managed_components = [self.comp1, self.comp2, self.comp3]
self.unmanaged_components = [self.client]
self.app.add_components(
[self.client, self.comp1, self.comp2, self.comp3])
def test_list_managed_components(self):
for managed_component in self.app.list_managed_components():
self.assertTrue(managed_component in self.managed_components)
def test_list_unmanaged_components(self):
for unmanaged_component in self.app.list_unmanaged_components():
self.assertTrue(unmanaged_component in self.unmanaged_components)
def setUp(self) -> None:
self.app = Application("sample_app")
# client --> comp1 --> comp1-1 --> comp1-1-1
# \
# -> comp1-2
self.client = Component(self.app, "client", "client",
ComponentType.UNMANAGED)
self.comp1 = Component(self.app, "comp1", "comp1",
ComponentType.MANAGED)
self.comp1_1 = Component(self.app, "comp1-1", "comp1-1",
ComponentType.MANAGED)
self.comp1_2 = Component(self.app, "comp1-2", "comp1-2",
ComponentType.MANAGED)
self.comp1_1_1 = Component(self.app, "comp1-1-1", "comp1-1-1",
ComponentType.MANAGED)
self._managed_components = [
self.comp1, self.comp1_1, self.comp1_2, self.comp1_1_1
]
self.client.add_dependency(self.comp1)
self.comp1.add_dependency(self.comp1_1)
self.comp1.add_dependency(self.comp1_2)
self.comp1_1.add_dependency(self.comp1_1_1)
# Dependencies for client.
self._dependencies_in_levels = [[self.comp1],
[self.comp1_1, self.comp1_2],
[self.comp1_1_1]]
self._dependency_chains = [[self.comp1, self.comp1_1, self.comp1_1_1],
[self.comp1, self.comp1_2]]
self.app.add_components([self.client] + self._managed_components)
class ComponentTests(unittest.TestCase):
def setUp(self) -> None:
self.app = Application("sample_app")
# client --> comp1 --> comp1-1 --> comp1-1-1
# \
# -> comp1-2
self.client = Component(self.app, "client", "client",
ComponentType.UNMANAGED)
self.comp1 = Component(self.app, "comp1", "comp1",
ComponentType.MANAGED)
self.comp1_1 = Component(self.app, "comp1-1", "comp1-1",
ComponentType.MANAGED)
self.comp1_2 = Component(self.app, "comp1-2", "comp1-2",
ComponentType.MANAGED)
self.comp1_1_1 = Component(self.app, "comp1-1-1", "comp1-1-1",
ComponentType.MANAGED)
self._managed_components = [
self.comp1, self.comp1_1, self.comp1_2, self.comp1_1_1
]
self.client.add_dependency(self.comp1)
self.comp1.add_dependency(self.comp1_1)
self.comp1.add_dependency(self.comp1_2)
self.comp1_1.add_dependency(self.comp1_1_1)
# Dependencies for client.
self._dependencies_in_levels = [[self.comp1],
[self.comp1_1, self.comp1_2],
[self.comp1_1_1]]
self._dependency_chains = [[self.comp1, self.comp1_1, self.comp1_1_1],
[self.comp1, self.comp1_2]]
self.app.add_components([self.client] + self._managed_components)
def test_list_managed_components(self):
managed_comps = list(self.app.list_managed_components())
self.assertEqual(len(managed_comps), len(self._managed_components))
found_ids = (component.id for component in managed_comps)
for expected_id in (component.id
for component in self._managed_components):
self.assertIn(expected_id, found_ids)
def test_get_all_dependencies_in_levels(self):
deps_in_levels = self.client.get_all_dependencies_in_levels()
expected_deps_in_levels = self._dependencies_in_levels
for level, expected_level in zip(deps_in_levels,
expected_deps_in_levels):
for component, expected_component in zip(level, expected_level):
self.assertEqual(component, expected_component)
def test_get_dependency_chains(self):
dep_chains = self.client.get_dependency_chains()
expected_dep_chains = self._dependency_chains
for chain, expected_chain in zip(dep_chains, expected_dep_chains):
for component, expected_component in zip(chain, expected_chain):
self.assertEqual(component, expected_component)
def _create_second_application(self):
self.second_app = Application("second_application")
self.second_component = Component(self.second_app, "second_component",
"second_component",
ComponentType.MANAGED)
self.second_app.add_component(self.second_component)
self.second_compin = ManagedCompin(self.second_component,
id_="second_compin",
node="node",
chain_id="2")
def __init__(self, app: Application):
self._app_pb = app.get_pb_representation()
self._app_name = app.name
super(AddApplicationToCCTask,
self).__init__(task_id=self.generate_id())
self.add_precondition(namespace_active, (self._app_name, ))
self.add_precondition(application_deployed, (self._app_name, ))
def register_app(self, app: Application) -> None:
logger.info("App %s registered" % app.name)
for component in app.list_managed_components():
for probe in component.probes:
with self._lock:
self._plan.append(
Scenario(controlled_probe=probe,
background_probes=[],
hw_id=self._default_hw_id))
def JudgeApp(self, request, context):
"""
Decides whether the specified application can be accepted (i.e. whether its QoS requirements
are realistic) based on the available measurement data.
"""
app = Application.init_from_pb(request)
# The application has to be already registered before, otherwise we cannot judge it
if not app.name in self.applications:
return predictor_pb.JudgeReply(
result=predictor_pb.JudgeResult.Value("NEEDS_DATA"))
with self._lock:
# All isolation measurements need to be finished before we can judge the app.
# if self._measuring_phases[app.name] == MeasuringPhase.ISOLATION:
# Check every QoS requirement one-by-one:
for component in app.components.values():
for probe in component.probes:
measurement_name = \
MeasurementAggregator.compose_measurement_name(DEFAULT_HARDWARE_ID, [probe.alias])
if not self._single_process_predictor.has_measurement(
measurement_name):
return predictor_pb.JudgeReply(
result=predictor_pb.JudgeResult.Value(
"NEEDS_DATA"))
for requirement in probe.requirements:
prediction = False
if isinstance(requirement, TimeContract):
prediction = self._single_process_predictor.predict_time(
probe_name=measurement_name,
time_limit=requirement.time,
percentile=requirement.percentile)
elif isinstance(requirement, ThroughputContract):
prediction = self._single_process_predictor.predict_throughput(
probe_name=measurement_name,
max_mean_time=requirement.mean_request_time)
if not prediction:
return predictor_pb.JudgeReply(
result=predictor_pb.JudgeResult.Value(
"REJECTED"))
if not app.is_complete:
return predictor_pb.JudgeReply(
result=predictor_pb.JudgeResult.Value("MEASURED"))
# Application is accepted now
# However, some QoS requirements may have been added between app registration and app evaluation.
# Thus, we re-register all the probes to include these requirements
self.applications[app.name] = app
for component in app.components.values():
self._probes_by_component[probe.component.id] = set()
for probe in component.probes:
assert probe.alias in self._probes_by_id
self._register_probe(probe)
self._probes_by_component[probe.component.id].add(
probe.alias)
return predictor_pb.JudgeReply(
result=predictor_pb.JudgeResult.Value("ACCEPTED"))
def _create_first_application(self):
self.app = Application("sample_application")
# component1 --> component2
self.component1 = Component(self.app, "component1", "component1",
ComponentType.MANAGED)
self.component2 = Component(self.app, "component2", "component2",
ComponentType.MANAGED)
self.component1.add_dependency(self.component2)
self.app.add_components([self.component1, self.component2])
# Note there is no dependency between compins on purpose.
self.compin1 = ManagedCompin(self.component1,
id_="compin1",
node="node",
chain_id="1")
self.compin2 = ManagedCompin(self.component2,
id_="compin2",
node="node",
chain_id="1")
def setUp(self) -> None:
self.app = Application("sample_app")
# client --> comp1 --> comp2 --> comp3
self.client = Component(self.app, "client", "client",
ComponentType.UNMANAGED)
self.comp1 = Component(self.app, "comp1", "comp1",
ComponentType.MANAGED)
self.comp2 = Component(self.app, "comp2", "comp2",
ComponentType.MANAGED)
self.comp3 = Component(self.app, "comp3", "comp3",
ComponentType.MANAGED)
self.client.add_dependency(self.comp1)
self.comp1.add_dependency(self.comp2)
self.comp2.add_dependency(self.comp3)
self.managed_components = [self.comp1, self.comp2, self.comp3]
self.unmanaged_components = [self.client]
self.app.add_components(
[self.client, self.comp1, self.comp2, self.comp3])
def add_application(self, app_pb) -> None:
"""
Adds an application to the Knowledge. Converts the protobuf representation into the object representation. Also,
processes the application's docker secret.
:param app_pb: protobuf representation of the application architecture.
"""
app = Application.init_from_pb(app_pb)
self.applications[app_pb.name] = app
for component in self.applications[app_pb.name].components.values():
self.components[component.full_name] = component
if app_pb.HasField("secret"):
self._add_secret(app_pb.name, app_pb.secret.value)
logging.info("An architecture for the %s application was processed." % app_pb.name)
def add_new_application(self, application_pb):
"""
Adds support for a new application.
:param application_pb: Protobuf representation of the application
"""
app = Application.init_from_pb(application_pb)
app_name = application_pb.name
self.clients[app_name] = {}
self.virtual_clients[app_name] = {}
for component in app.components.values():
if component.type == ComponentType.UNMANAGED:
self.virtual_clients[app_name][component.name] = []
self.check_threshold(app_name, component.name)
self.applications[app_name] = application_pb
def register_app(self, app: Application) -> None:
with self._lock:
for ctl_component in app.list_managed_components():
for ctl_probe in ctl_component.probes:
# Test all components alone
self._probes.append(ctl_probe)
for hw_config in self._hw_ids:
scenario = Scenario(ctl_probe, [], hw_config)
self._plan.append(scenario)
# Test all pairs of components
for background_probe in self._probes:
for hw_config in self._hw_ids:
scenario = Scenario(ctl_probe, [background_probe],
hw_config)
self._plan.append(scenario)
def RegisterApp(self, request, context):
"""
Registers the specified application architecture with the performance data aggregator.
"""
app = Application.init_from_pb(request)
with self._lock:
self.applications[app.name] = app
for component in app.components.values():
for probe in component.probes:
self._register_probe(probe)
self._scenario_generator.register_probe(probe)
if probe.signal_set != "":
if self._single_process_predictor.has_measurement(
MeasurementAggregator.compose_measurement_name(
DEFAULT_HARDWARE_ID, [probe.alias])):
self._single_process_predictor.report_measurements(
probe.alias, probe.signal_set,
probe.execution_time_signal,
probe.run_count_signal)
return predictor_pb.RegistrationAck()
def _cc_operations_required(self, app: Application) -> bool:
return self.knowledge.client_support and len(list(app.list_unmanaged_components())) > 0
class CloudStateTests(unittest.TestCase):
def setUp(self) -> None:
self._create_first_application()
self._create_second_application()
self.all_compins = [self.compin1, self.compin2, self.second_compin]
self._create_cloud_state(self.all_compins)
def _create_first_application(self):
self.app = Application("sample_application")
# component1 --> component2
self.component1 = Component(self.app, "component1", "component1",
ComponentType.MANAGED)
self.component2 = Component(self.app, "component2", "component2",
ComponentType.MANAGED)
self.component1.add_dependency(self.component2)
self.app.add_components([self.component1, self.component2])
# Note there is no dependency between compins on purpose.
self.compin1 = ManagedCompin(self.component1,
id_="compin1",
node="node",
chain_id="1")
self.compin2 = ManagedCompin(self.component2,
id_="compin2",
node="node",
chain_id="1")
def _create_second_application(self):
self.second_app = Application("second_application")
self.second_component = Component(self.second_app, "second_component",
"second_component",
ComponentType.MANAGED)
self.second_app.add_component(self.second_component)
self.second_compin = ManagedCompin(self.second_component,
id_="second_compin",
node="node",
chain_id="2")
def _create_cloud_state(self, all_compins):
self.cloud_state = CloudState()
self.cloud_state.add_application(self.app)
self.cloud_state.add_application(self.second_app)
self.cloud_state.add_instances(all_compins)
def test_get_compin(self):
compin = self.cloud_state.get_compin(self.app.name,
self.component1.name,
self.compin1.id)
self.assertEqual(compin, self.compin1)
compin2 = self.cloud_state.get_compin(self.app.name,
self.component2.name,
self.compin2.id)
self.assertEqual(compin2, self.compin2)
no_compin = self.cloud_state.get_compin("None-existing-application",
"non-existing-component",
"compin-id")
self.assertIsNone(no_compin)
def test_set_dependency(self):
self.cloud_state.set_dependency(self.app.name,
self.component1.name,
id_=self.compin1.id,
dependency=self.component2.name,
dependency_id=self.compin2.id)
self.assertIsNotNone(self.compin1.get_dependency(self.component2.name))
def test_list_all_compins(self):
for managed_compin in self.cloud_state.list_all_managed_compins():
self.assertTrue(managed_compin in self.all_compins)
self.assertEqual(
len(list(self.cloud_state.list_all_unmanaged_compins())), 0)
def __init__(self, architecture: arch_pb.Architecture):
self.application: Application = Application.init_from_pb(architecture)
self._name: str = architecture.name
self.status = AppStatus.RECEIVED