def assign_services_to_workers(self, composition):
"""
Assign the composition of services to the swarm workers
:param composition: Service composition
:return: bool
"""
llogger = local_logger.LocalLogger()
llogger.log_method_call(self.__class__.__name__,
sys._getframe().f_code.co_name)
open_allocations = composition.get_open_allocations()
llogger.debug("Open service allocations: %s", len(open_allocations))
service_allocation_dict = self.allocate_services_to_workers(
composition, open_allocations)
if service_allocation_dict is not None:
for worker, service_list in list(service_allocation_dict.items()):
composition.assign_worker_to_services(service_list, worker)
for service in service_list:
evaluation_logger.EvaluationLogger().write(
[service, worker.hostname],
evaluation_logger.LogType.ALLOCATIONS)
llogger.debug(composition._allocation)
return True
else:
llogger.debug(composition._allocation)
return False
def start_swarm_by_composition(self, composition, swarm_uuid):
"""
Start swarm by parsed service composition
:param composition: Object of service composition
:param swarm_uuid: UUID of the swarm
:raises SwarmException
"""
llogger = local_logger.LocalLogger()
llogger.log_method_call(self.__class__.__name__,
sys._getframe().f_code.co_name)
start_timer = datetime.datetime.now()
if self.swarm is None:
raise SwarmException("Swarm not initialized")
llogger.debug("Try to match swarm: %s with the following composition",
swarm_uuid)
llogger.debug("\n" + composition.format_service_composition_as_table())
result = self.assign_services_to_workers(composition)
if result is True:
llogger.debug("Worker matched. Try to start network")
network = self.create_docker_network()
llogger.debug(composition._allocation)
self.start_services_on_workers(composition, network)
total_time_of_run = datetime.datetime.now() - start_timer
evaluation_logger.EvaluationLogger().write([
"Total Time of Run", "*", "*",
total_time_of_run.total_seconds(),
self.swarm.get_worker_count(),
len(composition.get_open_allocations())
], evaluation_logger.LogType.ALLOC_METRICS)
llogger.debug("Time elapsed for Total Run: %f",
total_time_of_run.total_seconds())
def calculate_costs(self, worker, service):
"""
Implemented cost calculation method
:param service: Related service object
:param worker: Related worker object
:return:
"""
llogger = local_logger.LocalLogger()
llogger.log_method_call(self.__class__.__name__,
sys._getframe().f_code.co_name)
cpu_usage_of_worker, vram_usage_of_worker, swap_usage_of_worker, size_of_docker_image, bandwidth_in_percent = \
self.get_usage_of_worker(worker, service)
cpu_costs, vram_costs, swap_costs, image_download_costs = self.calculate_partial_costs(
cpu_usage_of_worker, vram_usage_of_worker, swap_usage_of_worker,
size_of_docker_image, bandwidth_in_percent)
overall_costs = self.calculate_overall_costs(cpu_costs, vram_costs,
swap_costs,
image_download_costs)
evaluation_logger.EvaluationLogger().write([
service.id, worker.hostname, cpu_usage_of_worker,
vram_usage_of_worker, swap_usage_of_worker, bandwidth_in_percent,
cpu_costs * self.cpu_cost_weight, vram_costs *
self.vram_cost_weight, swap_costs * self.swap_cost_weight,
image_download_costs * self.image_download_cost_weight,
overall_costs
], evaluation_logger.LogType.COSTS)
return int(overall_costs)
def reset_evaluation_logger(self):
"""
Resets the time of the evaluation logger to let it use a new file
:return:
"""
llogger = local_logger.LocalLogger()
llogger.log_method_call(self.__class__.__name__,
sys._getframe().f_code.co_name)
evaluation_logger.EvaluationLogger().reset_time()
def start_services_on_workers(self, composition, network):
"""
Start the services on the allocated workers
:param composition: Object of service composition
:param network: docker network
:raises SwarmException
"""
llogger = local_logger.LocalLogger()
llogger.log_method_call(self.__class__.__name__,
sys._getframe().f_code.co_name)
start_timer = datetime.datetime.now()
service_key_queue = deque(list(composition._allocation.keys()))
started_services = []
while len(service_key_queue) > 0:
service_key = service_key_queue.popleft()
worker_key = composition.get_worker_key(service_key)
service = composition.get_service(service_key)
if service.are_dependencies_started(started_services):
worker = self.swarm.get_worker(worker_key)
if worker is None:
llogger.debug("Error worker not found for worker key %s",
worker_key)
raise SwarmException("Worker not found for worker key " +
worker_key)
elif worker.start_service(jsonpickle.encode(service),
network) is False:
llogger.debug("Error starting service %s on worker %s",
service.tag, worker.uuid)
raise SwarmException("Failed to start service " +
service.tag + " on worker " +
worker.uuid)
started_services.append(service_key)
llogger.debug("Started service " + service_key +
" on worker " + worker_key)
else:
service_key_queue.append(service_key)
elapsed_time_until_service_start = datetime.datetime.now(
) - start_timer
evaluation_logger.EvaluationLogger().write([
"Time for starting containers", "*", "*",
elapsed_time_until_service_start.total_seconds(),
self.swarm.get_worker_count(),
composition.get_service_count()
], evaluation_logger.LogType.ALLOC_METRICS)
llogger.debug("Time elapsed for starting containers: %f",
elapsed_time_until_service_start.total_seconds())
def allocate_services_to_workers(self, composition, allocations):
llogger = local_logger.LocalLogger()
llogger.log_method_call(self.__class__.__name__,
sys._getframe().f_code.co_name)
from gortools import ortools_interface
hardware_matrix, cost_matrix = self.get_cost_and_hardware_matrix(
composition, allocations)
start_timer = datetime.datetime.now()
service_allocation_dict = ortools_interface.allocate_services_to_workers(
services=allocations,
workers=self.swarm._worker_list,
hardware_matrix=hardware_matrix,
cost_matrix=cost_matrix)
time_of_allocation = datetime.datetime.now() - start_timer
evaluation_logger.EvaluationLogger().write([
"Time of allocation", "*", "*",
time_of_allocation.total_seconds(),
self.swarm.get_worker_count(),
len(allocations)
], evaluation_logger.LogType.ALLOC_METRICS)
return service_allocation_dict
def configure_evaluation_logger(self,
log_folder=None,
log_ident=None,
enable=True):
"""
Configures the evaluation logger by setting its log_folder, log_ident, enabling or disabling it and
resetting its time
:param log_folder: None or log folder path
:param log_ident: identification string
:param enable: True when the evaluation logger should be enabled
:return:
"""
llogger = local_logger.LocalLogger()
llogger.log_method_call(self.__class__.__name__,
sys._getframe().f_code.co_name)
eval_logger = evaluation_logger.EvaluationLogger()
if log_folder is not None:
eval_logger.set_log_folder(log_folder)
if log_ident is not None:
eval_logger.set_log_ident(log_ident)
self.reset_evaluation_logger()
eval_logger.enable(enable)
def get_cost_and_hardware_matrix(self, composition, allocations):
llogger = local_logger.LocalLogger()
llogger.log_method_call(self.__class__.__name__,
sys._getframe().f_code.co_name)
hardware_matrix = []
cost_matrix = []
start_timer = datetime.datetime.now()
for service_key in allocations:
service = composition.get_service(service_key)
hardware_row_for_service, cost_row_for_service = self.get_cost_and_hardware_row_for_service(
service)
hardware_matrix.append(hardware_row_for_service)
cost_matrix.append(cost_row_for_service)
time_of_calculation = datetime.datetime.now() - start_timer
evaluation_logger.EvaluationLogger().write([
"Time of cost calculation", "*", "*",
time_of_calculation.total_seconds(),
self.swarm.get_worker_count(),
len(allocations)
], evaluation_logger.LogType.ALLOC_METRICS)
llogger.debug("Time elapsed for cost calculation: %f",
time_of_calculation.total_seconds())
return hardware_matrix, cost_matrix