def __init__(self, parent=None): self.parent = parent worker_types = set(config.WORKERS.values()) # Set bandwidth lock. self.set_bandwidth_lock = threading.Lock() # Track bandwidth requests. self.bandwidth_tracker = BandwidthRequestTracker() # Worker messages that deal with releasing or acquiring more bandwidth have to # be processed sequentially. self.bandwidth_lock = threading.Lock() # Will hold all information regarding purchased bandwidth. self.bandwidth_values = {} self.prev_bandwidth_values = {} # Will hold any scheduled bandwidth changes. Keys are worker types and values are # Timer objects. self.scheduled_bandwidth_changes = {} # Will hold a reference to MemFS. self.memfs = None # Start recording the time for accounting purposes. self.start_time = time.time() # Start MemFS. if config.START_MEMFS: self.start_memfs() self.memfs_logger = simple_logger.SimpleLogger(config.LOGDIR + '/memfs.log') # Used to gather worker statistics. self.waiting_for_workers_lock = threading.Lock() self.waiting_for_workers = {} self.waiting_current_max_bandwidth_lock = threading.Lock() self.waiting_current_max_bandwidth = {} self.worker_statistics_lock = threading.Lock() self.worker_statistics = {} for worker_type in worker_types: self.waiting_for_workers[worker_type] = 0 self.worker_statistics[worker_type] = [] # Start all the workers of with the maximum amount of bandwidth. max_bandwidth = max(config.AVAILABLE_BANDWIDTH_VALUES) self.bandwidth_loggers = {} self.bandwidth_purchased = {} for worker_type in worker_types: self.bandwidth_loggers[worker_type] = simple_logger.SimpleLogger(config.LOGDIR + '/' + str(worker_type) + '_bandwidth.log') self.bandwidth_purchased[worker_type] = [] self.bandwidth_values[worker_type] = { 'purchased_at': None, 'bandwidth': 0 } self.set_bandwidth(worker_type, max_bandwidth, False, True) self.prev_bandwidth_values = self.bandwidth_values.copy() # Holds information regarding the currently executing workflow. self.dag_statistics = {} self.dag_statistics_lock = threading.Lock() self.sent_task_executables = set() # Used to keep track of task ids. self.task_id = 1 # Used to log the cost. self.cost_logger = simple_logger.SimpleLogger(config.LOGDIR + '/cost.log') # Used to log the new task. self.new_task_logger = simple_logger.SimpleLogger(config.LOGDIR + '/new_task.log') self.request_bandwidth_lock = threading.Lock() self.request_bandwidth = None
class ResourceManager(object): """ The resource manager is in charge of acquiring or releasing resources based on information from the scheduler and the workers. """ def __init__(self, parent=None): self.parent = parent worker_types = set(config.WORKERS.values()) # Set bandwidth lock. self.set_bandwidth_lock = threading.Lock() # Track bandwidth requests. self.bandwidth_tracker = BandwidthRequestTracker() # Worker messages that deal with releasing or acquiring more bandwidth have to # be processed sequentially. self.bandwidth_lock = threading.Lock() # Will hold all information regarding purchased bandwidth. self.bandwidth_values = {} self.prev_bandwidth_values = {} # Will hold any scheduled bandwidth changes. Keys are worker types and values are # Timer objects. self.scheduled_bandwidth_changes = {} # Will hold a reference to MemFS. self.memfs = None # Start recording the time for accounting purposes. self.start_time = time.time() # Start MemFS. if config.START_MEMFS: self.start_memfs() self.memfs_logger = simple_logger.SimpleLogger(config.LOGDIR + '/memfs.log') # Used to gather worker statistics. self.waiting_for_workers_lock = threading.Lock() self.waiting_for_workers = {} self.waiting_current_max_bandwidth_lock = threading.Lock() self.waiting_current_max_bandwidth = {} self.worker_statistics_lock = threading.Lock() self.worker_statistics = {} for worker_type in worker_types: self.waiting_for_workers[worker_type] = 0 self.worker_statistics[worker_type] = [] # Start all the workers of with the maximum amount of bandwidth. max_bandwidth = max(config.AVAILABLE_BANDWIDTH_VALUES) self.bandwidth_loggers = {} self.bandwidth_purchased = {} for worker_type in worker_types: self.bandwidth_loggers[worker_type] = simple_logger.SimpleLogger(config.LOGDIR + '/' + str(worker_type) + '_bandwidth.log') self.bandwidth_purchased[worker_type] = [] self.bandwidth_values[worker_type] = { 'purchased_at': None, 'bandwidth': 0 } self.set_bandwidth(worker_type, max_bandwidth, False, True) self.prev_bandwidth_values = self.bandwidth_values.copy() # Holds information regarding the currently executing workflow. self.dag_statistics = {} self.dag_statistics_lock = threading.Lock() self.sent_task_executables = set() # Used to keep track of task ids. self.task_id = 1 # Used to log the cost. self.cost_logger = simple_logger.SimpleLogger(config.LOGDIR + '/cost.log') # Used to log the new task. self.new_task_logger = simple_logger.SimpleLogger(config.LOGDIR + '/new_task.log') self.request_bandwidth_lock = threading.Lock() self.request_bandwidth = None def start_memfs(self): """ Starts MemFS :return: """ # First create the config file. print('Starting MemFS') memfs_config_file = open(config.MEMFS_CONFIG_FILE, 'w') memfs_config_file.write(str(len(config.WORKERS)) + '\n' + str(len(config.WORKERS)) + '\n') memfs_config_file.write(str(config.MEMFS_BETA) + '\n' + str(config.MEMFS_BETA) + '\n') for worker_address in config.WORKERS: memfs_config_file.write(worker_address + '\n') memfs_config_file.close() # Now start MemFS. self.memfs = memfs.MemFS(config.MEMFS_CONFIG_FILE) self.memfs.initialize() print('MemFS initialized') def adapt_memfs(self): """ Redistributes MemFS partitions if necessary. :return: """ # If the current and previous bandwidth values are all the same, we don't need to redistribute MemFS # partitions. if self.equal_bandwidth_values(self.bandwidth_values) and self.equal_bandwidth_values(self.prev_bandwidth_values): print('No need to adapt MemFS') return # Otherwise we prepare the bandwidth file to adapt MemFS. print("Going to adapt MemFS") bandwidth_file = open('/home/' + getpass.getuser() + '/bandwidth.txt', 'w') for worker_address in config.WORKERS: worker_type = config.WORKERS[worker_address] bandwidth_file.write(worker_address + ':' + str(self.bandwidth_values[worker_type]['bandwidth']) + 'mbit\n') bandwidth_file.close() # Let MemFS adapt itself. self.memfs.scale_bandwidth() def equal_bandwidth_values(self, bandwidth_values): """ Determines if all given bandwidth values are the same. :param bandwidth_values: :return: """ if len(bandwidth_values) < 1: return True last = None for worker_type in bandwidth_values: if last is not None and bandwidth_values[worker_type]['bandwidth'] != last: return False else: last = bandwidth_values[worker_type]['bandwidth'] return True def stop_memfs(self): """ Stops MemFS :return: """ self.memfs.kill_all() def initialize_dag(self, tasks_data): """ Parse DAG information. We will maintain the total number of tasks for each phase of the workflow as well as which tasks still need to complete. :param tasks_data: :return: """ self.dag_statistics_lock.acquire() for task in tasks_data: executable = task['exec'] if executable not in self.dag_statistics: self.dag_statistics[executable] = {'total': 0, 'remaining': []} self.dag_statistics[executable]['workers'] = {} # Initialize the finished arrays for all workers. for worker in config.WORKERS.keys(): self.dag_statistics[executable]['workers'][worker] = {} self.dag_statistics[executable]['workers'][worker]['finished'] = [] self.dag_statistics[executable]['workers'][worker]['started_at'] = None self.dag_statistics[executable]['total'] += 1 self.dag_statistics[executable]['remaining'].append(self.task_id) self.task_id += 1 self.dag_statistics_lock.release() def sent_tasks_to_worker(self, worker, tasks): """ Indicates that the given tasks have been sent to the given worker. We record when the first of a task type has been sent as well as which task types are now running in general. :param worker: :param tasks: :return: """ self.dag_statistics_lock.acquire() now = time.time() for task in tasks: executable = task['exec'] # Record the first sending of this task. if executable not in self.sent_task_executables: self.sent_task_executables.add(executable) self.new_task_logger.log(str(now - self.start_time) + ',' + str(executable)) # Kill any outstanding bandwidth request. if config.ADAPT_BANDWIDTH: self.request_bandwidth_lock.acquire() if self.request_bandwidth is not None: self.request_bandwidth.cancel() self.request_bandwidth = None # Determine if this is a special phase. If so, request bandwidth in 20 seconds. if self.dag_statistics[executable]['total'] == 1: request_timer = threading.Timer(20, self.request_bandwidth_from_worker, (worker, executable)) self.request_bandwidth = request_timer request_timer.start() self.request_bandwidth_lock.release() if self.dag_statistics[executable]['workers'][worker]['started_at'] is None: self.dag_statistics[executable]['workers'][worker]['started_at'] = now self.dag_statistics_lock.release() def finished_tasks(self, worker, tasks): """ Marks the given tasks for the given worker as finished. :param worker: :param tasks: :return: """ self.dag_statistics_lock.acquire() current_exec = None for task in tasks: # To speed up the search we keep track of the current task executable. if current_exec is not None: if task in self.dag_statistics[current_exec]['remaining']: # Mark the task as finished by the worker. if task not in self.dag_statistics[current_exec]['workers'][worker]['finished']: self.dag_statistics[current_exec]['workers'][worker]['finished'].append(task) # Remove from remaining tasks self.dag_statistics[current_exec]['remaining'].remove(task) if len(self.dag_statistics[current_exec]['remaining']) == 0 and current_exec in self.sent_task_executables: self.sent_task_executables.remove(current_exec) continue # Otherwise we have no choice but to loop over the DAG to find the task. for executable in self.dag_statistics: if task in self.dag_statistics[executable]['remaining']: current_exec = executable # Mark the task as finished by the worker. if task not in self.dag_statistics[current_exec]['workers'][worker]['finished']: self.dag_statistics[current_exec]['workers'][worker]['finished'].append(task) # Remove from remaining tasks. self.dag_statistics[executable]['remaining'].remove(task) if len(self.dag_statistics[current_exec]['remaining']) == 0 and current_exec in self.sent_task_executables: self.sent_task_executables.remove(current_exec) self.dag_statistics_lock.release() def process_worker_message(self, message): """ Processes a workers message. :param message: :return: """ message_type = message[5] worker = message[0] data = pickle.loads(message[6]) # The network of the worker is fully utilized. if message_type == 'fully_utilized': print('==============================================') self.bandwidth_lock.acquire() can_report = self.process_worker_fully_utilized(worker, data) self.bandwidth_lock.release() print('==============================================') self.parent.send_message([message[0], PROTOCOL_HEADERS['RSMNG'], 'resource', pickle.dumps({'fully_utilized': can_report})]) # The network of the worker is under utilized, we can increase. elif message_type == 'under_utilized': print('==============================================') self.bandwidth_lock.acquire() can_report = self.process_worker_under_utilized(worker, data) self.bandwidth_lock.release() print('==============================================') self.parent.send_message([message[0], PROTOCOL_HEADERS['RSMNG'], 'resource', pickle.dumps({'under_utilized': can_report})]) # A worker has sent statistics elif message_type == 'worker_statistics': self.worker_statistics_lock.acquire() print('Received statistics from worker ' + worker) worker_type = config.WORKERS[worker] self.worker_statistics[worker_type].append(data) has_enough = (len(self.worker_statistics[worker_type]) == self.waiting_for_workers[worker_type]) self.worker_statistics_lock.release() # Check if these were the last statistics to come in. if has_enough: print('Got all needed statistics, going to determine if we can increase bandwidth') self.parse_worker_statistics(worker_type) def request_bandwidth_from_worker(self, worker, executable): """ Sends a request to a worker to submit his statistics. :param worker: :param executable: :return: """ print('Asking ' + str(worker) + ' to send statistics for running ' + str(executable)) sys.stdout.flush() self.parent.send_message([worker, PROTOCOL_HEADERS['RSMNG'], 'get_bandwidth', pickle.dumps(executable)]) # Ask again in a while. self.request_bandwidth_lock.acquire() if self.request_bandwidth is not None: self.request_bandwidth = threading.Timer(20, self.request_bandwidth_from_worker, (worker, executable)) self.request_bandwidth.start() self.request_bandwidth_lock.release() def process_worker_under_utilized(self, worker, data): """ Processes a request from a worker that states he is under utilizing his bandwidth. :param worker: :param data: :return: """ # Check if there is an outstanding bandwidth request. If so, do nothing. worker_type = config.WORKERS[worker] statistics = data['statistics'] sent_max_bandwidth = statistics['bandwidth']['out']['max'] minimal_bandwidth = self.get_minimal_bandwidth(statistics) if not minimal_bandwidth: return False # Count the request. self.bandwidth_tracker.got_bandwidth_request(data['executable'], minimal_bandwidth) # Another worker might have lowered the bandwidth already. if sent_max_bandwidth != self.bandwidth_values[worker_type]['bandwidth']: return False now = time.time() last_purchase = self.bandwidth_values[worker_type]['purchased_at'] time_in_period = (now - last_purchase) % config.BANDWIDTH_BILLING_PERIOD # Add 5% buffer time before the end of the billing period if possible. buffer_time = (config.BANDWIDTH_BILLING_PERIOD / 100.0) * 5 schedule_downgrade = config.BANDWIDTH_BILLING_PERIOD - (time_in_period + buffer_time) if schedule_downgrade < 0: schedule_downgrade = config.BANDWIDTH_BILLING_PERIOD - time_in_period # If we can honor the downgrade, cancel any existing downgrade timer before setting a new one. if self.bandwidth_tracker.can_honor_request(data['executable'], minimal_bandwidth): if worker_type in self.scheduled_bandwidth_changes: downgrade = self.scheduled_bandwidth_changes[worker_type] if downgrade['to'] != minimal_bandwidth: # Cancel the old timer. print('Cancelling downgrade timer') downgrade['timer'].cancel() self.scheduled_bandwidth_changes.pop(worker_type) # Set the new timer. downgrade_timer = threading.Timer(schedule_downgrade, self.set_bandwidth, (worker_type, minimal_bandwidth)) self.scheduled_bandwidth_changes[worker_type] = { 'to': minimal_bandwidth, 'timer': downgrade_timer } downgrade_timer.start() print('Scheduled downgrade for workers of type ' + str(worker_type) + ' to ' + str(minimal_bandwidth) + 'mbit/s in ' + str(schedule_downgrade) + ' seconds') else: print('Keeping existing downgrade timer') else: # Set a new timer. downgrade_timer = threading.Timer(schedule_downgrade, self.set_bandwidth, (worker_type, minimal_bandwidth)) self.scheduled_bandwidth_changes[worker_type] = { 'to': minimal_bandwidth, 'timer': downgrade_timer } downgrade_timer.start() print('Scheduled downgrade for workers of type ' + str(worker_type) + ' to ' + str(minimal_bandwidth) + 'mbit/s in ' + str(schedule_downgrade) + ' seconds') return False def process_worker_fully_utilized(self, worker, data): """ Processes a request from a worker that states he is fully utilizing his bandwidth. :param worker: :param data: :return: """ print('Got increase bandwidth request from ' + worker) worker_type = config.WORKERS[worker] current_bandwidth = self.bandwidth_values[worker_type]['bandwidth'] statistics = data['statistics'] # Get the max bandwidth. max_bandwidth = self.get_maximal_bandwidth(worker, worker_type, data) if not max_bandwidth: return False # Now we count the request. self.bandwidth_tracker.got_bandwidth_request(data['executable'], max_bandwidth) # Another worker might have already upped the bandwidth, in that case we do not # change it now. if statistics['bandwidth']['out']['max'] != current_bandwidth: print('Max bandwidth for ' + worker + ' (' + str(statistics['bandwidth']['out']['max']) + ') does not equal listed max of ' + str(current_bandwidth)) return False # Determine if enough requests have come in to up the bandwidth. if not self.bandwidth_tracker.can_honor_request(data['executable'], max_bandwidth): return False # If there is a scheduled downgrade for workers of this type, cancel it. if worker_type in self.scheduled_bandwidth_changes: print('Cancelling scheduled downgrade for workers of type ' + str(worker_type)) sys.stdout.flush() downgrade_timer = self.scheduled_bandwidth_changes[worker_type]['timer'] downgrade_timer.cancel() self.scheduled_bandwidth_changes.pop(worker_type) print('Statistics for increasing bandwidth: ' + str(statistics)) print('Increasing bandwidth for workers of type ' + str(worker_type) + ' to ' + str(max_bandwidth) + ' mbit/s') self.set_bandwidth(worker_type, max_bandwidth) return False def parse_worker_statistics(self, worker_type): """ When all statistics for a certain worker type have come in, we can check if it is cost effective to a certain bandwidth value. :param worker_type: :return: """ self.worker_statistics_lock.acquire() self.waiting_for_workers_lock.acquire() self.waiting_current_max_bandwidth_lock.acquire() # Combine colocated statistics. combined_history = {} for history in self.worker_statistics[worker_type]: for colocated in history: # We do not have figures for this number of colocated yet. if colocated not in combined_history: combined_history[colocated] = { 'runtime': [history[colocated][0]], 'bw_out': [history[colocated][1]], 'bw_in': [history[colocated][2]] } else: # Add the figures, will average later. combined_history[colocated]['runtime'].append(history[colocated][0]) combined_history[colocated]['bw_out'].append(history[colocated][1]) combined_history[colocated]['bw_in'].append(history[colocated][2]) # Average out all figures (combine bandwidth in and bandwidth out). averaged = {} for colocated in combined_history: averaged[colocated]['runtime'] = sum(combined_history[colocated]['runtime']) / float(len(combined_history[colocated]['runtime'])) averaged[colocated]['bw'] = (sum(combined_history[colocated]['bw_out']) / float(len(combined_history[colocated]['bw_out']))) + \ (sum(combined_history[colocated]['bw_in']) / float(len(combined_history[colocated]['bw_in']))) # See if setting the max bandwidth makes sense by looking at the throughput. nr_colocated_for_max = self.get_nr_colocated_tasks_for_bandwidth(averaged, max(config.AVAILABLE_BANDWIDTH_VALUES)) runtime_for_max = self.get_runtime_for_nr_colocated_tasks(averaged, nr_colocated_for_max) throughput_for_max = nr_colocated_for_max / float(runtime_for_max) # Clear the collected values. self.waiting_for_workers = {} self.worker_statistics[worker_type] = {} # Tell the workers they can resume reporting bandwidth. for w in config.WORKERS: if config.WORKERS[w] == worker_type: print('Worker ' + w + ' can resume reporting bandwidth') self.parent.send_message([w, PROTOCOL_HEADERS['RSMNG'], 'resource', pickle.dumps(True)]) self.waiting_current_max_bandwidth_lock.release() self.waiting_for_workers_lock.release() self.worker_statistics_lock.release() def get_nr_colocated_tasks_for_bandwidth(self, data, target_bandwidth): """ Applies a linear function to the given data to get the number of colocated tasks for the given target bandwidth. :param data: :param target_bandwidth: :return: """ colocated = data.keys() bandwidth = [data[c]['bw'] for c in data] # Apply a linear function. def curve(x, a, b): return a * x + b popt, pcov = curve_fit(curve, colocated, bandwidth) a = popt[0] b = popt[1] return (target_bandwidth - b) / a def get_runtime_for_nr_colocated_tasks(self, data, target_colocated): """ Applies a linear function to the given data to get the runtime for the given number of colocated tasks. :param data: :param target_colocated: :return: """ colocated = data.keys() runtime = [data[c]['runtime'] for c in data] # Apply a linear function. def curve(x, a, b): return a * x + b popt, pcov = curve_fit(curve, colocated, runtime) a = popt[0] b = popt[1] return curve(target_colocated, a, b) def get_maximal_bandwidth(self, worker, worker_type, data): """ Returns the new maximum bandwidth based on the current strategy and the current bandwidth. :param worker: :param worker_type: :param data: :return: """ if config.MAX_BANDWIDTH_STRATEGY == 'Performance': return max(config.AVAILABLE_BANDWIDTH_VALUES) else: send_workers = [] for w in config.WORKERS: if config.WORKERS[w] == worker_type and w != worker: send_workers.append(w) # Indicate that we are waiting for workers. self.waiting_for_workers_lock.acquire() if self.waiting_for_workers[worker_type] > 0: # We are already waiting for statistics to come in. self.waiting_for_workers_lock.release() return False else: self.waiting_for_workers[worker_type] = len(send_workers) + 1 self.waiting_for_workers_lock.release() # Save the current max, so we know where to increase from when we have gathered statistics. self.waiting_current_max_bandwidth_lock.acquire() self.waiting_current_max_bandwidth[worker_type] = data['statistics']['bandwidth']['out']['max'] self.waiting_current_max_bandwidth_lock.release() self.worker_statistics_lock.acquire() self.worker_statistics[worker_type].append(data['history']) self.worker_statistics_lock.release() # Send the actual messages. Once the last one is received, we will parse the information. for w in send_workers: # The workers can not report bandwidth in the meantime self.parent.send_message([w, PROTOCOL_HEADERS['RSMNG'], 'resource', pickle.dumps(False)]) self.parent.send_message([w, PROTOCOL_HEADERS['RSMNG'], 'statistics', pickle.dumps(data['executable'])]) return False def get_minimal_bandwidth(self, statistics): """ Tries to find the minimal bandwidth value that satisfies the minimum and maximum allowed network utilization based on the current bandwidth. :param statistics: :return: """ bandwidth_values = config.AVAILABLE_BANDWIDTH_VALUES.keys() bandwidth_values.sort() # The maximum of the out/in bandwidth determines the new value. current_out = statistics['bandwidth']['out']['current'] current_in = statistics['bandwidth']['in']['current'] if current_out > current_in: current_bandwidth = current_out else: current_bandwidth = current_in # The ideal bandwidth lies between the minimum and maximum allowed percentage. for bandwidth in bandwidth_values: # We want the smallest bandwidth that is larger than the current value. # Because the list is sorted, we can return the first one we find that # will not lead to contention. Also add a little buffer of 10% below the contention # limit. if (bandwidth * ((config.MAX_BANDWIDTH_PERCENT - 10) / 100.0)) > int(current_bandwidth): sys.stdout.flush() return bandwidth # If there is no bandwidth larger than the current value, do nothing. sys.stdout.flush() return False def set_bandwidth(self, worker_type, bandwidth, adapt_memfs=True, first=False): """ Sets the bandwidth of workers with the given type to a certain value. :param worker_type: :param bandwidth: :param adapt_memfs: :return: """ # Don't do anything if there is no bandwidth control binary or we are already at the target bandwidth. if config.BANDWIDTH_CONTROL_BINARY == '' or (self.bandwidth_values[worker_type]['bandwidth'] == bandwidth): print('Not setting bandwidth for workers of type ' + str(worker_type) + ' to ' + str(bandwidth) + ' Mbit/s') return self.set_bandwidth_lock.acquire() print('Setting bandwidth of workers with type ' + str(worker_type) + ' to: ' + str(bandwidth) + ' Mbit/s') workers = [] for worker_address in config.WORKERS: if config.WORKERS[worker_address] == worker_type: workers.append(worker_address) tmp_file_path = config.LOGDIR + '/tmp_bandwidth_file' tmp_file = open(tmp_file_path, 'w') for worker in workers: tmp_file.write(worker + ':' + str(bandwidth) + 'mbit\n') tmp_file.close() # Save the purchase time. if first: purchase_time = self.start_time else: purchase_time = time.time() command = config.BANDWIDTH_CONTROL_BINARY + ' --set-all-file ' + tmp_file_path + ' --port ' + str(config.BANDWIDTH_CONTROL_PORT) + ' > /dev/null' proc = psutil.Popen(command, shell=True) proc.wait() os.remove(tmp_file_path) # Save the purchased bandwidth. self.prev_bandwidth_values = self.bandwidth_values.copy() self.bandwidth_values[worker_type] = { 'purchased_at': purchase_time, 'bandwidth': bandwidth } logger = self.bandwidth_loggers[worker_type] logger.log(str(purchase_time - self.start_time) + ',' + str(bandwidth)) self.bandwidth_purchased[worker_type].append((purchase_time - self.start_time, config.AVAILABLE_BANDWIDTH_VALUES[bandwidth])) # Adapt MemFS if needed. if adapt_memfs and config.START_MEMFS: self.adapt_memfs() # Remove from scheduled changes. if worker_type in self.scheduled_bandwidth_changes: self.scheduled_bandwidth_changes.pop(worker_type) self.set_bandwidth_lock.release() def calculate_cost(self): """ Calculates the total cost for running up until now and logs it. :return: """ total = 0 end = time.time() - self.start_time for worker_type in self.bandwidth_purchased: # Get the number of workers of this type. nr_workers = 0 for worker_address in config.WORKERS: if config.WORKERS[worker_address] == worker_type: nr_workers += 1 purchased_for_type = self.bandwidth_purchased[worker_type] for idx, purchased in enumerate(purchased_for_type): # We subtract the next purchase time. if idx < (len(purchased_for_type) - 1): reference_time = purchased_for_type[idx + 1][0] else: reference_time = end running_time = reference_time - purchased[0] # Make sure to round up. total += (math.ceil(running_time / float(config.BANDWIDTH_BILLING_PERIOD)) * purchased[1] * nr_workers) self.cost_logger.log(str(total))