def scramble_tiers(tier_resources, times): def permute(a, l, r): if l == r: yield a else: for i in xrange(l, r + 1): a[l], a[i] = a[i], a[l] for p in permute(a, l + 1, r): yield p a[l], a[i] = a[i], a[l] def generate_splits(total): splits = [1.0 / 6.0, 1.0 / 3.0, 1.0 / 2.0] orderings = [] for permutation in permute(splits, 0, 2): orderings.append([total * x for x in permutation]) return orderings splits = generate_splits(1) done = [] t = 0 while t < times: not_changed = random.choice(resources) key = [] for r in resources: if r != not_changed: split = random.choice(splits) for i in range(1, 4): tier_resources[MR( i, r, [])] = split[i - 1] * tier_resources[MR(0, r, [])] key.append((r, split, [])) if key not in done: t += 1 yield tier_resources.copy()
def tier_to_configuration(tier_resources): config = {} for i in range(1, 4): for r in resources: value = tier_resources[MR(i, r, [])] for service in tiers[i]: config[MR(service, r, [])] = value return config
def write_config(title, tier_resources, f): f.write("{}\n".format(title)) for i in range(1, 4): f.write(" Tier {}: {}\n".format(i, tiers[i])) for r in resources: f.write(" {}: {}\n".format(r, tier_resources[MR(i, r, [])]))
def mr_str_to_obj(redis_db, mr_list): mr_object_list = [] for mr in mr_list: service_name, resource = mr.split(',') deployments = tbot_datastore.read_service_locations( redis_db, service_name) mr_object_list.append(MR(service_name, resource, deployments)) return mr_object_list
def parse_resource_config_file(resource_config_csv, sys_config): machine_type = sys_config['machine_type'] vm_list = get_actual_vms() service_placements = get_service_placements(vm_list) all_services = get_actual_services() all_resources = get_stressable_resources() mr_allocation = {} # Empty Config means that we should default resource allocation to only use # half of the total resource capacity on the machine if resource_config_csv is None: vm_to_service = get_vm_to_service(vm_list) # DEFAULT_ALLOCATION sets the initial configuration # Ensure that we will not violate resource provisioning in the machine # Assign resources equally to services without exceeding machine resource limitations max_num_services = 0 for vm in vm_to_service: if len(vm_to_service[vm]) > max_num_services: max_num_services = len(vm_to_service[vm]) default_alloc_percentage = 70.0 / max_num_services mr_list = get_all_mrs_cluster(vm_list, all_services, all_resources) for mr in mr_list: max_capacity = get_instance_specs(machine_type)[mr.resource] default_raw_alloc = (default_alloc_percentage / 100.0) * max_capacity mr_allocation[mr] = default_raw_alloc print mr_allocation else: # Manual Configuration Possible # Parse a CSV # Format of resource allocation: SERVICE,RESOURCE,TYPE,REPR mr_list = get_all_mrs_cluster(vm_list, all_services, all_resources) with open(resource_config_csv, 'rb') as resource_config: reader = csv.DictReader(resource_config) for row in reader: service_name = row['SERVICE'] resource = row['RESOURCE'] amount = float(row['AMOUNT']) amount_repr = row['REPR'] # Convert REPR to RAW AMOUNT if amount_repr == 'PERCENT': if amount <= 0 or amount > 100: print 'Error: invalid default percentage. Exiting...' exit() max_capacity = get_instance_specs(machine_type)[resource] amount = (amount / 100.0) * max_capacity mr = MR(service_name, resource, service_placements[service_name]) assert mr in mr_list mr_allocation[mr] = amount return mr_allocation
def combine_resources(mr_allocation, imr_list, instance_type, resource_type='CPU-QUOTA'): vm_list = get_actual_vms() vm_to_service = get_vm_to_service(vm_list) imr_service_names = [imr.service_name for imr in imr_list] resource_capacity = get_instance_specs(instance_type) single_core_capacity = 100 all_deploy_sets = [] for vm in vm_to_service.keys(): deploy_together = [] resource_usage = 0 # First determine if an IMR has been placed on the machine # Assume only one IMR per machine for service_name in vm_to_service[vm]: if service_name in imr_service_names: resource_usage += mr_allocation[MR(service_name, resource_type, None)] deploy_together.append(MR(service_name, resource_type, None)) break # Next, look at the remianing services for service_name in vm_to_service[vm]: if service_name in imr_service_names: continue relevant_mr = MR(service_name, resource_type, None) if mr_allocation[ relevant_mr] + resource_usage < single_core_capacity: resource_usage += mr_allocation[relevant_mr] deploy_together.append(MR(service_name, resource_type, None)) all_deploy_sets.append(deploy_together) # Can't fill up remaining resources because it is no guarantee that other # instances will be able to sustain same resource growths. return all_deploy_sets
def get_all_mrs(redis_db): mr_name = 'mr_alloc' all_mrs = redis_db.hgetall(mr_name) mr_list = list(all_mrs.keys()) mr_object_list = [] for mr in mr_list: service_name, resource = mr.split(',') deployments = tbot_datastore.read_service_locations( redis_db, service_name) mr_object_list.append(MR(service_name, resource, deployments)) return mr_object_list
def identify_tier_resources(mr_allocation): tier_resources = {} for mr, value in mr_allocation.items(): assert mr.resource in resources for tier, lst in tiers.items(): if mr.service_name in lst: tier_mr = MR(tier, mr.resource, []) if tier_mr in tier_resources: assert tier_resources[tier_mr] == value else: tier_resources[tier_mr] = value for resource in resources: total = 0 for i in range(1, 4): total += tier_resources[MR(i, resource, [])] tier_resources[MR(0, resource, [])] = total return tier_resources
def get_all_mrs_cluster(vm_list, services, resources): mr_schedule = [] service_to_deployment = get_service_placements(vm_list) for service in service_to_deployment: if service not in services: continue deployments = service_to_deployment[service] for resource in resources: mr_schedule.append(MR(service, resource, deployments)) return mr_schedule
def get_all_mrs_cluster(vm_list, services, resources): mr_schedule = [] service_to_deployment = get_service_placements(vm_list) logging.info('printing service to deployment {}'.format(service_to_deployment)) for service in service_to_deployment: deployments = service_to_deployment[service] for resource in resources: mr_schedule.append(MR(service, resource, deployments)) logging.info('mr schedule is {}'.format(mr_schedule)) return mr_schedule
def read_all_mr_alloc(redis_db): mr_name = 'mr_alloc' mr_to_score = redis_db.hgetall(mr_name) for mr in mr_to_score: mr_to_score[mr] = float(mr_to_score[mr]) mr_allocation_list = {} for mr in mr_to_score: service_name, resource = mr.split(',') deployments = tbot_datastore.read_service_locations( redis_db, service_name) mr_object = MR(service_name, resource, deployments) mr_allocation_list[mr_object] = mr_to_score[mr] return mr_allocation_list
def parse_config_csv(resource_config_csv): mr_allocation = {} with open(resource_config_csv, 'rb') as resource_config: reader = csv.DictReader(resource_config) for row in reader: service_name = row['SERVICE'] resource = row['RESOURCE'] amount = float(row['AMOUNT']) amount_repr = row['REPR'] # Convert REPR to RAW AMOUNT if amount_repr == 'RAW': mr = MR(service_name, resource, None) mr_allocation[mr] = amount return mr_allocation
def ffd_pack(past_mr_allocation, instance_type, sort_by='CPU-QUOTA', imr_list=[], deploy_together=[], round_up=False): mr_allocation = deepcopy(past_mr_allocation) if round_up is True: for imr in imr_list: assert imr in mr_allocation mr_allocation[imr] = roundup(mr_allocation[imr]) # Use the below if you have already deployed the containers on quilt # Otherwise, define it manually. vm_list = get_actual_vms() service_placements = get_service_placements(vm_list) service_configurations = [ mr.to_string().split(',')[0] for mr in mr_allocation.keys() ] service_containers = [] for service_name in service_placements: if service_name in service_configurations: service_containers += [ service_name for x in range(len(service_placements[service_name])) ] for affinity in deploy_together: for mr in affinity: service_containers.remove(mr.service_name) # Create a fake MR with the combination of MRs for affinity in deploy_together: affinity_services = '' total_resources = 0 for mr in affinity: affinity_services += '&' + mr.service_name total_resources += mr_allocation[mr] mr_allocation[MR(affinity_services, 'CPU-QUOTA', None)] = roundup(total_resources) service_containers.append(affinity_services) service_to_mr = set_service_specs( service_containers, mr_allocation) # Amend to include affinities # IMR Aware Scheduling def imr_aware(sc, num_machines): sc_copy = deepcopy(sc) if num_machines == 1: return None instance_specs = get_instance_specs(instance_type) machine_to_allocation = {} for x in range(num_machines): machine_to_allocation = initialize_capacity( machine_to_allocation, x, instance_specs) machine_to_service = {} for x in range(num_machines): machine_to_service[x] = [] resource_to_impacted_service = {} service_encountered = [] # Find the IMRs and place them for mr in imr_list: if mr.service_name in service_encountered: continue else: service_encountered.append(mr.service_name) if mr.resource not in resource_to_impacted_service: resource_to_impacted_service[mr.resource] = [] resource_to_impacted_service[mr.resource].append(mr.service_name) # First start by round robin the MIMR and other MRs that are of the same resource mimr_resource = imr_list[0].resource service_list = resource_to_impacted_service[mimr_resource] containers_seen = 0 for service_name in service_list: num_service_containers = sc_copy.count(service_name) mr_list = service_to_mr[service_name] # Round robin the placements for container_index in range(num_service_containers): machine_index = (containers_seen + container_index) % num_machines fit_found = place_if_possible(mr_list, mr_allocation, machine_to_allocation, machine_index, instance_specs) if fit_found: machine_to_service[machine_index].append(service_name) containers_seen += 1 else: logging.error( 'You have selected too many IMRs for consideration!') return {} # Don't double count the services for service_name in service_list: sc_copy = filter(lambda a: a != service_name, sc_copy) # Reverse the order of the services packed into the machine new_machine_to_allocation = {} new_machine_to_service = {} for machine_index in machine_to_allocation: new_machine_to_allocation[num_machines - machine_index - 1] = machine_to_allocation[machine_index] new_machine_to_service[num_machines - machine_index - 1] = machine_to_service[machine_index] machine_to_allocation = new_machine_to_allocation machine_to_service = new_machine_to_service # Then do a first fit with any remaining MR. for service_name in sc_copy: mr_list = service_to_mr[service_name] fit_found, machine_index = update_if_possible( mr_list, mr_allocation, machine_to_allocation, num_machines, instance_specs) if fit_found: machine_to_service[machine_index].append(service_name) else: logging.error('No valid placement found. Returning') return {} return machine_to_service def ff(sc): instance_specs = get_instance_specs(instance_type) machine_to_allocation = {} machine_to_allocation = initialize_capacity(machine_to_allocation, 0, instance_specs) machine_to_service = {} machine_to_service[0] = [] number_machines = 1 for service_name in sc: mr_list = service_to_mr[service_name] fit_found, machine_index = update_if_possible( mr_list, mr_allocation, machine_to_allocation, number_machines, instance_specs) if fit_found: machine_to_service[machine_index].append(service_name) if fit_found is False: number_machines += 1 machine_to_allocation[number_machines - 1] = {} machine_to_service[number_machines - 1] = [] initialize_capacity(machine_to_allocation, number_machines - 1, instance_specs) fit_found, machine_index = update_if_possible( mr_list, mr_allocation, machine_to_allocation, number_machines, instance_specs) machine_to_service[machine_index].append(service_name) if fit_found is False: logging.error('Fit not found. You have a bug. Exiting..') exit() return machine_to_service resource_index = {} for service in service_to_mr: for x in range(len(service_to_mr[service])): resource_index[service_to_mr[service][x].resource] = x # First find the number of machines from first fit imr_resource = imr_list[0].resource service_containers = sorted(service_containers, key=lambda x: mr_allocation[service_to_mr[x][ resource_index[imr_resource]]]) print service_containers first_fit_placements = ff(service_containers) print first_fit_placements logging.info( 'First fit service placement is {}'.format(first_fit_placements)) # First place the services that show up as MIMRs optimal_num_machines = len(first_fit_placements.keys()) imr_aware_service_placement = {} imr_aware_service_placement = imr_aware(service_containers, optimal_num_machines) if imr_aware_service_placement is None: imr_aware_service_placement = first_fit_placements while len(imr_aware_service_placement.keys()) == 0: optimal_num_machines += 1 imr_aware_service_placement = imr_aware(service_containers, optimal_num_machines) if len(imr_aware_service_placement.keys()) != 0: break logging.info('IMR Aware Service Placement is {}'.format( imr_aware_service_placement)) return first_fit_placements, imr_aware_service_placement
service_containers = sorted(service_containers, key=lambda x: mr_allocation[service_to_mr[x][resource_index[imr_resource]]]) first_fit_placements = ff(service_containers) logging.info('First fit service placement is {}'.format(first_fit_placements)) # First place the services that show up as MIMRs optimal_num_machines = len(first_fit_placements.keys()) imr_aware_service_placement = {} imr_aware_service_placement = imr_aware(service_containers, optimal_num_machines) while len(imr_aware_service_placement.keys()) == 0: optimal_num_machines += 1 imr_aware_service_placement = imr_aware(service_containers, optimal_num_machines) if len(imr_aware_service_placement.keys()) != 0: break logging.info('IMR Aware Service Placement is {}'.format(imr_aware_service_placement)) return first_fit_placements, imr_aware_service_placement if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--resource_csv", help='resource configuration file') parser.add_argument("--instance_type", help='instance type') args = parser.parse_args() allocations = parse_config_csv(args.resource_csv) # Must re-select this imr_list = [MR('tsaianson/node-apt-app', 'CPU-QUOTA', None), MR('haproxy:1.7', 'NET', None), MR('haproxy:1.7', 'NET', None)] ffp, iap = ffd_pack(allocations, args.instance_type, imr_list[0].resource, imr_list)
def run(sys_config, workload_config, filter_config, default_mr_config, last_completed_iter=0): redis_host = sys_config['redis_host'] baseline_trials = sys_config['baseline_trials'] experiment_trials = sys_config['trials'] stress_weights = sys_config['stress_weights'] stress_policy = sys_config['stress_policy'] resource_to_stress = sys_config['stress_these_resources'] service_to_stress = sys_config['stress_these_services'] vm_to_stress = sys_config['stress_these_machines'] machine_type = sys_config['machine_type'] quilt_overhead = sys_config['quilt_overhead'] gradient_mode = sys_config['gradient_mode'] preferred_performance_metric = workload_config['tbot_metric'] optimize_for_lowest = workload_config['optimize_for_lowest'] redis_db = redis.StrictRedis(host=redis_host, port=6379, db=0) if last_completed_iter == 0: redis_db.flushall() ''' # Prompt the user to make sure they want to flush the db ok_to_flush = raw_input("Are you sure you want to flush the results of your last experiment? Please respond with Y or N: ") if ok_to_flush == 'Y': redis_db.flushall() elif ok_to_flush == 'N': print 'OK you said it boss. Exiting...' exit() else: print 'Only Y and N are acceptable responses. Exiting...' exit() ''' print '\n' * 2 print '*' * 20 print 'INFO: INITIALIZING RESOURCE CONFIG' # Initialize Redis and Cluster based on the default resource configuration init_cluster_capacities_r(redis_db, machine_type, quilt_overhead) init_service_placement_r(redis_db, default_mr_config) init_resource_config(redis_db, default_mr_config, machine_type) print '*' * 20 print 'INFO: INSTALLING DEPENDENCIES' #install_dependencies(workload_config) # Initialize time for data charts time_start = datetime.datetime.now() print '*' * 20 print 'INFO: RUNNING BASELINE' # Get the Current Performance -- not used for any analysis, just to benchmark progress!! current_performance = measure_baseline(workload_config, baseline_trials) current_performance[preferred_performance_metric] = remove_outlier( current_performance[preferred_performance_metric]) current_time_stop = datetime.datetime.now() time_delta = current_time_stop - time_start print 'Current (non-analytic) performance measured: {}'.format( current_performance) if last_completed_iter != 0: tbot_datastore.write_summary_redis( redis_db, 0, MR('initial', 'initial', []), 0, {}, mean_list(current_performance[preferred_performance_metric]), mean_list(current_performance[preferred_performance_metric]), time_delta.seconds, 0) print '============================================' print '\n' * 2 # Initialize the current configurations # Initialize the working set of MRs to all the MRs mr_working_set = resource_datastore.get_all_mrs(redis_db) resource_datastore.write_mr_working_set(redis_db, mr_working_set, 0) cumulative_mr_count = 0 experiment_count = last_completed_iter + 1 while experiment_count < 10: # Calculate the analytic baseline that is used to determine MRs analytic_provisions = prepare_analytic_baseline( redis_db, sys_config, min(stress_weights)) print 'The Analytic provisions are as follows {}'.format( analytic_provisions) for mr in analytic_provisions: resource_modifier.set_mr_provision(mr, analytic_provisions[mr]) analytic_baseline = measure_runtime(workload_config, experiment_trials) analytic_mean = mean_list( analytic_baseline[preferred_performance_metric]) print 'The analytic baseline is {}'.format(analytic_baseline) print 'This current performance is {}'.format(current_performance) analytic_baseline[preferred_performance_metric] = remove_outlier( analytic_baseline[preferred_performance_metric]) # Get a list of MRs to stress in the form of a list of MRs mr_to_consider = apply_filtering_policy(redis_db, mr_working_set, experiment_count, sys_config, workload_config, filter_config) for mr in mr_to_consider: print '\n' * 2 print '*' * 20 print 'Current MR is {}'.format(mr.to_string()) increment_to_performance = {} current_mr_allocation = resource_datastore.read_mr_alloc( redis_db, mr) print 'Current MR allocation is {}'.format(current_mr_allocation) for stress_weight in stress_weights: # Calculate Gradient Schedule and provision resources accordingly mr_gradient_schedule = calculate_mr_gradient_schedule( redis_db, [mr], sys_config, stress_weight) for change_mr in mr_gradient_schedule: resource_modifier.set_mr_provision( change_mr, mr_gradient_schedule[change_mr]) experiment_results = measure_runtime(workload_config, experiment_trials) # Write results of experiment to Redis # preferred_results = remove_outlier(experiment_results[preferred_performance_metric]) preferred_results = experiment_results[ preferred_performance_metric] mean_result = mean_list(preferred_results) tbot_datastore.write_redis_ranking( redis_db, experiment_count, preferred_performance_metric, mean_result, mr, stress_weight) # Revert the Gradient schedule and provision resources accordingly mr_revert_gradient_schedule = revert_mr_gradient_schedule( redis_db, [mr], sys_config, stress_weight) for change_mr in mr_revert_gradient_schedule: resource_modifier.set_mr_provision( change_mr, mr_revert_gradient_schedule[change_mr]) increment_to_performance[stress_weight] = experiment_results # Write the results of the iteration to Redis tbot_datastore.write_redis_results(redis_db, mr, increment_to_performance, experiment_count, preferred_performance_metric) print '*' * 20 print '\n' * 2 # Timing Information for the purpose of experiments current_time_stop = datetime.datetime.now() time_delta = current_time_stop - time_start cumulative_mr_count += len(mr_to_consider) chart_generator.get_summary_mimr_charts( redis_db, workload_config, current_performance, mr_working_set, experiment_count, stress_weights, preferred_performance_metric, time_start) # Move back into the normal operating basis by removing the baseline prep stresses reverted_analytic_provisions = revert_analytic_baseline( redis_db, sys_config) for mr in reverted_analytic_provisions: resource_modifier.set_mr_provision( mr, reverted_analytic_provisions[mr]) # Recover the results of the experiment from Redis max_stress_weight = min(stress_weights) mimr_list = tbot_datastore.get_top_n_mimr( redis_db, experiment_count, preferred_performance_metric, max_stress_weight, gradient_mode, optimize_for_lowest=optimize_for_lowest, num_results_returned=-1) imr_list, nimr_list = seperate_mr( mimr_list, mean_list(analytic_baseline[preferred_performance_metric]), optimize_for_lowest) if len(imr_list) == 0: print 'INFO: IMR list length is 0. Please choose a metric with more signal. Exiting...' break print 'INFO: IMR list is {}'.format( [mr.to_string() for mr in imr_list]) print 'INFO: NIMR list is {}'.format( [mr.to_string() for mr in nimr_list]) # Try all the MIMRs in the list until a viable improvement is determined # Improvement Amount mimr = None action_taken = {} for imr in imr_list: imr_improvement_percent = improve_mr_by(redis_db, imr, max_stress_weight) current_imr_alloc = resource_datastore.read_mr_alloc(redis_db, imr) new_imr_alloc = convert_percent_to_raw(imr, current_imr_alloc, imr_improvement_percent) imr_improvement_proposal = new_imr_alloc - current_imr_alloc # If the the Proposed MR cannot be improved by the proposed amount, there are two options # - Max out the resources to fill up the remaining resources on the machine # - Resource Stealing from NIMRs # Both functions will return VIABLE improvements to the IMR deployment nimr_diff_proposal = {} if check_improve_mr_viability(redis_db, imr, imr_improvement_proposal) is False: print 'INFO: MR {} to increase {} by {} is not viable'.format( imr.to_string(), current_imr_alloc, imr_improvement_proposal) print 'INFO: Attempting to max out the machines resources...' imr_improvement_proposal = fill_out_resource(redis_db, imr) if imr_improvement_proposal <= 0: print 'INFO: No more space to fill out resources. Stealing from NIMRs' # Calculate a plan to reduce the resource provisioning of NIMRs nimr_diff_proposal, imr_improvement_proposal = create_decrease_nimr_schedule( redis_db, imr, nimr_list, max_stress_weight) print 'INFO: Proposed NIMR {}'.format(nimr_diff_proposal) print 'INFO: New IMR improvement {}'.format( imr_improvement_proposal) if len(nimr_diff_proposal ) == 0 or imr_improvement_proposal == 0: action_taken[imr] = 0 continue # Decrease the amount of resources provisioned to the NIMR for nimr in nimr_diff_proposal: action_taken[nimr] = nimr_diff_proposal[nimr] new_nimr_alloc = resource_datastore.read_mr_alloc( redis_db, nimr) + nimr_diff_proposal[nimr] print 'NIMR stealing: imposing a change of {} on {}'.format( action_taken[nimr], nimr.to_string()) finalize_mr_provision(redis_db, nimr, new_nimr_alloc) # Improving the resource should always be viable at this step if check_improve_mr_viability(redis_db, imr, imr_improvement_proposal): new_imr_alloc = imr_improvement_proposal + current_imr_alloc action_taken[imr] = imr_improvement_proposal finalize_mr_provision(redis_db, imr, new_imr_alloc) print 'Improvement Calculated: MR {} increase from {} to {}'.format( mr.to_string(), current_imr_alloc, new_imr_alloc) mimr = imr break else: action_taken[imr] = 0 print 'Improvement Calculated: MR {} failed to improve from {}'.format( mr.to_string(), current_mr_allocation) print 'This IMR cannot be improved. Printing some debugging before exiting...' print 'Current MR allocation is {}'.format(current_imr_alloc) print 'Proposed (failed) allocation is {}, improved by {}'.format( new_imr_alloc, imr_improvement_proposal) for deployment in imr.instances: vm_ip, container = deployment capacity = resource_datastore.read_machine_capacity( redis_db, vm_ip) consumption = resource_datastore.read_machine_consumption( redis_db, vm_ip) print 'Machine {} Capacity is {}, and consumption is currently {}'.format( vm_ip, capacity, consumption) if mimr is None: print 'No viable improvement found' break #Compare against the baseline at the beginning of the program improved_performance = measure_runtime(workload_config, baseline_trials) # improved_performance[preferred_performance_metric] = remove_outlier(improved_performance[preferred_performance_metric]) improved_mean = mean_list( improved_performance[preferred_performance_metric]) previous_mean = mean_list( current_performance[preferred_performance_metric]) performance_improvement = improved_mean - previous_mean # Write a summary of the experiment's iterations to Redis tbot_datastore.write_summary_redis(redis_db, experiment_count, mimr, performance_improvement, action_taken, analytic_mean, improved_mean, time_delta.seconds, cumulative_mr_count) current_performance = improved_performance # Generating overall performance improvement chart_generator.get_summary_performance_charts(redis_db, workload_config, experiment_count, time_start) results = tbot_datastore.read_summary_redis(redis_db, experiment_count) print 'Results from iteration {} are {}'.format( experiment_count, results) # Checkpoint MR configurations and print current_mr_config = resource_datastore.read_all_mr_alloc(redis_db) print_csv_configuration(current_mr_config) experiment_count += 1 print '{} experiments completed'.format(experiment_count) print_all_steps(redis_db, experiment_count) current_mr_config = resource_datastore.read_all_mr_alloc(redis_db) for mr in current_mr_config: print '{} = {}'.format(mr.to_string(), current_mr_config[mr]) print_csv_configuration(current_mr_config)
def generate_mr_from_hashkey(redis_db, hashkey): _,service_name,resource,_ = hashkey.split(',') deployments = read_service_locations(redis_db, service_name) return MR(service_name, resource, deployments)
imr_aware_service_placement = imr_aware(service_containers, optimal_num_machines) if imr_aware_service_placement is None: imr_aware_service_placement = first_fit_placements while len(imr_aware_service_placement.keys()) == 0: optimal_num_machines += 1 imr_aware_service_placement = imr_aware(service_containers, optimal_num_machines) if len(imr_aware_service_placement.keys()) != 0: break logging.info('IMR Aware Service Placement is {}'.format( imr_aware_service_placement)) return first_fit_placements, imr_aware_service_placement if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--resource_csv", help='resource configuration file') parser.add_argument("--instance_type", help='instance type') args = parser.parse_args() allocations = parse_config_csv(args.resource_csv) # Must re-select this imr_list = [MR('node-app:node-todo.git', 'CPU-QUOTA', None)] ffp, iap = ffd_pack(allocations, args.instance_type, imr_list[0].resource, imr_list) print ffp print iap