def scramble_tiers(tier_resources, times):
def permute(a, l, r):
if l == r:
yield a
else:
for i in xrange(l, r + 1):
a[l], a[i] = a[i], a[l]
for p in permute(a, l + 1, r):
yield p
a[l], a[i] = a[i], a[l]
def generate_splits(total):
splits = [1.0 / 6.0, 1.0 / 3.0, 1.0 / 2.0]
orderings = []
for permutation in permute(splits, 0, 2):
orderings.append([total * x for x in permutation])
return orderings
splits = generate_splits(1)
done = []
t = 0
while t < times:
not_changed = random.choice(resources)
key = []
for r in resources:
if r != not_changed:
split = random.choice(splits)
for i in range(1, 4):
tier_resources[MR(
i, r,
[])] = split[i - 1] * tier_resources[MR(0, r, [])]
key.append((r, split, []))
if key not in done:
t += 1
yield tier_resources.copy()
def tier_to_configuration(tier_resources):
config = {}
for i in range(1, 4):
for r in resources:
value = tier_resources[MR(i, r, [])]
for service in tiers[i]:
config[MR(service, r, [])] = value
return config
def write_config(title, tier_resources, f):
f.write("{}\n".format(title))
for i in range(1, 4):
f.write(" Tier {}: {}\n".format(i, tiers[i]))
for r in resources:
f.write(" {}: {}\n".format(r,
tier_resources[MR(i, r,
[])]))
def mr_str_to_obj(redis_db, mr_list):
mr_object_list = []
for mr in mr_list:
service_name, resource = mr.split(',')
deployments = tbot_datastore.read_service_locations(
redis_db, service_name)
mr_object_list.append(MR(service_name, resource, deployments))
return mr_object_list
def parse_resource_config_file(resource_config_csv, sys_config):
machine_type = sys_config['machine_type']
vm_list = get_actual_vms()
service_placements = get_service_placements(vm_list)
all_services = get_actual_services()
all_resources = get_stressable_resources()
mr_allocation = {}
# Empty Config means that we should default resource allocation to only use
# half of the total resource capacity on the machine
if resource_config_csv is None:
vm_to_service = get_vm_to_service(vm_list)
# DEFAULT_ALLOCATION sets the initial configuration
# Ensure that we will not violate resource provisioning in the machine
# Assign resources equally to services without exceeding machine resource limitations
max_num_services = 0
for vm in vm_to_service:
if len(vm_to_service[vm]) > max_num_services:
max_num_services = len(vm_to_service[vm])
default_alloc_percentage = 70.0 / max_num_services
mr_list = get_all_mrs_cluster(vm_list, all_services, all_resources)
for mr in mr_list:
max_capacity = get_instance_specs(machine_type)[mr.resource]
default_raw_alloc = (default_alloc_percentage /
100.0) * max_capacity
mr_allocation[mr] = default_raw_alloc
print mr_allocation
else:
# Manual Configuration Possible
# Parse a CSV
# Format of resource allocation: SERVICE,RESOURCE,TYPE,REPR
mr_list = get_all_mrs_cluster(vm_list, all_services, all_resources)
with open(resource_config_csv, 'rb') as resource_config:
reader = csv.DictReader(resource_config)
for row in reader:
service_name = row['SERVICE']
resource = row['RESOURCE']
amount = float(row['AMOUNT'])
amount_repr = row['REPR']
# Convert REPR to RAW AMOUNT
if amount_repr == 'PERCENT':
if amount <= 0 or amount > 100:
print 'Error: invalid default percentage. Exiting...'
exit()
max_capacity = get_instance_specs(machine_type)[resource]
amount = (amount / 100.0) * max_capacity
mr = MR(service_name, resource,
service_placements[service_name])
assert mr in mr_list
mr_allocation[mr] = amount
return mr_allocation
def combine_resources(mr_allocation,
imr_list,
instance_type,
resource_type='CPU-QUOTA'):
vm_list = get_actual_vms()
vm_to_service = get_vm_to_service(vm_list)
imr_service_names = [imr.service_name for imr in imr_list]
resource_capacity = get_instance_specs(instance_type)
single_core_capacity = 100
all_deploy_sets = []
for vm in vm_to_service.keys():
deploy_together = []
resource_usage = 0
# First determine if an IMR has been placed on the machine
# Assume only one IMR per machine
for service_name in vm_to_service[vm]:
if service_name in imr_service_names:
resource_usage += mr_allocation[MR(service_name, resource_type,
None)]
deploy_together.append(MR(service_name, resource_type, None))
break
# Next, look at the remianing services
for service_name in vm_to_service[vm]:
if service_name in imr_service_names:
continue
relevant_mr = MR(service_name, resource_type, None)
if mr_allocation[
relevant_mr] + resource_usage < single_core_capacity:
resource_usage += mr_allocation[relevant_mr]
deploy_together.append(MR(service_name, resource_type, None))
all_deploy_sets.append(deploy_together)
# Can't fill up remaining resources because it is no guarantee that other
# instances will be able to sustain same resource growths.
return all_deploy_sets
def get_all_mrs(redis_db):
mr_name = 'mr_alloc'
all_mrs = redis_db.hgetall(mr_name)
mr_list = list(all_mrs.keys())
mr_object_list = []
for mr in mr_list:
service_name, resource = mr.split(',')
deployments = tbot_datastore.read_service_locations(
redis_db, service_name)
mr_object_list.append(MR(service_name, resource, deployments))
return mr_object_list
def identify_tier_resources(mr_allocation):
tier_resources = {}
for mr, value in mr_allocation.items():
assert mr.resource in resources
for tier, lst in tiers.items():
if mr.service_name in lst:
tier_mr = MR(tier, mr.resource, [])
if tier_mr in tier_resources:
assert tier_resources[tier_mr] == value
else:
tier_resources[tier_mr] = value
for resource in resources:
total = 0
for i in range(1, 4):
total += tier_resources[MR(i, resource, [])]
tier_resources[MR(0, resource, [])] = total
return tier_resources
def get_all_mrs_cluster(vm_list, services, resources):
mr_schedule = []
service_to_deployment = get_service_placements(vm_list)
for service in service_to_deployment:
if service not in services:
continue
deployments = service_to_deployment[service]
for resource in resources:
mr_schedule.append(MR(service, resource, deployments))
return mr_schedule
def get_all_mrs_cluster(vm_list, services, resources):
mr_schedule = []
service_to_deployment = get_service_placements(vm_list)
logging.info('printing service to deployment {}'.format(service_to_deployment))
for service in service_to_deployment:
deployments = service_to_deployment[service]
for resource in resources:
mr_schedule.append(MR(service, resource, deployments))
logging.info('mr schedule is {}'.format(mr_schedule))
return mr_schedule
def read_all_mr_alloc(redis_db):
mr_name = 'mr_alloc'
mr_to_score = redis_db.hgetall(mr_name)
for mr in mr_to_score:
mr_to_score[mr] = float(mr_to_score[mr])
mr_allocation_list = {}
for mr in mr_to_score:
service_name, resource = mr.split(',')
deployments = tbot_datastore.read_service_locations(
redis_db, service_name)
mr_object = MR(service_name, resource, deployments)
mr_allocation_list[mr_object] = mr_to_score[mr]
return mr_allocation_list
def parse_config_csv(resource_config_csv):
mr_allocation = {}
with open(resource_config_csv, 'rb') as resource_config:
reader = csv.DictReader(resource_config)
for row in reader:
service_name = row['SERVICE']
resource = row['RESOURCE']
amount = float(row['AMOUNT'])
amount_repr = row['REPR']
# Convert REPR to RAW AMOUNT
if amount_repr == 'RAW':
mr = MR(service_name, resource, None)
mr_allocation[mr] = amount
return mr_allocation
def ffd_pack(past_mr_allocation,
instance_type,
sort_by='CPU-QUOTA',
imr_list=[],
deploy_together=[],
round_up=False):
mr_allocation = deepcopy(past_mr_allocation)
if round_up is True:
for imr in imr_list:
assert imr in mr_allocation
mr_allocation[imr] = roundup(mr_allocation[imr])
# Use the below if you have already deployed the containers on quilt
# Otherwise, define it manually.
vm_list = get_actual_vms()
service_placements = get_service_placements(vm_list)
service_configurations = [
mr.to_string().split(',')[0] for mr in mr_allocation.keys()
]
service_containers = []
for service_name in service_placements:
if service_name in service_configurations:
service_containers += [
service_name
for x in range(len(service_placements[service_name]))
]
for affinity in deploy_together:
for mr in affinity:
service_containers.remove(mr.service_name)
# Create a fake MR with the combination of MRs
for affinity in deploy_together:
affinity_services = ''
total_resources = 0
for mr in affinity:
affinity_services += '&' + mr.service_name
total_resources += mr_allocation[mr]
mr_allocation[MR(affinity_services, 'CPU-QUOTA',
None)] = roundup(total_resources)
service_containers.append(affinity_services)
service_to_mr = set_service_specs(
service_containers, mr_allocation) # Amend to include affinities
# IMR Aware Scheduling
def imr_aware(sc, num_machines):
sc_copy = deepcopy(sc)
if num_machines == 1:
return None
instance_specs = get_instance_specs(instance_type)
machine_to_allocation = {}
for x in range(num_machines):
machine_to_allocation = initialize_capacity(
machine_to_allocation, x, instance_specs)
machine_to_service = {}
for x in range(num_machines):
machine_to_service[x] = []
resource_to_impacted_service = {}
service_encountered = []
# Find the IMRs and place them
for mr in imr_list:
if mr.service_name in service_encountered:
continue
else:
service_encountered.append(mr.service_name)
if mr.resource not in resource_to_impacted_service:
resource_to_impacted_service[mr.resource] = []
resource_to_impacted_service[mr.resource].append(mr.service_name)
# First start by round robin the MIMR and other MRs that are of the same resource
mimr_resource = imr_list[0].resource
service_list = resource_to_impacted_service[mimr_resource]
containers_seen = 0
for service_name in service_list:
num_service_containers = sc_copy.count(service_name)
mr_list = service_to_mr[service_name]
# Round robin the placements
for container_index in range(num_service_containers):
machine_index = (containers_seen +
container_index) % num_machines
fit_found = place_if_possible(mr_list, mr_allocation,
machine_to_allocation,
machine_index, instance_specs)
if fit_found:
machine_to_service[machine_index].append(service_name)
containers_seen += 1
else:
logging.error(
'You have selected too many IMRs for consideration!')
return {}
# Don't double count the services
for service_name in service_list:
sc_copy = filter(lambda a: a != service_name, sc_copy)
# Reverse the order of the services packed into the machine
new_machine_to_allocation = {}
new_machine_to_service = {}
for machine_index in machine_to_allocation:
new_machine_to_allocation[num_machines - machine_index -
1] = machine_to_allocation[machine_index]
new_machine_to_service[num_machines - machine_index -
1] = machine_to_service[machine_index]
machine_to_allocation = new_machine_to_allocation
machine_to_service = new_machine_to_service
# Then do a first fit with any remaining MR.
for service_name in sc_copy:
mr_list = service_to_mr[service_name]
fit_found, machine_index = update_if_possible(
mr_list, mr_allocation, machine_to_allocation, num_machines,
instance_specs)
if fit_found:
machine_to_service[machine_index].append(service_name)
else:
logging.error('No valid placement found. Returning')
return {}
return machine_to_service
def ff(sc):
instance_specs = get_instance_specs(instance_type)
machine_to_allocation = {}
machine_to_allocation = initialize_capacity(machine_to_allocation, 0,
instance_specs)
machine_to_service = {}
machine_to_service[0] = []
number_machines = 1
for service_name in sc:
mr_list = service_to_mr[service_name]
fit_found, machine_index = update_if_possible(
mr_list, mr_allocation, machine_to_allocation, number_machines,
instance_specs)
if fit_found:
machine_to_service[machine_index].append(service_name)
if fit_found is False:
number_machines += 1
machine_to_allocation[number_machines - 1] = {}
machine_to_service[number_machines - 1] = []
initialize_capacity(machine_to_allocation, number_machines - 1,
instance_specs)
fit_found, machine_index = update_if_possible(
mr_list, mr_allocation, machine_to_allocation,
number_machines, instance_specs)
machine_to_service[machine_index].append(service_name)
if fit_found is False:
logging.error('Fit not found. You have a bug. Exiting..')
exit()
return machine_to_service
resource_index = {}
for service in service_to_mr:
for x in range(len(service_to_mr[service])):
resource_index[service_to_mr[service][x].resource] = x
# First find the number of machines from first fit
imr_resource = imr_list[0].resource
service_containers = sorted(service_containers,
key=lambda x: mr_allocation[service_to_mr[x][
resource_index[imr_resource]]])
print service_containers
first_fit_placements = ff(service_containers)
print first_fit_placements
logging.info(
'First fit service placement is {}'.format(first_fit_placements))
# First place the services that show up as MIMRs
optimal_num_machines = len(first_fit_placements.keys())
imr_aware_service_placement = {}
imr_aware_service_placement = imr_aware(service_containers,
optimal_num_machines)
if imr_aware_service_placement is None:
imr_aware_service_placement = first_fit_placements
while len(imr_aware_service_placement.keys()) == 0:
optimal_num_machines += 1
imr_aware_service_placement = imr_aware(service_containers,
optimal_num_machines)
if len(imr_aware_service_placement.keys()) != 0:
break
logging.info('IMR Aware Service Placement is {}'.format(
imr_aware_service_placement))
return first_fit_placements, imr_aware_service_placement
service_containers = sorted(service_containers,
key=lambda x: mr_allocation[service_to_mr[x][resource_index[imr_resource]]])
first_fit_placements = ff(service_containers)
logging.info('First fit service placement is {}'.format(first_fit_placements))
# First place the services that show up as MIMRs
optimal_num_machines = len(first_fit_placements.keys())
imr_aware_service_placement = {}
imr_aware_service_placement = imr_aware(service_containers, optimal_num_machines)
while len(imr_aware_service_placement.keys()) == 0:
optimal_num_machines += 1
imr_aware_service_placement = imr_aware(service_containers, optimal_num_machines)
if len(imr_aware_service_placement.keys()) != 0:
break
logging.info('IMR Aware Service Placement is {}'.format(imr_aware_service_placement))
return first_fit_placements, imr_aware_service_placement
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--resource_csv", help='resource configuration file')
parser.add_argument("--instance_type", help='instance type')
args = parser.parse_args()
allocations = parse_config_csv(args.resource_csv)
# Must re-select this
imr_list = [MR('tsaianson/node-apt-app', 'CPU-QUOTA', None), MR('haproxy:1.7', 'NET', None), MR('haproxy:1.7', 'NET', None)]
ffp, iap = ffd_pack(allocations, args.instance_type, imr_list[0].resource, imr_list)
def run(sys_config,
workload_config,
filter_config,
default_mr_config,
last_completed_iter=0):
redis_host = sys_config['redis_host']
baseline_trials = sys_config['baseline_trials']
experiment_trials = sys_config['trials']
stress_weights = sys_config['stress_weights']
stress_policy = sys_config['stress_policy']
resource_to_stress = sys_config['stress_these_resources']
service_to_stress = sys_config['stress_these_services']
vm_to_stress = sys_config['stress_these_machines']
machine_type = sys_config['machine_type']
quilt_overhead = sys_config['quilt_overhead']
gradient_mode = sys_config['gradient_mode']
preferred_performance_metric = workload_config['tbot_metric']
optimize_for_lowest = workload_config['optimize_for_lowest']
redis_db = redis.StrictRedis(host=redis_host, port=6379, db=0)
if last_completed_iter == 0:
redis_db.flushall()
'''
# Prompt the user to make sure they want to flush the db
ok_to_flush = raw_input("Are you sure you want to flush the results of your last experiment? Please respond with Y or N: ")
if ok_to_flush == 'Y':
redis_db.flushall()
elif ok_to_flush == 'N':
print 'OK you said it boss. Exiting...'
exit()
else:
print 'Only Y and N are acceptable responses. Exiting...'
exit()
'''
print '\n' * 2
print '*' * 20
print 'INFO: INITIALIZING RESOURCE CONFIG'
# Initialize Redis and Cluster based on the default resource configuration
init_cluster_capacities_r(redis_db, machine_type, quilt_overhead)
init_service_placement_r(redis_db, default_mr_config)
init_resource_config(redis_db, default_mr_config, machine_type)
print '*' * 20
print 'INFO: INSTALLING DEPENDENCIES'
#install_dependencies(workload_config)
# Initialize time for data charts
time_start = datetime.datetime.now()
print '*' * 20
print 'INFO: RUNNING BASELINE'
# Get the Current Performance -- not used for any analysis, just to benchmark progress!!
current_performance = measure_baseline(workload_config, baseline_trials)
current_performance[preferred_performance_metric] = remove_outlier(
current_performance[preferred_performance_metric])
current_time_stop = datetime.datetime.now()
time_delta = current_time_stop - time_start
print 'Current (non-analytic) performance measured: {}'.format(
current_performance)
if last_completed_iter != 0:
tbot_datastore.write_summary_redis(
redis_db, 0, MR('initial', 'initial', []), 0, {},
mean_list(current_performance[preferred_performance_metric]),
mean_list(current_performance[preferred_performance_metric]),
time_delta.seconds, 0)
print '============================================'
print '\n' * 2
# Initialize the current configurations
# Initialize the working set of MRs to all the MRs
mr_working_set = resource_datastore.get_all_mrs(redis_db)
resource_datastore.write_mr_working_set(redis_db, mr_working_set, 0)
cumulative_mr_count = 0
experiment_count = last_completed_iter + 1
while experiment_count < 10:
# Calculate the analytic baseline that is used to determine MRs
analytic_provisions = prepare_analytic_baseline(
redis_db, sys_config, min(stress_weights))
print 'The Analytic provisions are as follows {}'.format(
analytic_provisions)
for mr in analytic_provisions:
resource_modifier.set_mr_provision(mr, analytic_provisions[mr])
analytic_baseline = measure_runtime(workload_config, experiment_trials)
analytic_mean = mean_list(
analytic_baseline[preferred_performance_metric])
print 'The analytic baseline is {}'.format(analytic_baseline)
print 'This current performance is {}'.format(current_performance)
analytic_baseline[preferred_performance_metric] = remove_outlier(
analytic_baseline[preferred_performance_metric])
# Get a list of MRs to stress in the form of a list of MRs
mr_to_consider = apply_filtering_policy(redis_db, mr_working_set,
experiment_count, sys_config,
workload_config, filter_config)
for mr in mr_to_consider:
print '\n' * 2
print '*' * 20
print 'Current MR is {}'.format(mr.to_string())
increment_to_performance = {}
current_mr_allocation = resource_datastore.read_mr_alloc(
redis_db, mr)
print 'Current MR allocation is {}'.format(current_mr_allocation)
for stress_weight in stress_weights:
# Calculate Gradient Schedule and provision resources accordingly
mr_gradient_schedule = calculate_mr_gradient_schedule(
redis_db, [mr], sys_config, stress_weight)
for change_mr in mr_gradient_schedule:
resource_modifier.set_mr_provision(
change_mr, mr_gradient_schedule[change_mr])
experiment_results = measure_runtime(workload_config,
experiment_trials)
# Write results of experiment to Redis
# preferred_results = remove_outlier(experiment_results[preferred_performance_metric])
preferred_results = experiment_results[
preferred_performance_metric]
mean_result = mean_list(preferred_results)
tbot_datastore.write_redis_ranking(
redis_db, experiment_count, preferred_performance_metric,
mean_result, mr, stress_weight)
# Revert the Gradient schedule and provision resources accordingly
mr_revert_gradient_schedule = revert_mr_gradient_schedule(
redis_db, [mr], sys_config, stress_weight)
for change_mr in mr_revert_gradient_schedule:
resource_modifier.set_mr_provision(
change_mr, mr_revert_gradient_schedule[change_mr])
increment_to_performance[stress_weight] = experiment_results
# Write the results of the iteration to Redis
tbot_datastore.write_redis_results(redis_db, mr,
increment_to_performance,
experiment_count,
preferred_performance_metric)
print '*' * 20
print '\n' * 2
# Timing Information for the purpose of experiments
current_time_stop = datetime.datetime.now()
time_delta = current_time_stop - time_start
cumulative_mr_count += len(mr_to_consider)
chart_generator.get_summary_mimr_charts(
redis_db, workload_config, current_performance, mr_working_set,
experiment_count, stress_weights, preferred_performance_metric,
time_start)
# Move back into the normal operating basis by removing the baseline prep stresses
reverted_analytic_provisions = revert_analytic_baseline(
redis_db, sys_config)
for mr in reverted_analytic_provisions:
resource_modifier.set_mr_provision(
mr, reverted_analytic_provisions[mr])
# Recover the results of the experiment from Redis
max_stress_weight = min(stress_weights)
mimr_list = tbot_datastore.get_top_n_mimr(
redis_db,
experiment_count,
preferred_performance_metric,
max_stress_weight,
gradient_mode,
optimize_for_lowest=optimize_for_lowest,
num_results_returned=-1)
imr_list, nimr_list = seperate_mr(
mimr_list,
mean_list(analytic_baseline[preferred_performance_metric]),
optimize_for_lowest)
if len(imr_list) == 0:
print 'INFO: IMR list length is 0. Please choose a metric with more signal. Exiting...'
break
print 'INFO: IMR list is {}'.format(
[mr.to_string() for mr in imr_list])
print 'INFO: NIMR list is {}'.format(
[mr.to_string() for mr in nimr_list])
# Try all the MIMRs in the list until a viable improvement is determined
# Improvement Amount
mimr = None
action_taken = {}
for imr in imr_list:
imr_improvement_percent = improve_mr_by(redis_db, imr,
max_stress_weight)
current_imr_alloc = resource_datastore.read_mr_alloc(redis_db, imr)
new_imr_alloc = convert_percent_to_raw(imr, current_imr_alloc,
imr_improvement_percent)
imr_improvement_proposal = new_imr_alloc - current_imr_alloc
# If the the Proposed MR cannot be improved by the proposed amount, there are two options
# - Max out the resources to fill up the remaining resources on the machine
# - Resource Stealing from NIMRs
# Both functions will return VIABLE improvements to the IMR deployment
nimr_diff_proposal = {}
if check_improve_mr_viability(redis_db, imr,
imr_improvement_proposal) is False:
print 'INFO: MR {} to increase {} by {} is not viable'.format(
imr.to_string(), current_imr_alloc,
imr_improvement_proposal)
print 'INFO: Attempting to max out the machines resources...'
imr_improvement_proposal = fill_out_resource(redis_db, imr)
if imr_improvement_proposal <= 0:
print 'INFO: No more space to fill out resources. Stealing from NIMRs'
# Calculate a plan to reduce the resource provisioning of NIMRs
nimr_diff_proposal, imr_improvement_proposal = create_decrease_nimr_schedule(
redis_db, imr, nimr_list, max_stress_weight)
print 'INFO: Proposed NIMR {}'.format(nimr_diff_proposal)
print 'INFO: New IMR improvement {}'.format(
imr_improvement_proposal)
if len(nimr_diff_proposal
) == 0 or imr_improvement_proposal == 0:
action_taken[imr] = 0
continue
# Decrease the amount of resources provisioned to the NIMR
for nimr in nimr_diff_proposal:
action_taken[nimr] = nimr_diff_proposal[nimr]
new_nimr_alloc = resource_datastore.read_mr_alloc(
redis_db, nimr) + nimr_diff_proposal[nimr]
print 'NIMR stealing: imposing a change of {} on {}'.format(
action_taken[nimr], nimr.to_string())
finalize_mr_provision(redis_db, nimr, new_nimr_alloc)
# Improving the resource should always be viable at this step
if check_improve_mr_viability(redis_db, imr,
imr_improvement_proposal):
new_imr_alloc = imr_improvement_proposal + current_imr_alloc
action_taken[imr] = imr_improvement_proposal
finalize_mr_provision(redis_db, imr, new_imr_alloc)
print 'Improvement Calculated: MR {} increase from {} to {}'.format(
mr.to_string(), current_imr_alloc, new_imr_alloc)
mimr = imr
break
else:
action_taken[imr] = 0
print 'Improvement Calculated: MR {} failed to improve from {}'.format(
mr.to_string(), current_mr_allocation)
print 'This IMR cannot be improved. Printing some debugging before exiting...'
print 'Current MR allocation is {}'.format(current_imr_alloc)
print 'Proposed (failed) allocation is {}, improved by {}'.format(
new_imr_alloc, imr_improvement_proposal)
for deployment in imr.instances:
vm_ip, container = deployment
capacity = resource_datastore.read_machine_capacity(
redis_db, vm_ip)
consumption = resource_datastore.read_machine_consumption(
redis_db, vm_ip)
print 'Machine {} Capacity is {}, and consumption is currently {}'.format(
vm_ip, capacity, consumption)
if mimr is None:
print 'No viable improvement found'
break
#Compare against the baseline at the beginning of the program
improved_performance = measure_runtime(workload_config,
baseline_trials)
# improved_performance[preferred_performance_metric] = remove_outlier(improved_performance[preferred_performance_metric])
improved_mean = mean_list(
improved_performance[preferred_performance_metric])
previous_mean = mean_list(
current_performance[preferred_performance_metric])
performance_improvement = improved_mean - previous_mean
# Write a summary of the experiment's iterations to Redis
tbot_datastore.write_summary_redis(redis_db, experiment_count, mimr,
performance_improvement,
action_taken, analytic_mean,
improved_mean, time_delta.seconds,
cumulative_mr_count)
current_performance = improved_performance
# Generating overall performance improvement
chart_generator.get_summary_performance_charts(redis_db,
workload_config,
experiment_count,
time_start)
results = tbot_datastore.read_summary_redis(redis_db, experiment_count)
print 'Results from iteration {} are {}'.format(
experiment_count, results)
# Checkpoint MR configurations and print
current_mr_config = resource_datastore.read_all_mr_alloc(redis_db)
print_csv_configuration(current_mr_config)
experiment_count += 1
print '{} experiments completed'.format(experiment_count)
print_all_steps(redis_db, experiment_count)
current_mr_config = resource_datastore.read_all_mr_alloc(redis_db)
for mr in current_mr_config:
print '{} = {}'.format(mr.to_string(), current_mr_config[mr])
print_csv_configuration(current_mr_config)
def generate_mr_from_hashkey(redis_db, hashkey):
_,service_name,resource,_ = hashkey.split(',')
deployments = read_service_locations(redis_db, service_name)
return MR(service_name, resource, deployments)
imr_aware_service_placement = imr_aware(service_containers,
optimal_num_machines)
if imr_aware_service_placement is None:
imr_aware_service_placement = first_fit_placements
while len(imr_aware_service_placement.keys()) == 0:
optimal_num_machines += 1
imr_aware_service_placement = imr_aware(service_containers,
optimal_num_machines)
if len(imr_aware_service_placement.keys()) != 0:
break
logging.info('IMR Aware Service Placement is {}'.format(
imr_aware_service_placement))
return first_fit_placements, imr_aware_service_placement
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--resource_csv", help='resource configuration file')
parser.add_argument("--instance_type", help='instance type')
args = parser.parse_args()
allocations = parse_config_csv(args.resource_csv)
# Must re-select this
imr_list = [MR('node-app:node-todo.git', 'CPU-QUOTA', None)]
ffp, iap = ffd_pack(allocations, args.instance_type, imr_list[0].resource,
imr_list)
print ffp
print iap