def init_resource_config(redis_db, default_mr_config, machine_type):
print 'Initializing the Resource Configurations in the containers'
instance_specs = get_instance_specs(machine_type)
for mr in default_mr_config:
weight_change = default_mr_config[mr]
new_resource_provision = convert_percent_to_raw(
mr, instance_specs[mr.resource], weight_change)
# Enact the change in resource provisioning
set_mr_provision(mr, new_resource_provision)
# Reflect the change in Redis
resource_datastore.write_mr_alloc(redis_db, mr, new_resource_provision)
update_machine_consumption(redis_db, mr, new_resource_provision, 0)
def init_resource_config(redis_db, default_mr_config, machine_type):
print 'Initializing the Resource Configurations in the containers'
instance_specs = get_instance_specs(machine_type)
for mr in default_mr_config:
new_resource_provision = default_mr_config[mr]
if check_improve_mr_viability(redis_db, mr,
new_resource_provision) is False:
print 'Initial Resource provisioning for {} is too much. Exiting...'.format(
mr.to_string())
exit()
# Enact the change in resource provisioning
resource_modifier.set_mr_provision(mr, new_resource_provision)
# Reflect the change in Redis
resource_datastore.write_mr_alloc(redis_db, mr, new_resource_provision)
update_machine_consumption(redis_db, mr, new_resource_provision, 0)
def finalize_mr_provision(redis_db, mr, new_alloc):
resource_modifier.set_mr_provision(mr, new_alloc)
old_alloc = resource_datastore.read_mr_alloc(redis_db, mr)
resource_datastore.write_mr_alloc(redis_db, mr, new_alloc)
update_machine_consumption(redis_db, mr, new_alloc, old_alloc)
def run(system_config, workload_config, default_mr_config):
redis_host = system_config['redis_host']
baseline_trials = system_config['baseline_trials']
experiment_trials = system_config['trials']
stress_weights = system_config['stress_weights']
stress_policy = system_config['stress_policy']
resource_to_stress = system_config['stress_these_resources']
service_to_stress = system_config['stress_these_services']
vm_to_stress = system_config['stress_these_machines']
machine_type = system_config['machine_type']
quilt_overhead = system_config['quilt_overhead']
preferred_performance_metric = workload_config['tbot_metric']
optimize_for_lowest = workload_config['optimize_for_lowest']
redis_db = redis.StrictRedis(host=redis_host, port=6379, db=0)
redis_db.flushall()
# Initialize Redis and Cluster based on the default resource configuration
init_cluster_capacities_r(redis_db, machine_type, quilt_overhead)
init_service_placement_r(redis_db, default_mr_config)
init_resource_config(redis_db, default_mr_config, machine_type)
# Run the baseline experiment
experiment_count = 0
baseline_performance = measure_baseline(workload_config, baseline_trials)
# Initialize the current configurations
# Invariant: MR are the same between iterations
current_mr_config = resource_datastore.read_all_mr_alloc(redis_db)
while experiment_count < 10:
# Get a list of MRs to stress in the form of a list of MRs
mr_to_stress = generate_mr_from_policy(redis_db, stress_policy)
print mr_to_stress
for mr in mr_to_stress:
print 'Current MR is {}'.format(mr.to_string())
increment_to_performance = {}
current_mr_allocation = resource_datastore.read_mr_alloc(
redis_db, mr)
print 'Current MR allocation is {}'.format(current_mr_allocation)
for stress_weight in stress_weights:
new_alloc = convert_percent_to_raw(mr, current_mr_allocation,
stress_weight)
set_mr_provision(mr, new_alloc)
experiment_results = measure_runtime(workload_config,
experiment_trials)
#Write results of experiment to Redis
mean_result = float(
sum(experiment_results[preferred_performance_metric])
) / len(experiment_results[preferred_performance_metric])
tbot_datastore.write_redis_ranking(
redis_db, experiment_count, preferred_performance_metric,
mean_result, mr, stress_weight)
# Remove the effect of the resource stressing
new_alloc = convert_percent_to_raw(mr, current_mr_allocation,
0)
increment_to_performance[stress_weight] = experiment_results
# Write the results of the iteration to Redis
tbot_datastore.write_redis_results(redis_db, mr,
increment_to_performance,
experiment_count,
preferred_performance_metric)
# Recover the results of the experiment from Redis
max_stress_weight = min(stress_weights)
mimr_list = tbot_datastore.get_top_n_mimr(
redis_db,
experiment_count,
preferred_performance_metric,
max_stress_weight,
optimize_for_lowest=optimize_for_lowest,
num_results_returned=10)
# Try all the MIMRs in the list until a viable improvement is determined
# Improvement Amount
mimr = None
action_taken = 0
print 'The MR improvement is {}'.format(max_stress_weight)
for mr_score in mimr_list:
mr, score = mr_score
improvement_percent = improve_mr_by(redis_db, mr,
max_stress_weight)
current_mr_allocation = resource_datastore.read_mr_alloc(
redis_db, mr)
new_alloc = convert_percent_to_raw(mr, current_mr_allocation,
improvement_percent)
improvement_amount = new_alloc - current_mr_allocation
action_taken = improvement_amount
if check_improve_mr_viability(redis_db, mr, improvement_amount):
set_mr_provision(mr, new_alloc)
print 'Improvement Calculated: MR {} increase from {} to {}'.format(
mr.to_string(), current_mr_allocation, new_alloc)
old_alloc = resource_datastore.read_mr_alloc(redis_db, mr)
resource_datastore.write_mr_alloc(redis_db, mr, new_alloc)
update_machine_consumption(redis_db, mr, new_alloc, old_alloc)
current_mr_config = update_mr_config(redis_db,
current_mr_config)
mimr = mr
break
else:
print 'Improvement Calculated: MR {} failed to improve from {} to {}'.format(
mr.to_string(), current_mr_allocation, new_alloc)
if mimr is None:
print 'No viable improvement found'
break
#Compare against the baseline at the beginning of the program
improved_performance = measure_runtime(workload_config,
baseline_trials)
print improved_performance
improved_mean = sum(
improved_performance[preferred_performance_metric]) / float(
len(improved_performance[preferred_performance_metric]))
baseline_mean = sum(
baseline_performance[preferred_performance_metric]) / float(
len(baseline_performance[preferred_performance_metric]))
performance_improvement = improved_mean - baseline_mean
# Write a summary of the experiment's iterations to Redis
tbot_datastore.write_summary_redis(redis_db, experiment_count, mimr,
performance_improvement,
action_taken)
baseline_performance = improved_performance
results = tbot_datastore.read_summary_redis(redis_db, experiment_count)
print 'Results from iteration {} are {}'.format(
experiment_count, results)
experiment_count += 1
# TODO: Handle False Positive
# TODO: Compare against performance condition -- for now only do some number of experiments
print '{} experiments completed'.format(experiment_count)
print_all_steps(redis_db, experiment_count)
for mr in current_mr_config:
print '{} = {}'.format(mr.to_string(), current_mr_config[mr])