def test_nodes(self, active_nodes, new_domain):
# Go over all hosts
for node in active_nodes:
# Get actual CPU measurements
curr_cpu_demand = np.percentile(node.get_readings(), 95)
# Memory demand is calculated by summing up all VM reservations
curr_mem_demand = 0
# Add up the demands of all domains running on the host
for old_domain in node.domains.values():
# Domain size specification
spec = old_domain.domain_configuration.get_domain_spec()
# Sum up mem load
curr_mem_demand += spec.total_memory()
# Calculate metrics
new_domain_spec = conf_domainsize.get_domain_spec(new_domain.size)
mem_delta = conf_nodes.NODE_MEM - curr_mem_demand - new_domain_spec.total_memory()
# Calculate estiated CPU demand if VM is almost
vm_cpu_demand = conf_nodes.to_node_load(95, new_domain.size)
cpu_delta = conf_nodes.UTIL - curr_cpu_demand - vm_cpu_demand
# If metric is positive, the node can host the domain
if cpu_delta >= 0 and mem_delta >= 0:
return node.name
def __build_modified_profiles(mix):
for handle in mix.handles:
print 'processing modified profile %s ...' % (handle.name)
# Load TS
ts_name = wmeta.times_name(handle, wmeta.MUTATION_PNORM, mix)
# Modify CPU normal profile
profile = profile_modifier.process_trace(tc.connection(), ts_name,
handle.modifier, handle.additive,
handle.scale, handle.shift)
# Attach profile to handle
handle.profile_frequency, handle.profile = profile
# Store profiles
for handle in mix.handles:
profile = np.array(handle.profile)
profile_frequency = handle.profile_frequency
for size in xrange(conf_domainsize.count_domain_sizes()):
size = conf_domainsize.get_domain_spec(size).max_user_demand()
user_profile = (profile / 100.0) * size.users
profile_frequency = profile_frequency / (wmeta.CYCLE_TIME / schedule_builder.EXPERIMENT_DURATION)
user_profile = np.array(user_profile)
user_profile += 5
if not DRY_RUN:
print 'storing modified profile %s ...' % (handle.name)
__write_profile(wmeta.times_name(handle, wmeta.MUTATION_PUSER, mix), user_profile,
profile_frequency)
def test_nodes(self, new_domain, node_list):
host_choice = []
for node in node_list:
# Aggregate load for the complete host
cpu_load = 0
mem_load = 0
# Calculate the node utilization by accumulating all domain loads
for dom in node.domains.values():
spec = dom.domain_configuration.get_domain_spec()
cpu_load += spec.total_cpu_cores()
mem_load += spec.total_memory()
# Calculate metric
spec = conf_domainsize.get_domain_spec(new_domain.size)
cpu_delta = conf_nodes.NODE_CPU_CORES - (cpu_load + spec.total_cpu_cores())
mem_delta = conf_nodes.NODE_MEM - (mem_load + spec.total_memory())
metric = cpu_delta * mem_delta
# Server is not able to handle the domain
if cpu_delta < 0 or mem_delta < 0:
continue
# Add metric to the choice list
host_choice.append((node.name, metric))
# Check if we found at least one host
if not host_choice:
return None
# Sort host choice list
host_choice = self.sort(host_choice, lambda x: x[1])
# Pkc hte one with the lowest metric (best fit)
return host_choice[0][0]
def __store_profile_yarns(mix, handle, normalizing_value, profile, profile_frequency):
# 1) RAW profile (e.g. for plotting)
raw_profile = np.array(profile)
if not DRY_RUN:
__write_profile(wmeta.times_name(handle, wmeta.MUTATION_PRAW, mix), raw_profile, profile_frequency)
# 2) NORMALIZED profile (normalized with the set maximum, see above) (e.g. to feed into SSAPv)
maxval = float(normalizing_value[handle.htype.id])
profile /= maxval
norm_profile = np.array(profile)
norm_profile[norm_profile > 1] = 1
norm_profile *= 100 # Times does not support float values
if not DRY_RUN:
__write_profile(wmeta.times_name(handle, wmeta.MUTATION_PNORM, mix), norm_profile, profile_frequency)
# 3) Store USER profiles (-> e.g. for Rain workload driver)
for size in xrange(conf_domainsize.count_domain_sizes()):
size = conf_domainsize.get_domain_spec(size).max_user_demand()
user_profile = profile * size.users
user_profile = np.array(user_profile)
user_profile += 5
user_profile_frequency = profile_frequency / (wmeta.CYCLE_TIME / schedule_builder.EXPERIMENT_DURATION)
user_profile_name = wmeta.times_name(handle, wmeta.MUTATION_PUSER, mix, size)
print 'FREQ - USER YARN: %s @ %f' % (user_profile_name, user_profile_frequency)
if not DRY_RUN:
__write_profile(user_profile_name, user_profile, user_profile_frequency)
def test_nodes(self, new_domain, nodelist):
norms = []
for node in nodelist:
# Aggregate load for the complete host
mem_load = 0
for dom in node.domains.values():
spec = dom.domain_configuration.get_domain_spec()
mem_load += spec.total_memory()
# Get actual CPU measurements
curr_cpu_demand = np.percentile(node.get_readings(), 95)
resd_cpu = conf_nodes.UTIL - curr_cpu_demand
# Calculate residual vector
resd_mem = conf_nodes.NODE_MEM - mem_load
# VM resource demand
spec = conf_domainsize.get_domain_spec(new_domain.size)
# Calculate estiated CPU demand if VM is almost
domain_cpu_demand = conf_nodes.to_node_load(95, new_domain.size)
domain_mem_demand = spec.total_memory()
# Calculate the norm
norm = 1 * math.pow(domain_cpu_demand - resd_cpu, 2) + 1 * math.pow(domain_mem_demand - resd_mem, 2)
# Check if this host is able to handle the new domain
cpu_delta = resd_cpu - domain_cpu_demand
mem_delta = resd_mem - domain_mem_demand
if cpu_delta >= 0 and mem_delta >= 0:
norms.append((node, norm, curr_cpu_demand, mem_load))
else:
print 'failed for node %s - status: %i mem %i cpu' % (node.name, mem_delta, cpu_delta)
# Find the node with the lowest norm that is able to host the domain
norms.sort(key=lambda x: x[1])
spec = conf_domainsize.get_domain_spec(new_domain.size)
for norm in norms:
# Node found
return norm[0].name
def __check_capacity(self, host, domain):
# Calculate total demand of all active domains running on selected node
cpu_demand = 0
mem_demand = 0
for active_domain in host.domains.values():
active_domain_configuration = active_domain.domain_configuration
active_domain_spec = domainsize.get_domain_spec(active_domain_configuration.size)
cpu_demand += active_domain_spec.total_cpu_cores()
mem_demand += active_domain_spec.total_memory()
# Domain specification of the new domain to place
domain_spec = domainsize.get_domain_spec(domain.size)
# Calculate residual capacity
cpu_demand = nodes.NODE_CPU_CORES - domain_spec.total_cpu_cores() - cpu_demand
mem_demand = nodes.NODE_MEM - domain_spec.total_memory() - mem_demand
# Get new node if current one is overloaded
return cpu_demand >= 0 and mem_demand >= 0
def placement(self, domain):
# Get list of inactive nodes
inactive_nodes = self.__get_inactive_nodes()
# Set initial active node
if self.active_node is None:
self.active_node = inactive_nodes.pop()
# Domain specification of the new domain to place
domain_spec = domainsize.get_domain_spec(domain.size)
# Calculate total demand of all active domains running on selected node
cpu_demand = 0
mem_demand = 0
for active_domain in self.active_node.domains.values():
active_domain_configuration = active_domain.domain_configuration
active_domain_spec = domainsize.get_domain_spec(active_domain_configuration.size)
cpu_demand += active_domain_spec.total_cpu_cores()
mem_demand += active_domain_spec.total_memory()
# Calculate residual capacity
cpu_demand = nodes.NODE_CPU_CORES - domain_spec.total_cpu_cores() - cpu_demand
mem_demand = nodes.NODE_MEM - domain_spec.total_memory() - mem_demand
# Get new node if current one is overloaded
if cpu_demand < 0 or mem_demand < 0:
try:
self.active_node = inactive_nodes.pop()
except:
print 'inactive nodes length: %i' % len(inactive_nodes)
self.model.dump()
print 'PROBLEM IN SCHEDULE ID: %i' %conf_schedule.SCHEDULE_ID
raise ValueError('FATAL error: No more free nodes available %i - sc %i' % (len(inactive_nodes), conf_schedule.SCHEDULE_ID))
# Return selected node
return self.active_node.name
def test_nodes(self, new_domain, nodelist):
results = []
for node in nodelist:
# Aggregate load for the complete host
mem_load = 0
for dom in node.domains.values():
spec = dom.domain_configuration.get_domain_spec()
mem_load += spec.total_memory()
# Get actual CPU measurements
curr_cpu_demand = np.percentile(node.get_readings(), 95)
resd_cpu = conf_nodes.UTIL - curr_cpu_demand
# Calculate residual vector
resd_mem = conf_nodes.NODE_MEM - mem_load
# VM resource demand
spec = conf_domainsize.get_domain_spec(new_domain.size)
# Calculate estiated CPU demand if VM is almost
domain_cpu_demand = conf_nodes.to_node_load(95, new_domain.size)
domain_mem_demand = spec.total_memory()
# Calculate the dot product
w_cpu = w_mem = 1
abs_res = math.sqrt(math.pow(resd_cpu, 2) + math.pow(resd_mem, 2))
abs_vm = math.sqrt(math.pow(domain_cpu_demand, 2) + math.pow(domain_mem_demand, 2))
dot_product = w_cpu * resd_cpu * domain_cpu_demand + w_mem * resd_mem * domain_mem_demand
cosine = dot_product / (abs_res * abs_vm)
# Check if this host is able to handle the new domain
cpu_delta = resd_cpu - domain_cpu_demand
mem_delta = resd_mem - domain_mem_demand
if cpu_delta >= 0 and mem_delta >= 0:
results.append((node, dot_product, cosine))
else:
print 'failed for node %s - status: %i mem %i cpu' % (node.name, mem_delta, cpu_delta)
if results:
# Get the node with the best (greatest) dot product
results.sort(key=lambda x: x[1])
results.reverse()
best_dot_product = results[0][0]
# Get the node with the best (smallest) cosine
results.sort(key=lambda x: x[2])
best_cosine = results[0][0]
return (best_dot_product.name, best_cosine.name)
def test_nodes(self, new_domain, nodelist):
results = []
for node in nodelist:
# Aggregate load for the complete host
cpu_load = 0
mem_load = 0
for dom in node.domains.values():
spec = dom.domain_configuration.get_domain_spec()
cpu_load += spec.total_cpu_cores()
mem_load += spec.total_memory()
# VM resource demand
spec = conf_domainsize.get_domain_spec(new_domain.size)
# Calculate residual vector
resd_cpu = conf_nodes.NODE_CPU_CORES - cpu_load
resd_mem = conf_nodes.NODE_MEM - mem_load
# Calculate the dot product
w_cpu = w_mem = 1
abs_res = math.sqrt(math.pow(resd_cpu, 2) + math.pow(resd_mem, 2))
abs_vm = math.sqrt(math.pow(spec.total_cpu_cores(), 2) + math.pow(spec.total_memory(), 2))
dot_product = w_cpu * resd_cpu * spec.total_cpu_cores() + w_mem * resd_mem * spec.total_memory()
cosine = dot_product / (abs_res * abs_vm)
# Check if this host is able to handle the new domain
cpu_delta = resd_cpu - spec.total_cpu_cores()
mem_delta = resd_mem - spec.total_memory()
if cpu_delta >= 0 and mem_delta >= 0:
results.append((node, dot_product, cosine))
else:
print 'failed for node %s - status: %i mem %i cpu' % (node.name, mem_delta, cpu_delta)
if results:
# Get the node with the best (greatest) dot product
results.sort(key=lambda x: x[1])
results.reverse()
best_dot_product = results[0][0]
# Get the node with the best (smallest) cosine
results.sort(key=lambda x: x[2])
best_cosine = results[0][0]
return (best_dot_product.name, best_cosine.name)
def placement(self, domain):
# Get all hosts
hosts = self.model.get_hosts(model.types.NODE)
# Find interval for new domain
domain_spec = domainsize.get_domain_spec(domain.size)
for interval in self.intervals:
if interval.fits(domain_spec):
break
if interval == None:
raise ValueError("Could not find a suitable interval for domain")
# Find all active hosts for the given interval and unused hosts
interval_hosts = []
unused_hosts = []
for host in hosts:
if host.domains:
try:
if host.harmonic_interval == interval.index:
interval_hosts.append(host)
except:
unused_hosts.append(host)
pass
else:
unused_hosts.append(host)
# Selected target node
selected = None
# Check if one of the interval hosts has enough free resources to handle the new domain
for host in interval_hosts:
if self.__check_capacity(host, domain):
selected = host
break
# Open a new host
if selected is None:
selected = unused_hosts.pop()
# Set harmonic interval index
selected.harmonic_interval = interval.index
# Return selected node
return selected.name
def __calculate_reservation_based_lower_bounds(capacity_cpu, capacity_mem, events):
# Stack is used to keep track of active VMs
active_stack = []
# Demand duration list
demand_duration_list = []
# Go through all events and calculate lower bound values
last_event_offset = 0
for event in events:
# Update active VM stack
if event.is_start:
active_stack.append(event.entry)
else:
active_stack.remove(event.entry)
# Calculate total infrastructure demand
demand_cpu_cores = 0
demand_mem = 0
for entry in active_stack:
size = conf_domainsize.get_domain_spec(entry.domain_size)
demand_cpu_cores += size.total_cpu_cores()
demand_mem += size.total_memory()
# Calculate server demands
server_demand_cpu = math.ceil(demand_cpu_cores / capacity_cpu)
server_demand_mem = math.ceil(demand_mem / capacity_mem)
# Take bigger server demand (mem or cpu)
server_demand = max(server_demand_cpu, server_demand_mem)
# Update delta event duration calculation
event_duration= event.offset - last_event_offset
last_event_offset = event.offset
# Add server demand to list
demand_duration_list.append((server_demand, event_duration))
# Calculate overall lower bound on server demand over all event slots
lb_res_total = max([e[0] for e in demand_duration_list])
lb_res_avg = sum([e[0] * e[1] for e in demand_duration_list]) / sum([e[1] for e in demand_duration_list])
return lb_res_total, lb_res_avg
def test_nodes(self, new_domain, node_list):
host_choice = []
for node in node_list:
# Get actual CPU measurements
curr_cpu_demand = np.percentile(node.get_readings(), 95)
# Memory demand is calculated by summing up all VM reservations
mem_load = 0
# Calculate the node utilization by accumulating all domain loads
for dom in node.domains.values():
spec = dom.domain_configuration.get_domain_spec()
mem_load += spec.total_memory()
# Calculate metric
spec = conf_domainsize.get_domain_spec(new_domain.size)
mem_delta = conf_nodes.NODE_MEM - (mem_load + spec.total_memory())
# Calculate estiated CPU demand if VM is almost
vm_cpu_demand = conf_nodes.to_node_load(95, new_domain.size)
cpu_delta = conf_nodes.UTIL - curr_cpu_demand - vm_cpu_demand
# Calculate fit metric
metric = cpu_delta * mem_delta
# Server is not able to handle the domain
if cpu_delta < 0 or mem_delta < 0:
continue
# Add metric to the choice list
host_choice.append((node.name, metric))
# Check if we found at least one host
if not host_choice:
return None
# Sort host choice list
host_choice = self.sort(host_choice, lambda x: x[1])
# Pkc hte one with the lowest metric (best fit)
return host_choice[0][0]
def test_nodes(self, active_nodes, new_domain):
# Go over all hosts
for node in active_nodes:
# Aggregate resource load for this host
curr_cpu_demand = 0
curr_mem_demand = 0
# Add up the demands of all domains running on the host
for old_domain in node.domains.values():
# Domain size specification
spec = old_domain.domain_configuration.get_domain_spec()
# Sum up cpu and mem load
curr_cpu_demand += spec.total_cpu_cores()
curr_mem_demand += spec.total_memory()
# Calculate metric
new_domain_spec = conf_domainsize.get_domain_spec(new_domain.size)
cpu_delta = conf_nodes.NODE_CPU_CORES - curr_cpu_demand - new_domain_spec.total_cpu_cores()
mem_delta = conf_nodes.NODE_MEM - curr_mem_demand - new_domain_spec.total_memory()
# If metric is positive, the node can host the domain
if cpu_delta >= 0 and mem_delta >= 0:
return node.name
def __calculate_demand_based_lower_bounds(capacity_cpu, capacity_mem, events):
# Stack is used to keep track of active VMs
active_stack = []
# Demand duration list
demand_duration_list = []
# Go through all events and calculate lower bound values
last_event_offset = 0
# Go through all events and calculate lower bound values
for event in events:
# Update active VM stack
if event.is_start:
active_stack.append(event.entry)
else:
active_stack.remove(event.entry)
# Calculate total infrastructure demand
sum_cpu = 0
sum_mem = 0
for entry in active_stack:
# calculate current offset
delta_time = event.offset - entry.offset
delta_index = delta_time / entry.freq
# Include 10 minutes of load
delta_index_past = max(delta_index - 10, 0)
# Measurements
loads = entry.ts[delta_index_past:delta_index]
# Get minimum load
if len(loads) < 2:
load = 0;
else:
load = min(*loads)
size = conf_domainsize.get_domain_spec(entry.domain_size)
sum_cpu += conf_nodes.to_node_load(load, entry.domain_size)
sum_mem += size.total_memory()
# Calculate server demands
lb_cpu = math.ceil(sum_cpu / float(conf_nodes.UTIL))
lb_mem = math.ceil(sum_mem / float(capacity_mem))
# Take bigger server demand
lb = max(lb_cpu, lb_mem)
# Update delta event duration calculation
duration = event.offset - last_event_offset
last_event_offset = event.offset
# Add server demand to list
demand_duration_list.append((lb, duration))
# Calculate overall lower bound on server demand over all event slots
lb_dem_total = max([e[0] for e in demand_duration_list])
lb_dem_avg = sum([e[0] * e[1] for e in demand_duration_list]) / sum([e[1] for e in demand_duration_list])
return lb_dem_total, lb_dem_avg
def get_domain_spec(self):
return conf_domainsize.get_domain_spec(self.size)
def to_node_load(domain_load, domain_size):
domain_cores = domainsize.get_domain_spec(domain_size).phy_cpu_cores()
return float(domain_load * domain_cores) / float(NODE_CPU_CORES)