def start(self):
print 'Connecting with Times'
connection = times_client.connect()
self.min_ts_length = sys.maxint # Minimum length across all TS
ts_freq = 0 # Frequency of the TS from Times
# Iterate over all domains and assign them a TS
for domain in self.model.get_hosts(model.types.DOMAIN):
# Select and load TS (based on the configuration)
index = domains.index_of(domain.name)
mapping = domains.domain_profile_mapping[index]
# load = profiles.get_cpu_profile_for_initial_placement(mapping.profileId)
load = profiles.get_cpu_profile_for_initial_placement(mapping.profileId)
ts = connection.load(load)
# Convert TS to a numpy array
# select TS not time index
ts_freq = ts.frequency
ts = wutil.to_array(ts)[1]
# Add noise to the time series
if NOISE:
# random = np.random.lognormal(mean=NOISE_MEAN, sigma=NOISE_SIGMA, size=len(ts))
random = np.random.normal(loc=NOISE_MEAN, scale=NOISE_SIGMA, size=len(ts))
ts += random
ts[ts > 100] = 100
ts[ts < 0] = 0
# Attach TS to domain
domain.ts = ts
# Update max length
self.min_ts_length = min(self.min_ts_length, len(ts))
# Close times connection
times_client.close()
# Reduce length of time series to 6 hours
# Calculation: Adjust frequency by (new duration / current TS duration)
self.freq = (ts_freq * profiles.EXPERIMENT_DURATION) / (self.min_ts_length * ts_freq)
# Calculate ramp up delete time
self.ramp_up = profiles.RAMP_UP
self.ramp_down = profiles.RAMP_DOWN
# Schedule message pump
self.pump.callLater(0, self.run)
def __calc_stats(con):
stats = []
names = con.find('RAW_SIS_.*_cpu')
for i, name in enumerate(names):
print 'processing %s... (%i%%)' % (name, (i*100/len(names)))
timeSeries = con.load(name)
_, values = util.to_array(timeSeries)
stat = TsStat(name)
stats.append(stat)
stat.mean = np.mean(values)
stat.median = np.percentile(values, 50)
stat.min = np.min(values)
stat.max = np.max(values)
stat.std = np.std(values)
return stats
def main():
from service import times_client
from workload import util
con = times_client.connect()
data = con.load('SIS_222_cpu_profile_trace')
_ , demand = util.to_array(data)
times_client.close()
random = np.random.lognormal(mean=0.0, sigma=1.5, size=len(demand))
demand += random
# Run smoother
_, s0, _ = single_exponential_smoother(demand)
_, s1, _ = double_exponential_smoother(demand)
# Plot original data with forecasted
fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(demand)
ax.plot(s0)
ax.plot(s1)
plt.show()
def main():
# Solve allocation problem
nodecount = len(nodes.NODES)
model = placement.SSAPvPlacement(nodecount, nodes.NODE_CPU, nodes.NODE_MEM, nodes.DOMAIN_MEM)
model.execute()
assignment = model.assignment
if assignment != None:
node_assignment = {}
for domain in assignment.keys():
node = assignment[domain]
if not node_assignment.has_key(node):
node_assignment[node] = []
node_assignment[node].append(domain)
print node_assignment
# Load time series used by the drivers
# Connect with Times
print 'Connecting with Times'
tsdata = []
connection = times_client.connect()
# Loading services to combine the dmain_service_mapping with
service_count = len(domains.domain_profile_mapping)
import sys
ts_length = sys.maxint
for i_service in xrange(service_count):
name = profiles.get_cpu_current_profile(i_service)
tsd = connection.load(name)
tsd = wutil.to_array(tsd)[1]
tsdata.append(tsd)
ts_length = min(ts_length, len(tsd))
times_client.close()
print node_assignment
# Run simulation and report overload situations
acc_load = [[] for _ in xrange(len(nodes.NODES))]
for t in xrange(ts_length):
print '-- t -- %i' % t
for node in node_assignment.keys():
sum_load = 0
for domain in node_assignment[node]:
sum_load += tsdata[domain][t]
print node
acc_load[node].append(sum_load)
# Plot accumulated loads
fig = plt.figure()
fig.set
ax = fig.add_subplot(111)
ax.set_title("Workload %s Overload %s" % (profiles.selected_name, profiles.modified))
ax.axis([0.0, ts_length, 0, 500])
for load in acc_load:
ax.plot(range(0, len(load)), load)
plt.savefig('C:/temp/convolution/tesoverload_%s_%s.png' % (profiles.selected_name, profiles.modified))
else:
print 'Could not check overload - no feasible assignment found'
def execute(self):
# Execute super code
super(FirstFitPlacement, self).execute()
print 'Using First Fit for domain placmement ...'
# Logging
logger.info('Placement strategy: First Fit')
logger.info('Required servers: %i' % len(nodes.NODES))
# Connect with Times
print 'Connecting with Times'
connection = times_client.connect()
# Loading services to combine the dmain_service_mapping with
service_count = len(domains.domain_profile_mapping)
# For each node there is one bucket
buckets = []
buckets_mem = []
migrations = []
assignment = {}
for _ in xrange(len(nodes.NODES)):
buckets.append([0, nodes.NODE_CPU, []])
buckets_mem.append([0, nodes.NODE_MEM, []])
# Service which gets mapped
for service_index in xrange(service_count):
# Maps the service to a service profile
mapping = domains.domain_profile_mapping[service_index]
# Important: Load the trace of the workload profile
service = profiles.get_cpu_profile_for_initial_placement(service_index)
print 'loading service: %s' % (service)
ts = connection.load(service)
from workload import util
_, demand = util.to_array(ts)
# Determine max demand value of this service
max_value = np.percentile(demand, 95) # np.max(demand)
bin_found = False
try:
for node_index in xrange(len(buckets)):
bucket = buckets[node_index]
bucket_mem = buckets_mem[node_index]
if (bucket[0] + max_value) < bucket[1] and (bucket_mem[0] + nodes.DOMAIN_MEM) < bucket_mem[1]:
bin_found = True
bucket[2].append(service)
bucket[0] = bucket[0] + max_value
bucket_mem[0] = bucket_mem[0] + nodes.DOMAIN_MEM
migrations.append((mapping.domain, node_index))
assignment[service_index] = node_index
raise StopIteration()
print 'Error no target!'
except StopIteration:
if bin_found == False:
print 'WARN: Could not assign domain to a node!'
# Close Times connection
times_client.close()
for bucket in buckets:
print 'bucket length: %i' % len(bucket[2])
print 'Assignment: %s' % assignment
logger.info('Assignment: %s' % assignment)
print 'Migrations: %s' % migrations
logger.info('Migrations: %s' % migrations)
return migrations, self._count_active_servers(assignment)
def execute(self, num_buckets, aggregation, migration_limit):
from workload import util
# Execute super code
super(DSAPPlacement, self).execute()
# Connect with Times
print 'Connecting with Times'
connection = times_client.connect()
# Loading services to combine the dmain_service_mapping with
domain_count = len(domains.domain_profile_mapping)
domain_matrix = np.zeros((domain_count, num_buckets), dtype=float)
domain_log = ''
for domain_index in xrange(domain_count):
mapping = domains.domain_profile_mapping[domain_index]
# Important: Load the trace of the workload profile
domain = profiles.get_cpu_profile_for_initial_placement(mapping.profileId)
print 'loading domain: %s' % (domain)
domain_log += domain + '; '
ts = connection.load(domain)
ts_len = len(ts.elements)
# put TS into domain matrix
_time, data = util.to_array(ts)
data = data[0:profiles.PROFILE_INTERVAL_COUNT]
# Downsampling TS (domain_matrix)
self.experiment_length = ts_len * ts.frequency # length of the experiment measured in seconds
bucket_width = self.experiment_length / num_buckets # in sec
# elements = bucket_width / ts.frequency
elements = ts_len / num_buckets
buckets = []
for i in xrange(num_buckets):
start = i * elements
end = min(ts_len, (i + 1) * elements)
tmp = data[start : end]
buckets.append(aggregation(tmp))
domain_matrix[domain_index] = buckets
# print data
# Log services
logger.info('Selected profile: %s' % profiles.selected_name)
logger.info('Loading services: %s' % domain_log)
# Dumpservice_matrix
print 'Logging domain matrix...'
np.set_printoptions(linewidth=200, threshold=99999999)
logger.info('Service matrix: %s' % domain_matrix)
# Close Times connection
times_client.close()
print "Downsampling-Ratio:", ts_len, "elements TO", num_buckets, "buckets (freq=", ts.frequency, ", placement.experiment_length=", self.experiment_length, ", profiles.experiment_duration", profiles.EXPERIMENT_DURATION, ")"
print 'Solving model...'
logger.info('Placement strategy: DSAP')
server_list, assignment_list = dsap.solve(self.nodecount, self.node_capacity_cpu, self.node_capacity_mem, domain_matrix, self.domain_demand_mem, migration_limit)
# return values for initial placement only > A(0) < (#servers + assignment(t=0))
self.assignment_list = assignment_list
initial_placement = assignment_list[0]
self.server_list = server_list
initial_server_count = server_list[0]
# Set initial_placement for getter functions
if initial_placement != None:
print 'Required servers: %i' % (initial_server_count)
logger.info('Required servers: %i' % initial_server_count)
print initial_placement
logger.info('Assignment: %s' % initial_placement)
print 'Assigning domains to servers'
migrations = []
for key in initial_placement.keys():
mapping = domains.domain_profile_mapping[key]
migration = (mapping.domain, initial_placement[key])
migrations.append(migration)
print 'Migrations: %s' % migrations
logger.info('Migrations: %s' % migrations)
return migrations, self._count_active_servers(initial_placement)
else:
print 'model infeasible'
return None, None
##########################
connection = times_client.connect()
mean_deltas = []
mean_sds = []
mean_50til = []
mean_90til = []
mean_corr = []
print '%s \t %s \t %s \t %s \t %s' % ('name', 'mean', 'sd', '50th', '90th')
for desc in selected:
# Load normal CPU trace
default = desc.name + profiles.POSTFIX_NORM
ts_default = connection.load(default)
ar_default = wutil.to_array(ts_default)[1]
# Load modified CPU trace
modified = default + profiles.POSTFIX_MODIFIED
ts_modified = connection.load(modified)
ar_modified = wutil.to_array(ts_modified)[1]
# Calculate mean
mean_default = np.mean(ar_default)
mean_modified = np.mean(ar_modified)
mean_deltas.append(mean_modified - mean_default)
# Calculate SDs
stdev_default = np.std(ar_default)
stdev_modified = np.std(ar_modified)
mean_sds.append(stdev_modified - stdev_default)
