def process_sonar_trace(name, trace_ts, timestamps, save=False):
# Number of readings to aggregate to downsample the
# trace to width PROFILE_INTERVAL_COUNT
readings_to_aggregate = len(trace_ts) / PROFILE_INTERVAL_COUNT
# Create a new array to hold the profile
profile = np.zeros(PROFILE_INTERVAL_COUNT, dtype=float)
# Downsample the trace to the profile array
for i in xrange(PROFILE_INTERVAL_COUNT):
start = readings_to_aggregate * i
end = min(readings_to_aggregate * (i + 1), len(trace_ts))
# ATTENTION: Using mean here
profile[i] = np.mean(trace_ts[start:end])
# Calculate interval
interval = EXPERIMENT_DURATION / PROFILE_INTERVAL_COUNT
# Save the profile
if save:
connection = times_client.connect()
# Special case - no prefix is added as this TS is treated as a RAW TS
__write_profile(connection, __times_name(False, name, POSTFIX_TRACE), profile, interval, noprefix=True)
times_client.close()
def __plot_complete_mix():
'''
Plots all TS of a mix in a single image using multiple axis
'''
# Connect with times
connection = times_client.connect()
plot_mix = pdata.mix_2
cols = 5
rows = len(plot_mix) / cols + 1
index = 0
print rows
fig = plt.figure()
fig.set_figheight(20)
fig.set_figwidth(40)
for desc in plot_mix:
name = desc.name
timeSeries = connection.load(__times_name(True, name, POSTFIX_USER))
_, demand = util.to_array(timeSeries)
index += 1
ax = fig.add_subplot(rows, cols, index)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
ax.plot(range(0, len(demand)), demand)
plt.savefig(configuration.path('mix_overlay', 'png'))
# Close times connection
times_client.close()
def __dump_to_csv():
connection = times_client.connect()
demands = []
for desc in selected:
timeSeries = connection.load(__times_name(True, desc.name, POSTFIX_NORM))
_, demand = util.to_array(timeSeries)
demands.append((desc.name, demand))
import csv
with open(configuration.path('traces', 'csv'), 'wb') as csvfile:
spamwriter = csv.writer(csvfile, delimiter='\t')
row = []
for demand in demands:
row.append(demand[0])
spamwriter.writerow(row)
l = len(demands[0][1])
for i in xrange(0, l):
row = []
for demand in demands:
if i < len(demand[1]):
row.append(demand[1][i])
else:
print 'warn'
row.append(0)
spamwriter.writerow(row)
times_client.close()
def __build_profiles(mix, save):
'''
Build the profiles for all TS in mix.
Keyword arguments:
mix -- set of TS to generate a profile
save -- save profiles to Times
'''
# Times connection
connection = times_client.connect()
# Calculate profiles
for desc in mix:
print 'processing convolution: %s' % (desc.name)
import convolution
profile = convolution.process_trace(connection, __times_name(False, desc.name),
desc.sample_frequency, CYCLE_TIME)
# Add profile to mix
desc.profile = profile
# Get maximum for each set in mix
set_max = __get_and_apply_set_max(mix)
# Store profiles
for desc in mix:
profile, frequency = desc.profile
__store_profile(connection, desc, set_max, profile, frequency, save)
# Close Times connection
times_client.close()
def __build_modified_profiles(mix, save):
connection = times_client.connect()
for mi_element in mix:
# Operates on pre-processed data. Hence, a prefix is required
ts_name = __times_name(True, mi_element.name, POSTFIX_NORM)
# Modify CPU normal profile
modified_profile, interval = modifier.process_trace(connection, ts_name,
mi_element.modifier, mi_element.additive,
mi_element.scale, mi_element.shift)
if save:
name = __times_name(True, mi_element.name, POSTFIX_NORM, POSTFIX_MODIFIED)
__write_profile(connection, name, modified_profile, interval)
# Store USER profiles (-> feed into Rain)
# Adapt frequency for the benchmark duration
# Add padding for ramp up and ramp down
modified_profile /= 100.0
modified_profile *= MAX_USERS
interval = interval / (CYCLE_TIME / EXPERIMENT_DURATION)
user_profile = np.array(modified_profile)
user_profile = __padprofile(user_profile, interval)
if save:
name = __times_name(True, mi_element.name, POSTFIX_NORM, POSTFIX_MODIFIED)
__write_profile(connection, name, user_profile, interval)
times_client.close()
def process_trace(name):
print 'Downloading...'
connection = times_client.connect()
timeSeries = connection.load(name)
times_client.close()
print 'complete'
# 24 hours
periodicity = 24.0 * 3600.0 # day
frequency = 3600 # hour
smoothening = 30
return process_file(name, timeSeries.elements, periodicity, frequency, smoothening)
def __dump_user_profile_maxes():
'''
Used to verify that all user profiles have user values below MAX_USERS
'''
# Connect with times
connection = times_client.connect()
for name in connection.find('.*%s$' % (POSTFIX_USER)):
result = util.to_array(connection.load(name))[1]
if np.max(result) > MAX_USERS:
print '%s - %i' % (name, np.max(result))
times_client.close()
def start(self):
print 'Connecting with Times'
connection = times_client.connect()
self.min_ts_length = sys.maxint # Minimum length across all TS
ts_freq = 0 # Frequency of the TS from Times
# Iterate over all domains and assign them a TS
for domain in self.model.get_hosts(model.types.DOMAIN):
# Select and load TS (based on the configuration)
index = domains.index_of(domain.name)
mapping = domains.domain_profile_mapping[index]
# load = profiles.get_cpu_profile_for_initial_placement(mapping.profileId)
load = profiles.get_cpu_profile_for_initial_placement(mapping.profileId)
ts = connection.load(load)
# Convert TS to a numpy array
# select TS not time index
ts_freq = ts.frequency
ts = wutil.to_array(ts)[1]
# Add noise to the time series
if NOISE:
# random = np.random.lognormal(mean=NOISE_MEAN, sigma=NOISE_SIGMA, size=len(ts))
random = np.random.normal(loc=NOISE_MEAN, scale=NOISE_SIGMA, size=len(ts))
ts += random
ts[ts > 100] = 100
ts[ts < 0] = 0
# Attach TS to domain
domain.ts = ts
# Update max length
self.min_ts_length = min(self.min_ts_length, len(ts))
# Close times connection
times_client.close()
# Reduce length of time series to 6 hours
# Calculation: Adjust frequency by (new duration / current TS duration)
self.freq = (ts_freq * profiles.EXPERIMENT_DURATION) / (self.min_ts_length * ts_freq)
# Calculate ramp up delete time
self.ramp_up = profiles.RAMP_UP
self.ramp_down = profiles.RAMP_DOWN
# Schedule message pump
self.pump.callLater(0, self.run)
def __call__(self, parser, namespace, values, option_string=None):
try:
query = values
port = times_client.port_MKII
if namespace.version == 1:
port = times_client.port_MKI
else:
port = times_client.port_MKII
connection = times_client.connect(port)
results = connection.find(query)
print 'name'
for result in results:
print '%s' % result
finally:
times_client.close()
def __plot_overlay_mix():
'''
Plots all TS of a mix in a single axis graph
'''
# Connect with times
connection = times_client.connect()
# Plot selected
selected_mix0 = ['O2_retail_ADDORDER', 'SIS_163_cpu', 'SIS_393_cpu']
selected_mix1 = ['SIS_222_cpu', 'SIS_213_cpu', 'SIS_387_cpu']
plot_mix = selected_mix1
# Plot all from a set
# plot_mix = []
# for i in xrange(a, a + 100):
# print selected[i].name
# plot_mix.append(selected[i].name)
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_xlim([0, 300])
for name in plot_mix:
timeSeries = connection.load(__times_name(True, name, POSTFIX_USER))
print timeSeries
_, demand = util.to_array(timeSeries)
demand = demand[7:289 + 7]
ax.plot(range(0, len(demand)), demand, linewidth=0.7)
xt = [(t * 60 / 5) for t in xrange(0, 25)]
xl = [t for t in xrange(0, 25)]
ax.set_xticks(xt)
ax.set_xticklabels(xl)
ax.set_xlabel('Time in hours')
ax.set_ylabel('Load in number of users')
# plt.show()
plt.savefig(configuration.path('overlay', 'png'))
plt.savefig(configuration.path('mix1', 'pdf'))
# Close times connection
times_client.close()
def __build_sample_day(mix, save):
connection = times_client.connect()
# Calculate profiles
for desc in mix:
print 'processing sample day %s' % (desc.name)
import sampleday
profile = sampleday.process_trace(connection, __times_name(False, desc.name),
desc.sample_frequency, CYCLE_TIME, desc.profile_set.day)
desc.profile = profile
# Max value in each set of TS
set_max = __get_and_apply_set_max(mix)
# Store profiles
for desc in mix:
profile, frequency = desc.profile
__store_profile(connection, desc, set_max, profile, frequency, save)
times_client.close()
def allocate_domains(name):
print 'Downloading...'
connection = times_client.connect()
timeSeries = connection.load(name)
print 'done'
times_client.close()
print 'complete'
# Generate (x,y) signal
signal = []
for element in timeSeries.elements:
signal.append(element.value)
# Buckets (DSAP paper)
blen = len(signal) / 48
bsignal = []
for b in xrange(48):
values = signal[b * blen:(b+1)*blen]
b = np.percentile(values, 99)
bsignal.append(b)
signal = []
signal.append(bsignal[0])
signal.extend(bsignal)
signal.append(bsignal[-1])
# Setup the new plot
fig = plt.figure()
fig.set_size_inches((10, 10))
plt.xlabel('Time')
plt.ylabel('Load')
# Create a new subplot
ax = fig.add_subplot(1, 1, 1)
ax.set_title('TimeSeries %s' % (name));
ax.plot(range(0, len(signal)), signal)
# Display the plot
plt.show()
def plot(profile=None):
# Setup the new plot
fig = plt.figure()
# Create a new subplot
ax = fig.add_subplot(1, 1, 1)
if profile is None:
# Connect with Times
connection = times_client.connect()
# Download TS
timeSeries = connection.load('PUSER_MIX0_O2_business_ADDORDER')
# Disconnect from Times
times_client.close()
# Generate (x,y) signal
signal = []
for element in timeSeries.elements:
signal.append(element.value)
ax.plot(range(0, len(signal)), signal)
else:
ax.plot(range(0, len(profile)), profile)
# Display the plot
plt.show()
def __call__(self, parser, namespace, values, option_string=None):
try:
name = values
port = times_client.port_MKII
if namespace.version == 1:
if namespace.version == 1:
port = times_client.port_MKI
else:
port = times_client.port_MKII
connection = times_client.connect(port)
result = connection.find(name)
if len(result) == 0:
print '0,0'
sys.exit(1)
ts = connection.load(name)
print 'timestamp,value'
for element in ts.elements:
print '%i,%i' % (element.timestamp, element.value)
finally:
times_client.close()
def main():
from service import times_client
from workload import util
con = times_client.connect()
data = con.load('SIS_222_cpu_profile_trace')
_ , demand = util.to_array(data)
times_client.close()
random = np.random.lognormal(mean=0.0, sigma=1.5, size=len(demand))
demand += random
# Run smoother
_, s0, _ = single_exponential_smoother(demand)
_, s1, _ = double_exponential_smoother(demand)
# Plot original data with forecasted
fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(demand)
ax.plot(s0)
ax.plot(s1)
plt.show()
# Store profiles
for handle in handles:
print 'storing convolution: %s ...' % (handle.name)
__store_profile_yarns(mix, handle, normalizing_value, handle.profile, handle.profile_frequency)
def __build_profiles(mix):
sampleday = []
convolution = []
for handle in mix.handles:
if handle.htype.day != None:
sampleday.append(handle)
else:
convolution.append(handle)
print 'Build sample days %i' % (len(sampleday))
__build_sampleday(mix, sampleday)
print 'Build profiles %i' % (len(convolution))
__build_convolutions(mix, convolution)
if mix.modified:
print 'Building modified %i' % (len(mix.handles))
__build_modified_profiles(mix)
if __name__ == '__main__':
connection = tc.connect()
__build_profiles(conf_load.TIMES_SELECTED_MIX)
tc.close()
def main():
# Solve allocation problem
nodecount = len(nodes.NODES)
model = placement.SSAPvPlacement(nodecount, nodes.NODE_CPU, nodes.NODE_MEM, nodes.DOMAIN_MEM)
model.execute()
assignment = model.assignment
if assignment != None:
node_assignment = {}
for domain in assignment.keys():
node = assignment[domain]
if not node_assignment.has_key(node):
node_assignment[node] = []
node_assignment[node].append(domain)
print node_assignment
# Load time series used by the drivers
# Connect with Times
print 'Connecting with Times'
tsdata = []
connection = times_client.connect()
# Loading services to combine the dmain_service_mapping with
service_count = len(domains.domain_profile_mapping)
import sys
ts_length = sys.maxint
for i_service in xrange(service_count):
name = profiles.get_cpu_current_profile(i_service)
tsd = connection.load(name)
tsd = wutil.to_array(tsd)[1]
tsdata.append(tsd)
ts_length = min(ts_length, len(tsd))
times_client.close()
print node_assignment
# Run simulation and report overload situations
acc_load = [[] for _ in xrange(len(nodes.NODES))]
for t in xrange(ts_length):
print '-- t -- %i' % t
for node in node_assignment.keys():
sum_load = 0
for domain in node_assignment[node]:
sum_load += tsdata[domain][t]
print node
acc_load[node].append(sum_load)
# Plot accumulated loads
fig = plt.figure()
fig.set
ax = fig.add_subplot(111)
ax.set_title("Workload %s Overload %s" % (profiles.selected_name, profiles.modified))
ax.axis([0.0, ts_length, 0, 500])
for load in acc_load:
ax.plot(range(0, len(load)), load)
plt.savefig('C:/temp/convolution/tesoverload_%s_%s.png' % (profiles.selected_name, profiles.modified))
else:
print 'Could not check overload - no feasible assignment found'
def execute(self):
# Execute super code
super(FirstFitPlacement, self).execute()
print 'Using First Fit for domain placmement ...'
# Logging
logger.info('Placement strategy: First Fit')
logger.info('Required servers: %i' % len(nodes.NODES))
# Connect with Times
print 'Connecting with Times'
connection = times_client.connect()
# Loading services to combine the dmain_service_mapping with
service_count = len(domains.domain_profile_mapping)
# For each node there is one bucket
buckets = []
buckets_mem = []
migrations = []
assignment = {}
for _ in xrange(len(nodes.NODES)):
buckets.append([0, nodes.NODE_CPU, []])
buckets_mem.append([0, nodes.NODE_MEM, []])
# Service which gets mapped
for service_index in xrange(service_count):
# Maps the service to a service profile
mapping = domains.domain_profile_mapping[service_index]
# Important: Load the trace of the workload profile
service = profiles.get_cpu_profile_for_initial_placement(service_index)
print 'loading service: %s' % (service)
ts = connection.load(service)
from workload import util
_, demand = util.to_array(ts)
# Determine max demand value of this service
max_value = np.percentile(demand, 95) # np.max(demand)
bin_found = False
try:
for node_index in xrange(len(buckets)):
bucket = buckets[node_index]
bucket_mem = buckets_mem[node_index]
if (bucket[0] + max_value) < bucket[1] and (bucket_mem[0] + nodes.DOMAIN_MEM) < bucket_mem[1]:
bin_found = True
bucket[2].append(service)
bucket[0] = bucket[0] + max_value
bucket_mem[0] = bucket_mem[0] + nodes.DOMAIN_MEM
migrations.append((mapping.domain, node_index))
assignment[service_index] = node_index
raise StopIteration()
print 'Error no target!'
except StopIteration:
if bin_found == False:
print 'WARN: Could not assign domain to a node!'
# Close Times connection
times_client.close()
for bucket in buckets:
print 'bucket length: %i' % len(bucket[2])
print 'Assignment: %s' % assignment
logger.info('Assignment: %s' % assignment)
print 'Migrations: %s' % migrations
logger.info('Migrations: %s' % migrations)
return migrations, self._count_active_servers(assignment)
def execute(self, num_buckets, aggregation, migration_limit):
from workload import util
# Execute super code
super(DSAPPlacement, self).execute()
# Connect with Times
print 'Connecting with Times'
connection = times_client.connect()
# Loading services to combine the dmain_service_mapping with
domain_count = len(domains.domain_profile_mapping)
domain_matrix = np.zeros((domain_count, num_buckets), dtype=float)
domain_log = ''
for domain_index in xrange(domain_count):
mapping = domains.domain_profile_mapping[domain_index]
# Important: Load the trace of the workload profile
domain = profiles.get_cpu_profile_for_initial_placement(mapping.profileId)
print 'loading domain: %s' % (domain)
domain_log += domain + '; '
ts = connection.load(domain)
ts_len = len(ts.elements)
# put TS into domain matrix
_time, data = util.to_array(ts)
data = data[0:profiles.PROFILE_INTERVAL_COUNT]
# Downsampling TS (domain_matrix)
self.experiment_length = ts_len * ts.frequency # length of the experiment measured in seconds
bucket_width = self.experiment_length / num_buckets # in sec
# elements = bucket_width / ts.frequency
elements = ts_len / num_buckets
buckets = []
for i in xrange(num_buckets):
start = i * elements
end = min(ts_len, (i + 1) * elements)
tmp = data[start : end]
buckets.append(aggregation(tmp))
domain_matrix[domain_index] = buckets
# print data
# Log services
logger.info('Selected profile: %s' % profiles.selected_name)
logger.info('Loading services: %s' % domain_log)
# Dumpservice_matrix
print 'Logging domain matrix...'
np.set_printoptions(linewidth=200, threshold=99999999)
logger.info('Service matrix: %s' % domain_matrix)
# Close Times connection
times_client.close()
print "Downsampling-Ratio:", ts_len, "elements TO", num_buckets, "buckets (freq=", ts.frequency, ", placement.experiment_length=", self.experiment_length, ", profiles.experiment_duration", profiles.EXPERIMENT_DURATION, ")"
print 'Solving model...'
logger.info('Placement strategy: DSAP')
server_list, assignment_list = dsap.solve(self.nodecount, self.node_capacity_cpu, self.node_capacity_mem, domain_matrix, self.domain_demand_mem, migration_limit)
# return values for initial placement only > A(0) < (#servers + assignment(t=0))
self.assignment_list = assignment_list
initial_placement = assignment_list[0]
self.server_list = server_list
initial_server_count = server_list[0]
# Set initial_placement for getter functions
if initial_placement != None:
print 'Required servers: %i' % (initial_server_count)
logger.info('Required servers: %i' % initial_server_count)
print initial_placement
logger.info('Assignment: %s' % initial_placement)
print 'Assigning domains to servers'
migrations = []
for key in initial_placement.keys():
mapping = domains.domain_profile_mapping[key]
migration = (mapping.domain, initial_placement[key])
migrations.append(migration)
print 'Migrations: %s' % migrations
logger.info('Migrations: %s' % migrations)
return migrations, self._count_active_servers(initial_placement)
else:
print 'model infeasible'
return None, None
def execute(self, aggregation=False, bucketCount=24):
# Execute super code
super(SSAPvPlacement, self).execute()
# Connect with Times
print 'Connecting with Times'
connection = times_client.connect()
# Loading services to combine the dmain_service_mapping with
service_count = len(domains.domain_profile_mapping)
if aggregation:
llen = bucketCount
else:
llen = profiles.PROFILE_INTERVAL_COUNT
service_matrix = np.zeros((service_count, llen), dtype=float)
service_log = ''
for service_index in xrange(service_count):
mapping = domains.domain_profile_mapping[service_index]
service = profiles.get_cpu_profile_for_initial_placement(mapping.profileId)
print 'loading service: %s' % (service)
service_log += service + '; '
ts = connection.load(service)
ts_len = len(ts.elements)
# put TS into service matrix
data = np.empty((ts_len), dtype=float)
for i in xrange(ts_len):
data[i] = ts.elements[i].value
data = data[0:profiles.PROFILE_INTERVAL_COUNT]
# Downsample TS
if aggregation:
elements = ts_len / bucketCount
bucket_data = []
for i in xrange(bucketCount):
start = i * elements
end = min(ts_len, (i + 1) * elements)
tmp = data[start : end]
bucket_data.append(np.max(tmp))
service_matrix[service_index] = bucket_data
else:
service_matrix[service_index] = data
# Log services
logger.info('Selected profile: %s' % profiles.selected_name)
logger.info('Loading services: %s' % service_log)
# Dumpservice_matrix
print 'Logging service matrix...'
np.set_printoptions(linewidth=200, threshold=99999999)
logger.info('Service matrix: %s' % service_matrix)
# Close Times connection
times_client.close()
print 'Solving model...'
logger.info('Placement strategy: SSAPv')
server, assignment = ssapv.solve(self.nodecount, self.node_capacity_cpu, self.node_capacity_mem, service_matrix, self.domain_demand_mem)
# Set assignment for getter functions
self.assignment = assignment
if assignment != None:
print 'Required servers: %i' % (server)
logger.info('Required servers: %i' % server)
print assignment
logger.info('Assignment: %s' % assignment)
print 'Assigning domains to servers'
migrations = []
for key in assignment.keys():
mapping = domains.domain_profile_mapping[key]
migration = (mapping.domain, assignment[key])
migrations.append(migration)
print 'Migrations: %s' % migrations
logger.info('Migrations: %s' % migrations)
return migrations, self._count_active_servers(assignment)
else:
print 'model infeasible'
return None, None
value = h
# Add hour to periods if it doensn't exist
if value not in periods:
periods.append(value)
# Get the longest period
ret = np.max(periods)
return ret, acf, ped
if __name__ == '__main__':
import matplotlib.pyplot as plt
# Connect and load TS
connection = times_client.connect()
ll = connection.find('RAW_O2_.*')
for name in ll:
timeSeries = connection.load(name)
time, demand = util.to_array(timeSeries)
period, acf, ped = __period_calc(timeSeries.frequency, demand, time)
# Calculate profile
freq, profile = __extract_profile(time, demand, timeSeries.frequency, period, hour(1), lambda x: np.mean(x))
freq, profile_upper = __extract_profile(time, demand, timeSeries.frequency, period, hour(1), lambda x: np.percentile(x, 95))
freq, profile_lower = __extract_profile(time, demand, timeSeries.frequency, period, hour(1), lambda x: np.percentile(x, 15))
# Plotting
fig = plt.figure()
def main():
tc.connect()
createBenchmarkProfile('PUSER_BENCHMARK_SMALL', 100, sec(3), hour(6))
createBenchmarkProfile('PUSER_BENCHMARK_MEDIUM', 150, sec(3), hour(6))
createBenchmarkProfile('PUSER_BENCHMARK_LARGE', 200, sec(3), hour(6))
tc.close()
