def slo_agg_1minute(self, output_file, file):
converters = Converters()
with open(output_file, 'w') as slo_fp:
with open(file) as fp:
next(fp) # skip header
timestamps = {}
for line in fp:
timestamp, estimated_time, url, response_code, status, attr1, attr2 = line.split(",")
#df = datetime.datetime.strptime(datetime.datetime.fromtimestamp(float(timestamp)/1000).strftime("%Y-%m-%d %H:%M"), "%Y-%m-%d %H:%M") # remove microseconds from time
response_code = converters.response_code_converter(response_code)
new_timestamp = int(float(timestamp)/60000)*60000
if not timestamps.has_key(new_timestamp):
timestamps[new_timestamp] = {'num' : 0, 'num_all_requests': 0}
timestamps[new_timestamp]['num_all_requests'] += 1
i=0
if self.does_violate_slo(url, estimated_time, response_code):
i = 1
timestamps[new_timestamp]['num'] += i
slo_fp.write('"date","num","num_all_requests"\n')
for timestamp in timestamps.keys():
slo_fp.write('%s,%s,%s\n' % (timestamp, timestamps[timestamp]['num'],timestamps[timestamp]['num_all_requests']))
def delete_records_that_violates_slo(self, output_file, input_file):
converters = Converters()
with open(output_file, 'w') as slo_fp:
slo_fp.write('"date","response_time","url","response_code","status","autoscalable","instance_id"\n')
with open(input_file) as fp:
next(fp) # skip header
for line in fp:
timestamp, estimated_time, url, response_code, status, attr1, attr2 = line.split(",")
response_code = converters.response_code_converter(response_code)
if self.does_violate_slo(url, estimated_time, response_code):
slo_fp.write('%s,%s,%s,%s,%s,%s,%s' % (timestamp, estimated_time, url, response_code, status, attr1, attr2))
def parse(self, output_file, file):
with open(file) as fp:
next(fp) # skip the header
with open(output_file, "w") as parsed_fp:
parsed_fp.write("date,response_time,url,response_code,status,attr1,attr2\n")
converters = Converters()
for line in fp:
timestamp, estimated_time, url, response_code, status, attr1, attr2 = line.split(",")
response_code = converters.response_code_converter(response_code)
url = converters.url_converter(url)
if url is not None:
parsed_fp.write('%s,%s,%s,%s,%s,%s,%s' % (timestamp, estimated_time, url, response_code, status, attr1, attr2))
def check(file_path):
output = ""
urls = {}
unsuccessfull = 0
all_requests = 0
fp = open(file_path)
for line in fp:
converters = Converters()
try:
timestamp, estimated_time, url, response_code, _, _, _ = line.split(
",")
url = converters.url_converter(url)
if max_time.has_key(url):
all_requests += 1
if not urls.has_key(url):
urls[url] = {}
urls[url]['times'] = []
urls[url]['times'].append([estimated_time, response_code])
if response_code != "200":
unsuccessfull += 1
except Exception as e:
output += "Exception occured\n"
output += e.message + "\n"
pass
for k in urls:
count_succ = 0
all = len(urls[k]['times'])
for time, response_code in urls[k]['times']:
if int(time) <= max_time[k] and response_code == "200":
count_succ += 1
dist = (all * 100.0) / all_requests
if count_succ >= (all * 90) / 100:
output += "%-50s %-50s prob = %.2f%%\n" % (
k, "OK [no. requests = %s]" % all, dist)
else:
p = (count_succ * 100) / all
output += "%-50s %-50s prob = %.2f%%\n" % (
k, "NOT OK [all = %s, succ = %s (%s%%) ]" %
(all, count_succ, p), dist)
fp.close()
output += "--------------------------------------------------\n"
output += "ALL = %s, UNSUCCESSFULL = %s\n" % (all_requests, unsuccessfull)
return output
def check(file_path):
output = ""
urls = {}
unsuccessfull = 0
all_requests = 0
fp = open(file_path)
for line in fp:
converters = Converters()
try:
timestamp, estimated_time, url, response_code, _, _, _ = line.split(",")
url = converters.url_converter(url)
if max_time.has_key(url):
all_requests+=1
if not urls.has_key(url):
urls[url] = {}
urls[url]['times'] = []
urls[url]['times'].append([estimated_time, response_code])
if response_code != "200":
unsuccessfull += 1
except Exception as e:
output += "Exception occured\n"
output += e.message + "\n"
pass
for k in urls:
count_succ = 0
all = len(urls[k]['times'])
for time, response_code in urls[k]['times']:
if int(time) <= max_time[k] and response_code == "200":
count_succ += 1
dist = (all*100.0)/all_requests
if count_succ >= (all * 90) / 100:
output += "%-50s %-50s prob = %.2f%%\n" % (k, "OK [no. requests = %s]" % all, dist)
else:
p = (count_succ*100)/all
output += "%-50s %-50s prob = %.2f%%\n" % (k, "NOT OK [all = %s, succ = %s (%s%%) ]" % (all, count_succ, p), dist)
fp.close()
output += "--------------------------------------------------\n"
output += "ALL = %s, UNSUCCESSFULL = %s\n" % (all_requests, unsuccessfull)
return output
def to_dataframe(self, file, as_file):
print "Parsing " + file
converters = Converters()
self.data_indexed = pd.read_csv(file,
index_col='date',
converters={
'date':
converters.timestamp_converter,
'response_code':
converters.response_code_converter,
'url': converters.url_converter
})
self.data = pd.read_csv(file,
converters={
'date': converters.timestamp_converter,
'response_code':
converters.response_code_converter,
'url': converters.url_converter
})
# print "Parsing " + as_file
# self.autoscalability_data = pd.read_csv(as_file, converters={
# 'start_time' : converters.datetime_to_timestamp,
# 'end_time' : converters.datetime_to_timestamp,
# 'action' : converters.action_to_number
# })
return
def _calc_slo_violations(self, sorted_data, ref_timestamp, seconds):
converters = Converters()
timestamps = {ref_timestamp: {'num': 0, 'num_all_requests': 0}}
for line in sorted_data:
timestamp, estimated_time, url, response_code, status, attr1, attr2 = line
timestamp = (int(timestamp) / 1000)
response_code = converters.response_code_converter(response_code)
if ref_timestamp - timestamp < 0:
ref_timestamp = ref_timestamp + seconds
if not timestamps.has_key(ref_timestamp):
timestamps[ref_timestamp] = {'num': 0, 'num_all_requests': 0}
timestamps[ref_timestamp]['num_all_requests'] += 1
timestamps[ref_timestamp]['num'] += 1 if self.does_violate_slo(url, estimated_time, response_code) else 0
return timestamps
def slo_agg_seconds(self, parsed_file, output_file, seconds):
print "Seconds: %s" % seconds
converters = Converters()
with open(output_file, 'w') as slo_fp:
with open(parsed_file) as fp:
next(fp) # skip header
parsed_file_data = csv.reader(fp)
sorted_data = sorted(parsed_file_data, key = lambda row: int(row[0]))
timestamps = {}
ref_timestamp, _, _, _, _, _, _ = sorted_data[0]
ref_timestamp = (int(ref_timestamp)/1000)
timestamps[ref_timestamp] = {'num' : 0, 'num_all_requests': 0}
min_date = sys.maxint
for line in sorted_data:
timestamp, estimated_time, url, response_code, status, attr1, attr2 = line
timestamp = (int(timestamp)/1000)
if timestamp < min_date:
min_date = timestamp
response_code = converters.response_code_converter(response_code)
time_delta = datetime.datetime.fromtimestamp(timestamp) - datetime.datetime.fromtimestamp(ref_timestamp)
# print "time_delta: %s" % time_delta.seconds
# print "time_delta: %s" % datetime.datetime.fromtimestamp(timestamp)
if time_delta.seconds >= seconds:
# print "new ref timestamp: %s" % datetime.datetime.fromtimestamp(timestamp)
ref_timestamp = timestamp
if not timestamps.has_key(ref_timestamp):
timestamps[ref_timestamp] = {'num' : 0, 'num_all_requests': 0}
timestamps[ref_timestamp]['num_all_requests'] += 1
i=0
if self.does_violate_slo(url, estimated_time, response_code):
i = 1
timestamps[ref_timestamp]['num'] += i
print min_date
slo_fp.write('"date","num","num_all_requests"\n')
slo_fp.write('%s,%s,%s\n' % (0, 0, 0))
for timestamp in timestamps.keys():
timestamp_subtract = (timestamp - min_date)+seconds
slo_fp.write('%s,%s,%s\n' % (timestamp_subtract*1000, timestamps[timestamp]['num'],timestamps[timestamp]['num_all_requests']))
def _calc_slo_violations(self, sorted_data, ref_timestamp, seconds):
converters = Converters()
timestamps = {ref_timestamp: {'num': 0, 'num_all_requests': 0}}
for line in sorted_data:
timestamp, estimated_time, url, response_code, status, attr1, attr2 = line
timestamp = (int(timestamp) / 1000)
response_code = converters.response_code_converter(response_code)
if ref_timestamp - timestamp < 0:
ref_timestamp = ref_timestamp + seconds
if not timestamps.has_key(ref_timestamp):
timestamps[ref_timestamp] = {
'num': 0,
'num_all_requests': 0
}
timestamps[ref_timestamp]['num_all_requests'] += 1
timestamps[ref_timestamp]['num'] += 1 if self.does_violate_slo(
url, estimated_time, response_code) else 0
return timestamps
def check(file_path):
output = ""
urls = {}
unsuccessfull = 0
all_requests = 0
fp = open(file_path)
for line in fp:
converters = Converters()
try:
timestamp, estimated_time, operation, response_code, _, _, _ = line.split(",")
url = converters.url_converter(operation)
if url != None and max_time.has_key(url):
all_requests += 1
if not urls.has_key(url):
urls[url] = {}
urls[url]["times"] = []
urls[url]["times"].append([estimated_time, response_code])
if response_code != "200":
unsuccessfull += 1
except Exception as e:
output += "Exception occured\n"
output += e.message + "\n"
pass
dist_sum = 0
web_interactions = WebInteractions()
output += "%-40s %s %-20s %s %-20s %s %-25s %s %-20s %s %-20s %s %-20s %s %-20s %s %-20s\n" % (
"operation",
"|",
"status",
"|",
"# all request",
"|",
"# successfull requests",
"|",
"% of successfull",
"|",
"% of operation",
"|",
"allowed deviation",
"|",
"actual deviation",
"|",
"deviation ok?",
)
separator = "".join(["-" for _ in xrange(len(output))]) + "\n"
output += separator
for k in urls:
count_succ = 0
all = len(urls[k]["times"])
for time, response_code in urls[k]["times"]:
if int(time) <= max_time[k] and response_code == "200":
count_succ += 1
dist = (all * 100.0) / all_requests
dist_sum = dist_sum + dist
allowed_deviation = 0.05 * web_interactions.get_probability(k)
status = "NOT OK"
p = (count_succ * 100) / all
if count_succ >= (all * 90) / 100:
status = "OK"
p = 0
actual_deviation = get_actual_deviation(dist, k)
deviation_ok = is_deviation_ok(dist, allowed_deviation, k)
output += "%-40s %s %-20s %s %-20s %s %-25s %s %-20s %s %-20s %s %-20.3f %s %-20s %s %-20s\n" % (
k,
"|",
status,
"|",
all,
"|",
count_succ,
"|",
p,
"|",
"%.3f%% (%.3f%%)" % (dist, web_interactions.get_probability(k)),
"|",
allowed_deviation,
"|",
actual_deviation,
"|",
deviation_ok,
)
fp.close()
output += separator
output += "# ALL REQUESTS = %s, # UNSUCCESSFULL REQUESTS = %s, PROB SUM = %s\n" % (
all_requests,
unsuccessfull,
dist_sum,
)
return output
def check(file_path):
output = ""
urls = {}
unsuccessfull = 0
all_requests = 0
fp = open(file_path)
for line in fp:
converters = Converters()
try:
timestamp, estimated_time, operation, response_code, _, _, _ = line.split(",")
url = converters.url_converter(operation)
if url != None and max_time.has_key(url):
all_requests+=1
if not urls.has_key(url):
urls[url] = {}
urls[url]['times'] = []
urls[url]['times'].append([estimated_time, response_code])
if response_code != "200":
unsuccessfull += 1
except Exception as e:
output += "Exception occured\n"
output += e.message + "\n"
pass
dist_sum = 0
web_interactions = WebInteractions()
output += "%-40s %s %-20s %s %-20s %s %-25s %s %-20s %s %-20s %s %-20s %s %-20s %s %-20s\n" % ("operation", "|", "status", "|", "# all request", "|", "# successfull requests", "|", "% of successfull", "|", "% of operation", "|", "allowed deviation", "|", "actual deviation", "|", "deviation ok?")
separator = "".join(["-" for _ in xrange(len(output))]) + "\n"
output += separator
cummulative_violations = 0
cummulative_successful = 0
cummulative_all = 0
for k in urls:
count_succ = 0
all = len(urls[k]['times'])
for time, response_code in urls[k]['times']:
if int(time) <= max_time[k] and response_code == "200":
count_succ += 1
cummulative_successful += count_succ
cummulative_all += all
dist = (all*100.0)/all_requests
dist_sum = dist_sum + dist
allowed_deviation = 0.05*web_interactions.get_probability(k)
status = "NOT OK"
p = (count_succ*100)/all
if count_succ >= (all * 90) / 100:
status = "OK"
# p = 0
cummulative_violations += p
actual_deviation = get_actual_deviation(dist, k)
deviation_ok = is_deviation_ok(dist, allowed_deviation, k)
output += "%-40s %s %-20s %s %-20s %s %-25s %s %-20s %s %-20s %s %-20.3f %s %-20s %s %-20s\n" % (k, "|", status, "|", all, "|", count_succ, "|", p, "|", "%.3f%% (%.3f%%)" % (dist, web_interactions.get_probability(k)), "|", allowed_deviation, "|", actual_deviation, "|", deviation_ok)
fp.close()
output += separator
output += "# ALL REQUESTS = %s, # UNSUCCESSFULL REQUESTS = %s, PROB SUM = %s, # SLO VIOLATIONS = %s%%\n" % (all_requests, unsuccessfull, dist_sum, (((cummulative_all-cummulative_successful)*100)/cummulative_all))
return output
