def slo_agg_1minute(self, output_file, file):
converters = Converters()
with open(output_file, 'w') as slo_fp:
with open(file) as fp:
next(fp) # skip header
timestamps = {}
for line in fp:
timestamp, estimated_time, url, response_code, status, attr1, attr2 = line.split(",")
#df = datetime.datetime.strptime(datetime.datetime.fromtimestamp(float(timestamp)/1000).strftime("%Y-%m-%d %H:%M"), "%Y-%m-%d %H:%M") # remove microseconds from time
response_code = converters.response_code_converter(response_code)
new_timestamp = int(float(timestamp)/60000)*60000
if not timestamps.has_key(new_timestamp):
timestamps[new_timestamp] = {'num' : 0, 'num_all_requests': 0}
timestamps[new_timestamp]['num_all_requests'] += 1
i=0
if self.does_violate_slo(url, estimated_time, response_code):
i = 1
timestamps[new_timestamp]['num'] += i
slo_fp.write('"date","num","num_all_requests"\n')
for timestamp in timestamps.keys():
slo_fp.write('%s,%s,%s\n' % (timestamp, timestamps[timestamp]['num'],timestamps[timestamp]['num_all_requests']))
def delete_records_that_violates_slo(self, output_file, input_file):
converters = Converters()
with open(output_file, 'w') as slo_fp:
slo_fp.write('"date","response_time","url","response_code","status","autoscalable","instance_id"\n')
with open(input_file) as fp:
next(fp) # skip header
for line in fp:
timestamp, estimated_time, url, response_code, status, attr1, attr2 = line.split(",")
response_code = converters.response_code_converter(response_code)
if self.does_violate_slo(url, estimated_time, response_code):
slo_fp.write('%s,%s,%s,%s,%s,%s,%s' % (timestamp, estimated_time, url, response_code, status, attr1, attr2))
def parse(self, output_file, file):
with open(file) as fp:
next(fp) # skip the header
with open(output_file, "w") as parsed_fp:
parsed_fp.write("date,response_time,url,response_code,status,attr1,attr2\n")
converters = Converters()
for line in fp:
timestamp, estimated_time, url, response_code, status, attr1, attr2 = line.split(",")
response_code = converters.response_code_converter(response_code)
url = converters.url_converter(url)
if url is not None:
parsed_fp.write('%s,%s,%s,%s,%s,%s,%s' % (timestamp, estimated_time, url, response_code, status, attr1, attr2))
def _calc_slo_violations(self, sorted_data, ref_timestamp, seconds):
converters = Converters()
timestamps = {ref_timestamp: {'num': 0, 'num_all_requests': 0}}
for line in sorted_data:
timestamp, estimated_time, url, response_code, status, attr1, attr2 = line
timestamp = (int(timestamp) / 1000)
response_code = converters.response_code_converter(response_code)
if ref_timestamp - timestamp < 0:
ref_timestamp = ref_timestamp + seconds
if not timestamps.has_key(ref_timestamp):
timestamps[ref_timestamp] = {'num': 0, 'num_all_requests': 0}
timestamps[ref_timestamp]['num_all_requests'] += 1
timestamps[ref_timestamp]['num'] += 1 if self.does_violate_slo(url, estimated_time, response_code) else 0
return timestamps
def slo_agg_seconds(self, parsed_file, output_file, seconds):
print "Seconds: %s" % seconds
converters = Converters()
with open(output_file, 'w') as slo_fp:
with open(parsed_file) as fp:
next(fp) # skip header
parsed_file_data = csv.reader(fp)
sorted_data = sorted(parsed_file_data, key = lambda row: int(row[0]))
timestamps = {}
ref_timestamp, _, _, _, _, _, _ = sorted_data[0]
ref_timestamp = (int(ref_timestamp)/1000)
timestamps[ref_timestamp] = {'num' : 0, 'num_all_requests': 0}
min_date = sys.maxint
for line in sorted_data:
timestamp, estimated_time, url, response_code, status, attr1, attr2 = line
timestamp = (int(timestamp)/1000)
if timestamp < min_date:
min_date = timestamp
response_code = converters.response_code_converter(response_code)
time_delta = datetime.datetime.fromtimestamp(timestamp) - datetime.datetime.fromtimestamp(ref_timestamp)
# print "time_delta: %s" % time_delta.seconds
# print "time_delta: %s" % datetime.datetime.fromtimestamp(timestamp)
if time_delta.seconds >= seconds:
# print "new ref timestamp: %s" % datetime.datetime.fromtimestamp(timestamp)
ref_timestamp = timestamp
if not timestamps.has_key(ref_timestamp):
timestamps[ref_timestamp] = {'num' : 0, 'num_all_requests': 0}
timestamps[ref_timestamp]['num_all_requests'] += 1
i=0
if self.does_violate_slo(url, estimated_time, response_code):
i = 1
timestamps[ref_timestamp]['num'] += i
print min_date
slo_fp.write('"date","num","num_all_requests"\n')
slo_fp.write('%s,%s,%s\n' % (0, 0, 0))
for timestamp in timestamps.keys():
timestamp_subtract = (timestamp - min_date)+seconds
slo_fp.write('%s,%s,%s\n' % (timestamp_subtract*1000, timestamps[timestamp]['num'],timestamps[timestamp]['num_all_requests']))
def _calc_slo_violations(self, sorted_data, ref_timestamp, seconds):
converters = Converters()
timestamps = {ref_timestamp: {'num': 0, 'num_all_requests': 0}}
for line in sorted_data:
timestamp, estimated_time, url, response_code, status, attr1, attr2 = line
timestamp = (int(timestamp) / 1000)
response_code = converters.response_code_converter(response_code)
if ref_timestamp - timestamp < 0:
ref_timestamp = ref_timestamp + seconds
if not timestamps.has_key(ref_timestamp):
timestamps[ref_timestamp] = {
'num': 0,
'num_all_requests': 0
}
timestamps[ref_timestamp]['num_all_requests'] += 1
timestamps[ref_timestamp]['num'] += 1 if self.does_violate_slo(
url, estimated_time, response_code) else 0
return timestamps
