def slo_agg_1minute(self, output_file, file): converters = Converters() with open(output_file, 'w') as slo_fp: with open(file) as fp: next(fp) # skip header timestamps = {} for line in fp: timestamp, estimated_time, url, response_code, status, attr1, attr2 = line.split(",") #df = datetime.datetime.strptime(datetime.datetime.fromtimestamp(float(timestamp)/1000).strftime("%Y-%m-%d %H:%M"), "%Y-%m-%d %H:%M") # remove microseconds from time response_code = converters.response_code_converter(response_code) new_timestamp = int(float(timestamp)/60000)*60000 if not timestamps.has_key(new_timestamp): timestamps[new_timestamp] = {'num' : 0, 'num_all_requests': 0} timestamps[new_timestamp]['num_all_requests'] += 1 i=0 if self.does_violate_slo(url, estimated_time, response_code): i = 1 timestamps[new_timestamp]['num'] += i slo_fp.write('"date","num","num_all_requests"\n') for timestamp in timestamps.keys(): slo_fp.write('%s,%s,%s\n' % (timestamp, timestamps[timestamp]['num'],timestamps[timestamp]['num_all_requests']))
def delete_records_that_violates_slo(self, output_file, input_file): converters = Converters() with open(output_file, 'w') as slo_fp: slo_fp.write('"date","response_time","url","response_code","status","autoscalable","instance_id"\n') with open(input_file) as fp: next(fp) # skip header for line in fp: timestamp, estimated_time, url, response_code, status, attr1, attr2 = line.split(",") response_code = converters.response_code_converter(response_code) if self.does_violate_slo(url, estimated_time, response_code): slo_fp.write('%s,%s,%s,%s,%s,%s,%s' % (timestamp, estimated_time, url, response_code, status, attr1, attr2))
def parse(self, output_file, file): with open(file) as fp: next(fp) # skip the header with open(output_file, "w") as parsed_fp: parsed_fp.write("date,response_time,url,response_code,status,attr1,attr2\n") converters = Converters() for line in fp: timestamp, estimated_time, url, response_code, status, attr1, attr2 = line.split(",") response_code = converters.response_code_converter(response_code) url = converters.url_converter(url) if url is not None: parsed_fp.write('%s,%s,%s,%s,%s,%s,%s' % (timestamp, estimated_time, url, response_code, status, attr1, attr2))
def check(file_path): output = "" urls = {} unsuccessfull = 0 all_requests = 0 fp = open(file_path) for line in fp: converters = Converters() try: timestamp, estimated_time, url, response_code, _, _, _ = line.split( ",") url = converters.url_converter(url) if max_time.has_key(url): all_requests += 1 if not urls.has_key(url): urls[url] = {} urls[url]['times'] = [] urls[url]['times'].append([estimated_time, response_code]) if response_code != "200": unsuccessfull += 1 except Exception as e: output += "Exception occured\n" output += e.message + "\n" pass for k in urls: count_succ = 0 all = len(urls[k]['times']) for time, response_code in urls[k]['times']: if int(time) <= max_time[k] and response_code == "200": count_succ += 1 dist = (all * 100.0) / all_requests if count_succ >= (all * 90) / 100: output += "%-50s %-50s prob = %.2f%%\n" % ( k, "OK [no. requests = %s]" % all, dist) else: p = (count_succ * 100) / all output += "%-50s %-50s prob = %.2f%%\n" % ( k, "NOT OK [all = %s, succ = %s (%s%%) ]" % (all, count_succ, p), dist) fp.close() output += "--------------------------------------------------\n" output += "ALL = %s, UNSUCCESSFULL = %s\n" % (all_requests, unsuccessfull) return output
def check(file_path): output = "" urls = {} unsuccessfull = 0 all_requests = 0 fp = open(file_path) for line in fp: converters = Converters() try: timestamp, estimated_time, url, response_code, _, _, _ = line.split(",") url = converters.url_converter(url) if max_time.has_key(url): all_requests+=1 if not urls.has_key(url): urls[url] = {} urls[url]['times'] = [] urls[url]['times'].append([estimated_time, response_code]) if response_code != "200": unsuccessfull += 1 except Exception as e: output += "Exception occured\n" output += e.message + "\n" pass for k in urls: count_succ = 0 all = len(urls[k]['times']) for time, response_code in urls[k]['times']: if int(time) <= max_time[k] and response_code == "200": count_succ += 1 dist = (all*100.0)/all_requests if count_succ >= (all * 90) / 100: output += "%-50s %-50s prob = %.2f%%\n" % (k, "OK [no. requests = %s]" % all, dist) else: p = (count_succ*100)/all output += "%-50s %-50s prob = %.2f%%\n" % (k, "NOT OK [all = %s, succ = %s (%s%%) ]" % (all, count_succ, p), dist) fp.close() output += "--------------------------------------------------\n" output += "ALL = %s, UNSUCCESSFULL = %s\n" % (all_requests, unsuccessfull) return output
def to_dataframe(self, file, as_file): print "Parsing " + file converters = Converters() self.data_indexed = pd.read_csv(file, index_col='date', converters={ 'date': converters.timestamp_converter, 'response_code': converters.response_code_converter, 'url': converters.url_converter }) self.data = pd.read_csv(file, converters={ 'date': converters.timestamp_converter, 'response_code': converters.response_code_converter, 'url': converters.url_converter }) # print "Parsing " + as_file # self.autoscalability_data = pd.read_csv(as_file, converters={ # 'start_time' : converters.datetime_to_timestamp, # 'end_time' : converters.datetime_to_timestamp, # 'action' : converters.action_to_number # }) return
def _calc_slo_violations(self, sorted_data, ref_timestamp, seconds): converters = Converters() timestamps = {ref_timestamp: {'num': 0, 'num_all_requests': 0}} for line in sorted_data: timestamp, estimated_time, url, response_code, status, attr1, attr2 = line timestamp = (int(timestamp) / 1000) response_code = converters.response_code_converter(response_code) if ref_timestamp - timestamp < 0: ref_timestamp = ref_timestamp + seconds if not timestamps.has_key(ref_timestamp): timestamps[ref_timestamp] = {'num': 0, 'num_all_requests': 0} timestamps[ref_timestamp]['num_all_requests'] += 1 timestamps[ref_timestamp]['num'] += 1 if self.does_violate_slo(url, estimated_time, response_code) else 0 return timestamps
def slo_agg_seconds(self, parsed_file, output_file, seconds): print "Seconds: %s" % seconds converters = Converters() with open(output_file, 'w') as slo_fp: with open(parsed_file) as fp: next(fp) # skip header parsed_file_data = csv.reader(fp) sorted_data = sorted(parsed_file_data, key = lambda row: int(row[0])) timestamps = {} ref_timestamp, _, _, _, _, _, _ = sorted_data[0] ref_timestamp = (int(ref_timestamp)/1000) timestamps[ref_timestamp] = {'num' : 0, 'num_all_requests': 0} min_date = sys.maxint for line in sorted_data: timestamp, estimated_time, url, response_code, status, attr1, attr2 = line timestamp = (int(timestamp)/1000) if timestamp < min_date: min_date = timestamp response_code = converters.response_code_converter(response_code) time_delta = datetime.datetime.fromtimestamp(timestamp) - datetime.datetime.fromtimestamp(ref_timestamp) # print "time_delta: %s" % time_delta.seconds # print "time_delta: %s" % datetime.datetime.fromtimestamp(timestamp) if time_delta.seconds >= seconds: # print "new ref timestamp: %s" % datetime.datetime.fromtimestamp(timestamp) ref_timestamp = timestamp if not timestamps.has_key(ref_timestamp): timestamps[ref_timestamp] = {'num' : 0, 'num_all_requests': 0} timestamps[ref_timestamp]['num_all_requests'] += 1 i=0 if self.does_violate_slo(url, estimated_time, response_code): i = 1 timestamps[ref_timestamp]['num'] += i print min_date slo_fp.write('"date","num","num_all_requests"\n') slo_fp.write('%s,%s,%s\n' % (0, 0, 0)) for timestamp in timestamps.keys(): timestamp_subtract = (timestamp - min_date)+seconds slo_fp.write('%s,%s,%s\n' % (timestamp_subtract*1000, timestamps[timestamp]['num'],timestamps[timestamp]['num_all_requests']))
def _calc_slo_violations(self, sorted_data, ref_timestamp, seconds): converters = Converters() timestamps = {ref_timestamp: {'num': 0, 'num_all_requests': 0}} for line in sorted_data: timestamp, estimated_time, url, response_code, status, attr1, attr2 = line timestamp = (int(timestamp) / 1000) response_code = converters.response_code_converter(response_code) if ref_timestamp - timestamp < 0: ref_timestamp = ref_timestamp + seconds if not timestamps.has_key(ref_timestamp): timestamps[ref_timestamp] = { 'num': 0, 'num_all_requests': 0 } timestamps[ref_timestamp]['num_all_requests'] += 1 timestamps[ref_timestamp]['num'] += 1 if self.does_violate_slo( url, estimated_time, response_code) else 0 return timestamps
def check(file_path): output = "" urls = {} unsuccessfull = 0 all_requests = 0 fp = open(file_path) for line in fp: converters = Converters() try: timestamp, estimated_time, operation, response_code, _, _, _ = line.split(",") url = converters.url_converter(operation) if url != None and max_time.has_key(url): all_requests += 1 if not urls.has_key(url): urls[url] = {} urls[url]["times"] = [] urls[url]["times"].append([estimated_time, response_code]) if response_code != "200": unsuccessfull += 1 except Exception as e: output += "Exception occured\n" output += e.message + "\n" pass dist_sum = 0 web_interactions = WebInteractions() output += "%-40s %s %-20s %s %-20s %s %-25s %s %-20s %s %-20s %s %-20s %s %-20s %s %-20s\n" % ( "operation", "|", "status", "|", "# all request", "|", "# successfull requests", "|", "% of successfull", "|", "% of operation", "|", "allowed deviation", "|", "actual deviation", "|", "deviation ok?", ) separator = "".join(["-" for _ in xrange(len(output))]) + "\n" output += separator for k in urls: count_succ = 0 all = len(urls[k]["times"]) for time, response_code in urls[k]["times"]: if int(time) <= max_time[k] and response_code == "200": count_succ += 1 dist = (all * 100.0) / all_requests dist_sum = dist_sum + dist allowed_deviation = 0.05 * web_interactions.get_probability(k) status = "NOT OK" p = (count_succ * 100) / all if count_succ >= (all * 90) / 100: status = "OK" p = 0 actual_deviation = get_actual_deviation(dist, k) deviation_ok = is_deviation_ok(dist, allowed_deviation, k) output += "%-40s %s %-20s %s %-20s %s %-25s %s %-20s %s %-20s %s %-20.3f %s %-20s %s %-20s\n" % ( k, "|", status, "|", all, "|", count_succ, "|", p, "|", "%.3f%% (%.3f%%)" % (dist, web_interactions.get_probability(k)), "|", allowed_deviation, "|", actual_deviation, "|", deviation_ok, ) fp.close() output += separator output += "# ALL REQUESTS = %s, # UNSUCCESSFULL REQUESTS = %s, PROB SUM = %s\n" % ( all_requests, unsuccessfull, dist_sum, ) return output
def check(file_path): output = "" urls = {} unsuccessfull = 0 all_requests = 0 fp = open(file_path) for line in fp: converters = Converters() try: timestamp, estimated_time, operation, response_code, _, _, _ = line.split(",") url = converters.url_converter(operation) if url != None and max_time.has_key(url): all_requests+=1 if not urls.has_key(url): urls[url] = {} urls[url]['times'] = [] urls[url]['times'].append([estimated_time, response_code]) if response_code != "200": unsuccessfull += 1 except Exception as e: output += "Exception occured\n" output += e.message + "\n" pass dist_sum = 0 web_interactions = WebInteractions() output += "%-40s %s %-20s %s %-20s %s %-25s %s %-20s %s %-20s %s %-20s %s %-20s %s %-20s\n" % ("operation", "|", "status", "|", "# all request", "|", "# successfull requests", "|", "% of successfull", "|", "% of operation", "|", "allowed deviation", "|", "actual deviation", "|", "deviation ok?") separator = "".join(["-" for _ in xrange(len(output))]) + "\n" output += separator cummulative_violations = 0 cummulative_successful = 0 cummulative_all = 0 for k in urls: count_succ = 0 all = len(urls[k]['times']) for time, response_code in urls[k]['times']: if int(time) <= max_time[k] and response_code == "200": count_succ += 1 cummulative_successful += count_succ cummulative_all += all dist = (all*100.0)/all_requests dist_sum = dist_sum + dist allowed_deviation = 0.05*web_interactions.get_probability(k) status = "NOT OK" p = (count_succ*100)/all if count_succ >= (all * 90) / 100: status = "OK" # p = 0 cummulative_violations += p actual_deviation = get_actual_deviation(dist, k) deviation_ok = is_deviation_ok(dist, allowed_deviation, k) output += "%-40s %s %-20s %s %-20s %s %-25s %s %-20s %s %-20s %s %-20.3f %s %-20s %s %-20s\n" % (k, "|", status, "|", all, "|", count_succ, "|", p, "|", "%.3f%% (%.3f%%)" % (dist, web_interactions.get_probability(k)), "|", allowed_deviation, "|", actual_deviation, "|", deviation_ok) fp.close() output += separator output += "# ALL REQUESTS = %s, # UNSUCCESSFULL REQUESTS = %s, PROB SUM = %s, # SLO VIOLATIONS = %s%%\n" % (all_requests, unsuccessfull, dist_sum, (((cummulative_all-cummulative_successful)*100)/cummulative_all)) return output