def __init__(self, owner):
AbstractInfoWidget.__init__(self)
MonitoringDataDecoder.__init__(self)
self.owner = owner
self.data = {}
self.sign = {}
self.max_metric_len = 0
def __init__(self, owner):
AbstractInfoWidget.__init__(self)
MonitoringDataDecoder.__init__(self)
self.owner = owner
self.data = {}
self.sign = {}
self.max_metric_len = 0
def __init__(self, core):
AbstractPlugin.__init__(self, core)
self.decoder = MonitoringDataDecoder()
self.mon_data = {}
def create_storage():
return {
'avg': defaultdict(list),
'quantiles': defaultdict(list),
'threads': {
'active_threads': []
},
'rps': {
'RPS': []
},
'http_codes': defaultdict(list),
'net_codes': defaultdict(list),
}
self.overall = create_storage()
self.cases = defaultdict(create_storage)
self.start_time = None
self.end_time = None
self.show_graph = None
self.template = os.path.dirname(__file__) + "/report.tpl"
def __init__(self, core):
AbstractPlugin.__init__(self, core)
self.decoder = MonitoringDataDecoder()
self.mon_data = {}
def create_storage():
return {
'avg': defaultdict(list),
'quantiles': defaultdict(list),
'threads': {
'active_threads': []
},
'rps': {
'RPS': []
},
'http_codes': defaultdict(list),
'net_codes': defaultdict(list),
}
self.overall = create_storage()
self.cases = defaultdict(create_storage)
self.start_time = None
self.end_time = None
self.show_graph = None
self.template = None
def __init__(self, autostop, param_str):
AbstractCriteria.__init__(self)
MonitoringDataDecoder.__init__(self)
try:
self.mon = autostop.core.get_plugin_of_type(MonitoringPlugin)
self.mon.monitoring.add_listener(self)
except KeyError:
self.log.warning("No monitoring module, mon autostop disabled")
self.triggered = False
self.autostop = autostop
self.host = param_str.split(",")[0].strip()
self.metric = param_str.split(",")[1].strip()
self.value_limit = float(param_str.split(",")[2])
self.seconds_limit = tankcore.expand_to_seconds(param_str.split(",")[3])
self.last_second = None
self.seconds_count = 0
def __init__(self, autostop, param_str):
AbstractCriteria.__init__(self)
MonitoringDataDecoder.__init__(self)
try:
self.mon = autostop.core.get_plugin_of_type(MonitoringPlugin)
self.mon.monitoring.add_listener(self)
except KeyError:
self.log.warning("No monitoring module, mon autostop disabled")
self.triggered = False
self.autostop = autostop
self.host = param_str.split(',')[0].strip()
self.metric = param_str.split(',')[1].strip()
self.value_limit = float(param_str.split(',')[2])
self.seconds_limit = tankcore.expand_to_seconds(
param_str.split(',')[3])
self.last_second = None
self.seconds_count = 0
class ReportPlugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
"""Graphite data uploader"""
SECTION = 'report'
@staticmethod
def get_key():
return __file__
def __init__(self, core):
AbstractPlugin.__init__(self, core)
self.decoder = MonitoringDataDecoder()
self.mon_data = {}
def create_storage():
return {
'avg': defaultdict(list),
'quantiles': defaultdict(list),
'threads': {
'active_threads': []
},
'rps': {
'RPS': []
},
'http_codes': defaultdict(list),
'net_codes': defaultdict(list),
}
self.overall = create_storage()
self.cases = defaultdict(create_storage)
self.start_time = None
self.end_time = None
self.show_graph = None
self.template = None
def monitoring_data(self, data_string):
self.log.debug("Mon report data: %s", data_string)
for line in data_string.splitlines():
if not line.strip():
continue
def append_data(host, ts, data):
if host not in self.mon_data:
self.mon_data[host] = {}
host_data = self.mon_data[host]
for key, value in data.iteritems():
try:
value = float(value)
if '_' in key:
group, key = key.split('_', 1)
else:
group = key
if group not in host_data:
host_data[group] = {}
group_data = host_data[group]
if key not in group_data:
group_data[key] = []
group_data[key].append((int(ts), value))
except ValueError:
pass
host1, data1, _, ts1 = self.decoder.decode_line(line)
append_data(host1, ts1, data1)
def get_available_options(self):
return ["show_graph", "template"]
def start_test(self):
start_time = datetime.datetime.now()
self.start_time = start_time.strftime("%H:%M%%20%Y%m%d")
def end_test(self, retcode):
end_time = datetime.datetime.now() + datetime.timedelta(minutes=1)
self.end_time = end_time.strftime("%H:%M%%20%Y%m%d")
def configure(self):
"""Read configuration"""
self.show_graph = self.get_option("show_graph", "")
default_template = "/report.tpl"
self.template = self.get_option("template", os.path.dirname(__file__) + default_template)
try:
aggregator = self.core.get_plugin_of_type(AggregatorPlugin)
aggregator.add_result_listener(self)
except KeyError:
self.log.warning("No aggregator module, no valid report will be available")
try:
mon = self.core.get_plugin_of_type(MonitoringPlugin)
if mon.monitoring:
mon.monitoring.add_listener(self)
except KeyError:
self.log.warning("No monitoring module, monitroing report disabled")
def aggregate_second(self, data):
"""
@data: SecondAggregateData
"""
ts = int(time.mktime(data.time.timetuple()))
def add_aggreagted_second(data_item, storage):
data_dict = data_item.__dict__
avg = storage['avg']
for key in ["avg_connect_time", "avg_send_time", "avg_latency", "avg_receive_time"]:
avg[key].append((ts, data_dict.get(key, None)))
quantiles = storage['quantiles']
for key, value in data_item.quantiles.iteritems():
quantiles[key].append((ts, value))
storage['threads']['active_threads'].append((ts, data_item.active_threads))
storage['rps']['RPS'].append((ts, data_item.RPS))
http_codes = storage['http_codes']
for key, value in data_item.http_codes.iteritems():
http_codes[key].append((ts, value))
net_codes = storage['net_codes']
for key, value in data_item.net_codes.iteritems():
net_codes[key].append((ts, value))
add_aggreagted_second(data.overall, self.overall)
for case, case_data in data.cases.iteritems():
add_aggreagted_second(case_data, self.cases[case])
def post_process(self, retcode):
self.log.info("Building HTML report...")
results = {
'overall': self.overall,
'cases': self.cases,
'monitoring': self.mon_data,
}
template = open(self.template, 'r').read()
report_html = self.core.mkstemp(".html", "report_")
self.core.add_artifact_file(report_html)
with open(report_html, 'w') as report_html_file:
report_html_file.write(
string.Template(template).safe_substitute(
metrics=json.dumps(results),
)
)
return retcode
import time
from Tank.MonCollector.collector import MonitoringDataDecoder
mon_decoder = MonitoringDataDecoder()
def uts(dt):
return int(time.mktime(dt.timetuple()))
def parse_number(val):
try:
return float(val)
except ValueError:
return None
def decode_monitoring(data):
data_items = (mon_decoder.decode_line(line) for line in data.splitlines()
if line.strip())
result = {}
for host, metrics, _, ts in data_items:
host_metrics = result \
.setdefault(int(ts), {}) \
.setdefault("monitoring", {}) \
.setdefault(host, {})
for metric_name, value in metrics.iteritems():
if '_' in metric_name:
group, metric_name = metric_name.split('_', 1)
else:
group = metric_name
class ReportPlugin(AbstractPlugin, AggregateResultListener,
MonitoringDataListener):
"""Graphite data uploader"""
SECTION = 'report'
@staticmethod
def get_key():
return __file__
def __init__(self, core):
AbstractPlugin.__init__(self, core)
self.decoder = MonitoringDataDecoder()
self.mon_data = {}
def create_storage():
return {
'avg': defaultdict(list),
'quantiles': defaultdict(list),
'threads': {
'active_threads': []
},
'rps': {
'RPS': []
},
'http_codes': defaultdict(list),
'net_codes': defaultdict(list),
}
self.overall = create_storage()
self.cases = defaultdict(create_storage)
self.start_time = None
self.end_time = None
self.show_graph = None
self.template = os.path.dirname(__file__) + "/report.tpl"
def monitoring_data(self, data_string):
self.log.debug("Mon report data: %s", data_string)
for line in data_string.splitlines():
if not line.strip():
continue
def append_data(host, ts, data):
if host not in self.mon_data:
self.mon_data[host] = {}
host_data = self.mon_data[host]
for key, value in data.iteritems():
try:
value = float(value)
if '_' in key:
group, key = key.split('_', 1)
else:
group = key
if group not in host_data:
host_data[group] = {}
group_data = host_data[group]
if key not in group_data:
group_data[key] = []
group_data[key].append((int(ts), value))
except ValueError:
pass
host1, data1, _, ts1 = self.decoder.decode_line(line)
append_data(host1, ts1, data1)
def get_available_options(self):
return ["show_graph", "template"]
def start_test(self):
start_time = datetime.datetime.now()
self.start_time = start_time.strftime("%H:%M%%20%Y%m%d")
def end_test(self, retcode):
end_time = datetime.datetime.now() + datetime.timedelta(minutes=1)
self.end_time = end_time.strftime("%H:%M%%20%Y%m%d")
def configure(self):
"""Read configuration"""
self.show_graph = self.get_option("show_graph", "")
self.template = self.get_option("template", self.template)
try:
aggregator = self.core.get_plugin_of_type(AggregatorPlugin)
aggregator.add_result_listener(self)
except KeyError:
self.log.warning(
"No aggregator module, no valid report will be available")
try:
mon = self.core.get_plugin_of_type(MonitoringPlugin)
if mon.monitoring:
mon.monitoring.add_listener(self)
except KeyError:
self.log.warning(
"No monitoring module, monitroing report disabled")
def aggregate_second(self, data):
"""
@data: SecondAggregateData
"""
ts = int(time.mktime(data.time.timetuple()))
def add_aggregated_second(data_item, storage):
data_dict = data_item.__dict__
avg = storage['avg']
for key in [
"avg_connect_time", "avg_send_time", "avg_latency",
"avg_receive_time"
]:
avg[key].append((ts, data_dict.get(key, None)))
quantiles = storage['quantiles']
for key, value in data_item.quantiles.iteritems():
quantiles[key].append((ts, value))
storage['threads']['active_threads'].append(
(ts, data_item.active_threads))
storage['rps']['RPS'].append((ts, data_item.RPS))
http_codes = storage['http_codes']
for key, value in data_item.http_codes.iteritems():
http_codes[key].append((ts, value))
net_codes = storage['net_codes']
for key, value in data_item.net_codes.iteritems():
net_codes[key].append((ts, value))
add_aggregated_second(data.overall, self.overall)
for case, case_data in data.cases.iteritems():
add_aggregated_second(case_data, self.cases[case])
def post_process(self, retcode):
self.log.info("Building HTML report...")
results = {
'overall': self.overall,
'cases': self.cases,
'monitoring': self.mon_data,
}
template = open(self.template, 'r').read()
report_html = self.core.mkstemp(".html", "report_")
self.core.add_artifact_file(report_html)
with open(report_html, 'w') as report_html_file:
report_html_file.write(
string.Template(template).safe_substitute(
metrics=json.dumps(results), ))
return retcode