def __init__ (self, metric_type, infile, hostname, output_directory, resource_path, label, ts_start, ts_end,
rule_strings, **other_options):
Metric.__init__(self, metric_type, infile, hostname, output_directory, resource_path, label, ts_start, ts_end,
rule_strings)
self.sub_metric_description = {
'lb': 'Transaction Name',
'lt': 'Time to First byte',
'ts': 'Timestamp',
'tn': 'Transaction Name (Parent)',
's': 'Status',
'ResponseTime': 'Response Time',
'rc': 'Response Code',
'rm': 'Response Message',
'dt': 'Data Type',
'ResponseSize': 'Response Size',
'qps': 'Successful Transactions per second',
'ErrorsPerSecond': 'Errors per second',
'DataThroughput': 'Data Throughput'
}
self.sub_metric_units = {
'lt': 'ms',
'ResponseTime': 'ms',
'ResponseSize': 'bytes',
'qps': 'qps',
'DataThroughput': 'mbps',
'ErrorsPerSecond': 'qps'
}
self.calculated_stats = {}
self.aggregation_granularity = 'minute'
self.calculated_percentiles = {}
self.summary_stats = defaultdict(dict)
self.important_sub_metrics = naarad.naarad_imports.important_sub_metrics_import['JMETER']
if other_options:
for (key, val) in other_options.iteritems():
setattr(self, key, val)
def __init__ (self, metric_type, infile, access, outdir, label, ts_start, ts_end, **other_options):
Metric.__init__(self, metric_type, infile, access, outdir, label, ts_start, ts_end)
for (key,val) in other_options.iteritems():
if key == 'gc-options':
self.gc_options = val.split()
else:
setattr(self, key, val)
self.metric_description = {
"appstop" :"approximate application stop times",
"gen0" :" young gen collection time, excluding gc_prologue & gc_epilogue",
"gen0t" :" young gen collection time, including gc_prologue & gc_epilogue",
"gen0usr" :" young gen collection time in cpu user secs",
"gen0sys" :" young gen collection time in cpu sys secs",
"gen1i" :" train generation incremental collection",
"gen1t" :" old generation collection/full GC",
"cmsIM" :" CMS initial mark pause",
"cmsRM" :" CMS remark pause",
"cmsRS" :" CMS resize pause",
"GC" :" all stop-the-world GC pauses",
"cmsCM" :" CMS concurrent mark phase",
"cmsCP" :" CMS concurrent preclean phase",
"cmsCS" :" CMS concurrent sweep phase",
"cmsCR" :" CMS concurrent reset phase",
"alloc":" object allocation in MB (approximate***)",
"promo":" object promotion in MB (approximate***)",
"used0":" young gen used memory size (before gc)",
"used1":" old gen used memory size (before gc)",
"used":" heap space used memory size (before gc) (excludes perm gen)",
"commit0":" young gen committed memory size (after gc)",
"commit1":" old gen committed memory size (after gc)",
"commit":" heap committed memory size (after gc) (excludes perm gen)",
"apptime" :" amount of time application threads were running",
"safept" :" amount of time the VM spent at safepoints (app threads stopped)"
}
def initialize_metric(section, infile_list, hostname, output_directory, resource_path, label, ts_start, ts_end, rule_strings, important_sub_metrics, anomaly_detection_metrics, other_options):
"""
Initialize appropriate metric based on type of metric.
:param: section: config section name or auto discovered metric type
:param: infile_list: list of input log files for the metric
:param: hostname: hostname associated with the logs origin
:param: output_directory: report location
:param: resource_path: resource path for report
:param: label: label for config section or auto discovered metric type
:param: ts_start: start time for analysis
:param: ts_end: end time for analysis
:param: rule_strings: list of slas
:param: important_sub_metrics: list of important sub metrics
:param: anomaly_detection_metrics: list of metrics to use for anomaly detection.
:param: other_options: kwargs
:return: metric object
"""
bin_path = os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),'bin'))
metric = None
metric_type = section.split('-')[0]
if metric_type in metric_classes:
if 'SAR' in metric_type:
metric = metric_classes['SAR'](section, infile_list, hostname, output_directory, resource_path, label, ts_start, ts_end, rule_strings, important_sub_metrics, anomaly_detection_metrics, **other_options)
else:
metric = metric_classes[metric_type](section, infile_list, hostname, output_directory, resource_path, label, ts_start, ts_end, rule_strings, important_sub_metrics, anomaly_detection_metrics, **other_options)
else:
metric = Metric(section, infile_list, hostname, output_directory, resource_path, label, ts_start, ts_end, rule_strings, important_sub_metrics, anomaly_detection_metrics, **other_options)
metric.bin_path = bin_path
return metric
def parse_metric_section(config_obj, section, metric_classes, metrics, aggregate_metric_classes, outdir_default, resource_path):
"""
Parse a metric section and create a Metric object
:param config_obj: ConfigParser object
:param section: Section name
:param metric_classes: List of valid metric types
:param metrics: List of all regular metric objects (used by aggregate metric)
:param aggregate_metric_classes: List of all valid aggregate metric types
:param outdir_default: Default output directory
:param resource_path: Default resource directory
:return: An initialized Metric object
"""
hostname, infile, aggr_hosts, aggr_metrics, label, ts_start, ts_end, precision, kwargs, rule_strings = parse_basic_metric_options(config_obj, section)
#TODO: Make user specify metric_type in config and not infer from section
metric_type = section.split('-')[0]
if metric_type in metric_classes: # regular metrics
new_metric = metric_classes[metric_type](section, infile, hostname, outdir_default, resource_path, label, ts_start, ts_end, rule_strings, **kwargs)
elif metric_type in aggregate_metric_classes: #aggregate metrics
new_metric = aggregate_metric_classes[metric_type](section, aggr_hosts, aggr_metrics, metrics, outdir_default, resource_path, label, ts_start, ts_end, rule_strings, **kwargs)
else: # new metrics.
new_metric = Metric(section, infile, hostname, outdir_default, resource_path, label, ts_start, ts_end, rule_strings, **kwargs)
if config_obj.has_option(section, 'ignore') and config_obj.getint(section, 'ignore') == 1:
new_metric.ignore = True
if config_obj.has_option(section, 'calc_metrics'):
new_metric.calc_metrics = config_obj.get(section, 'calc_metrics')
new_metric.precision = precision
return new_metric
def __init__(self, metric_type, infile, hostname, aggr_metrics,
output_directory, resource_path, label, ts_start, ts_end,
rule_strings, important_sub_metrics,
anomaly_detection_metrics, **other_options):
Metric.__init__(self, metric_type, infile, hostname, aggr_metrics,
output_directory, resource_path, label, ts_start,
ts_end, rule_strings, important_sub_metrics,
anomaly_detection_metrics)
# Allow user to specify interested processes; in the format of 'PID=11 22' and 'COMMAND=firefox top'
# It will search for any processes that match the PIDs listed or the commands listed. It's not an intersection of the PIDs and commands.
self.PID = []
self.COMMAND = []
self.ts_valid_lines = True
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
self.sub_metrics = None
self.process_headers = []
self.ts = ''
self.ts_date = ''
self.ts_time = ''
self.saw_pid = False # Controls when to process individual commands;
self.data = {} # Stores all data to be written out
for key, val in other_options.iteritems():
setattr(self, key, val.split())
self.sub_metric_description = {
'uptime_minute': 'uptime of the machine',
'num_users': 'users sessions logged in',
'load_aver_1_minute': 'average load on the system (last 1 minute)',
'load_aver_5_minute':
'average load on the system (last 5 minutes)',
'load_aver_15_minute':
'average load on the system (last 15 minutes)',
'tasks_total': 'total processes',
'tasks_running': 'processes running',
'tasks_sleeping': 'processes sleeping',
'tasks_stopped': 'processes stopped',
'tasks_zombie': 'zombies',
'cpu_us': 'cpu percentage of running user processes',
'cpu_sy': 'cpu percentage of running system processes',
'cpu_id': 'cpu percentage of idel time',
'cpu_ni': 'cpu percentage of running niced processes',
'cpu_wa': 'cpu percentage of waiting for IO',
'cpu_hi': 'cpu percentage of serving hardware IRQ',
'cpu_si': 'cpu percentage of serving software IRQ',
'cpu_st': 'cpu percentage of being stolen',
'mem_total': 'total memory in GB',
'mem_used': 'total memory in use in GB',
'mem_free': 'total free memory in GB',
'mem_buffers': 'total buffers in GB',
'swap_total': 'total swap size in GB',
'swap_used': 'total swap in use in GB',
'swap_free': 'total free swap in GB',
'swap_cached': 'total swap cache in GB',
}
def initialize_metric(section, infile_list, hostname, output_directory, resource_path, label, ts_start, ts_end, rule_strings, important_sub_metrics, other_options):
"""
Initialize appropriate metric based on type of metric.
:param: section: config section name or auto discovered metric type
:param: infile_list: list of input log files for the metric
:param: hostname: hostname associated with the logs origin
:param: output_directory: report location
:param: resource_path: resource path for report
:param: label: label for config section or auto discovered metric type
:param: ts_start: start time for analysis
:param: ts_end: end time for analysis
:param: rule_strings: list of slas
:param: important_sub_metrics: list of important sub metrics
:param: other_options: kwargs
:return: metric object
"""
bin_path = os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),'bin'))
metric = None
metric_type = section.split('-')[0]
if metric_type in metric_classes:
if 'SAR' in metric_type:
metric = metric_classes['SAR'](section, infile_list, hostname, output_directory, resource_path, label, ts_start, ts_end, rule_strings, important_sub_metrics, **other_options)
else:
metric = metric_classes[metric_type](section, infile_list, hostname, output_directory, resource_path, label, ts_start, ts_end, rule_strings, important_sub_metrics, **other_options)
else:
metric = Metric(section, infile_list, hostname, output_directory, resource_path, label, ts_start, ts_end, rule_strings, important_sub_metrics, **other_options)
metric.bin_path = bin_path
return metric
def __init__(self, metric_type, infile, hostname, outdir, resource_path,
label, ts_start, ts_end, rule_strings, important_sub_metrics,
anomaly_detection_metrics, **other_options):
Metric.__init__(self, metric_type, infile, hostname, outdir,
resource_path, label, ts_start, ts_end, rule_strings,
important_sub_metrics, anomaly_detection_metrics)
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
def __init__ (self, metric_type, infile, hostname, outdir, resource_path, label, ts_start, ts_end, rule_strings,
**other_options):
Metric.__init__(self, metric_type, infile, hostname, outdir, resource_path, label, ts_start, ts_end, rule_strings)
self.sub_metrics = self.val_types
self.sub_metric_description = {
"launch_time" :"the time taken to launch the client application",
"nus_update_time" :"the time taken to update NUS list after launch"
}
def get_two_metrics(output_directory, resource_path, rules):
metrics = [
Metric('MetricOne', 'TestOne.csv', 'HostnameOne', output_directory,
resource_path, 'MetricOne', None, None, rules, None, None),
Metric('MetricTwo', 'TestTwo.csv', 'HostnameOne', output_directory,
resource_path, 'MetricTwo', None, None, rules, None, None)
]
return metrics
def __init__(self, metric_type, infile_list, hostname, output_directory, resource_path, label, ts_start, ts_end,
rule_strings, important_sub_metrics, anomaly_detection_metrics, **other_options):
Metric.__init__(self, metric_type, infile_list, hostname, output_directory, resource_path, label, ts_start, ts_end,
rule_strings, important_sub_metrics, anomaly_detection_metrics)
self.sub_metrics = None
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
self._extract_input_connections()
self._extract_input_processes()
def __init__(self, metric_type, infile, hostname, outdir, resource_path, label, ts_start, ts_end, rule_strings,
**other_options):
metric_type = self.extract_metric_name(metric_type)
Metric.__init__(self, metric_type, infile, hostname, outdir, resource_path, label, ts_start, ts_end, rule_strings)
if self.metric_type in important_sub_metrics_import.keys():
self.important_sub_metrics = important_sub_metrics_import[self.metric_type]
self.options = None
self.devices = None
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
def __init__(self, metric_type, infile_list, hostname, aggr_metrics, output_directory, resource_path, label, ts_start, ts_end,
rule_strings, important_sub_metrics, anomaly_detection_metrics, **other_options):
Metric.__init__(self, metric_type, infile_list, hostname, aggr_metrics, output_directory, resource_path, label, ts_start, ts_end,
rule_strings, important_sub_metrics, anomaly_detection_metrics)
self.sub_metrics = None
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
self._extract_input_connections()
self._extract_input_processes()
def __init__(self, metric_type, infile, hostname, aggr_metrics, output_directory, resource_path, label, ts_start, ts_end,
rule_strings, important_sub_metrics, anomaly_detection_metrics, **other_options):
Metric.__init__(self, metric_type, infile, hostname, aggr_metrics, output_directory, resource_path, label, ts_start, ts_end,
rule_strings, important_sub_metrics, anomaly_detection_metrics)
# Allow user to specify interested processes; in the format of 'PID=11 22' and 'COMMAND=firefox top'
# It will search for any processes that match the PIDs listed or the commands listed. It's not an intersection of the PIDs and commands.
self.PID = []
self.COMMAND = []
self.ts_valid_lines = True
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
self.sub_metrics = None
self.process_headers = []
self.ts = ''
self.ts_date = ''
self.ts_time = ''
self.saw_pid = False # Controls when to process individual commands;
self.data = {} # Stores all data to be written out
for key, val in other_options.iteritems():
setattr(self, key, val.split())
self.sub_metric_description = {
'uptime_minute': 'uptime of the machine',
'num_users': 'users sessions logged in',
'load_aver_1_minute': 'average load on the system (last 1 minute)',
'load_aver_5_minute': 'average load on the system (last 5 minutes)',
'load_aver_15_minute': 'average load on the system (last 15 minutes)',
'tasks_total': 'total processes',
'tasks_running': 'processes running',
'tasks_sleeping': 'processes sleeping',
'tasks_stopped': 'processes stopped',
'tasks_zombie': 'zombies',
'cpu_us': 'cpu percentage of running user processes',
'cpu_sy': 'cpu percentage of running system processes',
'cpu_id': 'cpu percentage of idel time',
'cpu_ni': 'cpu percentage of running niced processes',
'cpu_wa': 'cpu percentage of waiting for IO',
'cpu_hi': 'cpu percentage of serving hardware IRQ',
'cpu_si': 'cpu percentage of serving software IRQ',
'cpu_st': 'cpu percentage of being stolen',
'mem_total': 'total memory in GB',
'mem_used': 'total memory in use in GB',
'mem_free': 'total free memory in GB',
'mem_buffers': 'total buffers in GB',
'swap_total': 'total swap size in GB',
'swap_used': 'total swap in use in GB',
'swap_free': 'total free swap in GB',
'swap_cached': 'total swap cache in GB',
}
def __init__(self, metric_type, infile_list, hostname, aggr_metrics, outdir, resource_path, label, ts_start, ts_end, rule_strings,
important_sub_metrics, anomaly_detection_metrics, **other_options):
Metric.__init__(self, metric_type, infile_list, hostname, aggr_metrics, outdir, resource_path, label, ts_start, ts_end, rule_strings,
important_sub_metrics, anomaly_detection_metrics)
self.sub_metrics = self.val_types
if not self.important_sub_metrics:
self.important_sub_metrics = CONSTANTS.important_sub_metrics_import['LINKEDINANDROIDRUM']
self.sub_metric_description = {
"launch_time": "the time taken to launch the client application",
"nus_update_time": "the time taken to update NUS list after launch"
}
def __init__(self, metric_type, infile_list, hostname, outdir, resource_path, label, ts_start, ts_end, rule_strings,
important_sub_metrics, anomaly_detection_metrics, **other_options):
Metric.__init__(self, metric_type, infile_list, hostname, outdir, resource_path, label, ts_start, ts_end, rule_strings,
important_sub_metrics, anomaly_detection_metrics)
if not self.important_sub_metrics and self.metric_type in important_sub_metrics_import.keys():
self.important_sub_metrics = important_sub_metrics_import[self.metric_type]
self.options = None
self.CPUS = None
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
def __init__(self, metric_type, infile_list, hostname, outdir, resource_path, label, ts_start, ts_end, rule_strings,
important_sub_metrics, **other_options):
metric_type = self.extract_metric_name(metric_type)
Metric.__init__(self, metric_type, infile_list, hostname, outdir, resource_path, label, ts_start, ts_end, rule_strings,
important_sub_metrics)
if not self.important_sub_metrics and self.metric_type in important_sub_metrics_import.keys():
self.important_sub_metrics = important_sub_metrics_import[self.metric_type]
self.options = None
self.devices = None
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
def __init__ (self, section, aggregate_hosts, aggregate_metrics, metrics, output_directory, resource_path, label,
ts_start, ts_end, rule_strings, important_sub_metrics, anomaly_detection_metrics, **other_options):
self.metrics = metrics
self.aggr_metrics = aggregate_metrics.split()
self.aggr_hosts = aggregate_hosts.split()
#Metric arguments take 'infile' and 'hostname', for ClusterMetric, they are invalid, so just provide empty strings.
Metric.__init__(self, section, '', '', output_directory, resource_path, label, ts_start, ts_end, rule_strings,
important_sub_metrics, anomaly_detection_metrics)
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
def __init__ (self, section, aggregate_hosts, aggregate_metrics, metrics, output_directory, resource_path, label, ts_start, ts_end,
rule_strings, **other_options):
self.metrics = metrics
self.aggr_metrics = re.split(",| |:", aggregate_metrics) #support both "," and " ", ":" as separator
self.aggr_hosts = re.split(",| |:", aggregate_hosts)
#Metric arguments take 'infile' and 'hostname', for ClusterMetric, they are invalid, so just provide empty strings.
Metric.__init__(self, section, '', '', output_directory, resource_path, label, ts_start, ts_end, rule_strings)
# in particular, Section can specify a subset of all rows (default has 43 rows): "sub_metrics=nr_free_pages nr_inactive_anon"
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
def __init__(self, section, aggregate_hosts, aggregate_metrics, metrics, output_directory, resource_path, label,
ts_start, ts_end, rule_strings, important_sub_metrics, anomaly_detection_metrics, **other_options):
self.metrics = metrics
self.aggr_hosts = aggregate_hosts.split()
# Metric arguments take 'infile' and 'hostname', for ClusterMetric, they are invalid, so just provide empty strings.
Metric.__init__(self, section, '', '', '', output_directory, resource_path, label, ts_start, ts_end, rule_strings,
important_sub_metrics, anomaly_detection_metrics)
self.aggr_metrics = aggregate_metrics.split()
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
def __init__ (self, metric_type, infile_list, hostname, output_directory, resource_path, label, ts_start, ts_end,
rule_strings, important_sub_metrics, anomaly_detection_metrics, **other_options):
Metric.__init__(self, metric_type, infile_list, hostname, output_directory, resource_path, label, ts_start, ts_end,
rule_strings, important_sub_metrics, anomaly_detection_metrics)
# in particular, Section can specify a subset of all rows (default has 43 rows): "sub_metrics=nr_free_pages nr_inactive_anon"
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
self.sub_metric_description = {
'MemTotal': 'Total memory in KB',
'MemFree': 'Total free memory in KB',
'Buffers': 'Size of buffers in KB',
'Cached': 'Size of page cache in KB',
}
def plot_timeseries(self, graphing_library='matplotlib'):
if graphing_library != 'matplotlib':
return Metric.plot_timeseries(self, graphing_library)
else:
logger.info('Using graphing_library {lib} for metric {name}'.format(lib=graphing_library, name=self.label))
plot_data = {}
# plot time series data for submetrics
for out_csv in sorted(self.csv_files, reverse=True):
csv_filename = os.path.basename(out_csv)
# The last element is .csv, don't need that in the name of the chart
column = csv_filename.split('.')[-2]
transaction_name = ' '.join(csv_filename.split('.')[1:-2])
plot = PD(input_csv=out_csv, csv_column=1, series_name=transaction_name,
y_label=self.sub_metric_description[column] + ' (' + self.sub_metric_units[column] + ')', precision=None, graph_height=500, graph_width=1200,
graph_type='line')
if transaction_name in plot_data:
plot_data[transaction_name].append(plot)
else:
plot_data[transaction_name] = [plot]
for transaction in plot_data:
graphed, div_file = Metric.graphing_modules[graphing_library].graph_data(plot_data[transaction], self.resource_directory, self.resource_path,
self.label + '.' + transaction)
if graphed:
self.plot_files.append(div_file)
return True
def plot_timeseries(self, graphing_library='matplotlib'):
if graphing_library != 'matplotlib':
return Metric.plot_timeseries(self, graphing_library)
else:
logger.info(
'Using graphing_library {lib} for metric {name}'.format(
lib=graphing_library, name=self.label))
plot_data = {}
# plot time series data for submetrics
for out_csv in sorted(self.csv_files, reverse=True):
csv_filename = os.path.basename(out_csv)
# The last element is .csv, don't need that in the name of the chart
column = csv_filename.split('.')[-2]
transaction_name = ' '.join(csv_filename.split('.')[1:-2])
plot = PD(input_csv=out_csv,
csv_column=1,
series_name=transaction_name,
y_label=self.sub_metric_description[column] + ' (' +
self.sub_metric_units[column] + ')',
precision=None,
graph_height=500,
graph_width=1200,
graph_type='line')
if transaction_name in plot_data:
plot_data[transaction_name].append(plot)
else:
plot_data[transaction_name] = [plot]
for transaction in plot_data:
graphed, div_file = Metric.graphing_modules[
graphing_library].graph_data(
plot_data[transaction], self.resource_directory,
self.resource_path, self.label + '.' + transaction)
if graphed:
self.plot_files.append(div_file)
return True
def __init__(self, metric_type, infile_list, hostname, aggr_metrics,
outdir, resource_path, label, ts_start, ts_end, rule_strings,
important_sub_metrics, anomaly_detection_metrics,
**other_options):
Metric.__init__(self, metric_type, infile_list, hostname, aggr_metrics,
outdir, resource_path, label, ts_start, ts_end,
rule_strings, important_sub_metrics,
anomaly_detection_metrics)
if not self.important_sub_metrics and self.metric_type in important_sub_metrics_import.keys(
):
self.important_sub_metrics = important_sub_metrics_import[
self.metric_type]
self.options = None
self.CPUS = None
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
def __init__(self, metric_type, infile_list, hostname, output_directory,
resource_path, label, ts_start, ts_end, rule_strings,
important_sub_metrics, **other_options):
Metric.__init__(self, metric_type, infile_list, hostname,
output_directory, resource_path, label, ts_start,
ts_end, rule_strings, important_sub_metrics)
# in particular, Section can specify a subset of all rows (default has 43 rows): "sub_metrics=nr_free_pages nr_inactive_anon"
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
self.sub_metric_description = {
'MemTotal': 'Total memory in KB',
'MemFree': 'Total free memory in KB',
'Buffers': 'Size of buffers in KB',
'Cached': 'Size of page cache in KB',
}
def __init__(
self,
metric_type,
infile_list,
hostname,
aggr_metrics,
output_directory,
resource_path,
label,
ts_start,
ts_end,
rule_strings,
important_sub_metrics,
anomaly_detection_metrics,
**other_options
):
Metric.__init__(
self,
metric_type,
infile_list,
hostname,
aggr_metrics,
output_directory,
resource_path,
label,
ts_start,
ts_end,
rule_strings,
important_sub_metrics,
anomaly_detection_metrics,
)
self.sub_metrics = None
# in particular, Section can specify a subset of all metrics: sub_metrics=pages.min nr_free_pages
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
self.sub_metric_description = {
"nr_free_pages": "Number of free pages",
"nr_inactive_anon": "Number of inactive anonymous pages",
"nr_active_anon": "Number of active anonymous pages",
"nr_inactive_file": "Number of inactive file cache pages",
"nr_active_file": "Number of active file cache pages",
}
def __init__ (self, metric_type, infile_list, hostname, output_directory, resource_path, label, ts_start, ts_end,
rule_strings, important_sub_metrics, anomaly_detection_metrics, **other_options):
Metric.__init__(self, metric_type, infile_list, hostname, output_directory, resource_path, label, ts_start, ts_end,
rule_strings, important_sub_metrics, anomaly_detection_metrics)
self.sub_metrics = None
# in particular, Section can specify a subset of all rows (default has 86 rows): "sub_metrics=nr_free_pages nr_inactive_anon"
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
self.sub_metric_description = {
'nr_free_pages': 'Number of free pages',
'nr_inactive_anon': 'Number of inactive anonymous pages',
'nr_active_anon': 'Number of active anonymous pages',
'nr_inactive_file': 'Number of inactive file pages',
'nr_active_file': 'Number of active file pages',
}
def __init__ (self, metric_type, infile, hostname, outdir, resource_path, label, ts_start, ts_end, rule_strings,
**other_options):
Metric.__init__(self, metric_type, infile, hostname, outdir, resource_path, label, ts_start, ts_end, rule_strings)
# TODO: Make this list configurable
self.important_sub_metrics = important_sub_metrics_import['GC']
self.sub_metrics = self.val_types
self.beginning_ts = None
self.beginning_date = None
for (key, val) in other_options.iteritems():
if key == 'gc-options':
self.sub_metrics = val.split()
else:
setattr(self, key, val)
self.sub_metric_description = {
'appstop' : 'approximate application stop times',
'gen0' : 'young gen collection time, excluding gc_prologue & gc_epilogue',
'gen0t' : 'young gen collection time, including gc_prologue & gc_epilogue',
'gen0usr' : 'young gen collection time in cpu user secs',
'gen0sys' : 'young gen collection time in cpu sys secs',
'gen0real' : 'young gen collection time in elapsed secs',
'gen1i' : 'train generation incremental collection',
'gen1t' : 'old generation collection/full GC',
'cmsIM' : 'CMS initial mark pause',
'cmsRM' : 'CMS remark pause',
'cmsRS' : 'CMS resize pause',
'GCPause' : 'all stop-the-world GC pauses',
'cmsCM' : 'CMS concurrent mark phase',
'cmsCP' : 'CMS concurrent preclean phase',
'cmsCS' : 'CMS concurrent sweep phase',
'cmsCR' : 'CMS concurrent reset phase',
'alloc' : 'object allocation in MB (approximate***)',
'promo' : 'object promotion in MB (approximate***)',
'used0' : 'young gen used memory size (before gc)',
'used1' : 'old gen used memory size (before gc)',
'used' : 'heap space used memory size (before gc) (excludes perm gen)',
'commit0' : 'young gen committed memory size (after gc)',
'commit1' : 'old gen committed memory size (after gc)',
'commit' : 'heap committed memory size (after gc) (excludes perm gen)',
'apptime' : 'amount of time application threads were running',
'safept' : 'amount of time the VM spent at safepoints (app threads stopped)',
'used0AfterGC' : 'young gen used memory size (after gc)',
'used1AfterGC' : 'old gen used memory size (after gc)',
'usedAfterGC' : 'heap space used memory size (after gc)'
}
def __init__(self, metric_type, infile_list, hostname, output_directory,
resource_path, label, ts_start, ts_end, rule_strings,
important_sub_metrics, anomaly_detection_metrics,
**other_options):
Metric.__init__(self, metric_type, infile_list, hostname,
output_directory, resource_path, label, ts_start,
ts_end, rule_strings, important_sub_metrics,
anomaly_detection_metrics)
self.sub_metric_description = {
'lb': 'Transaction Name',
'lt': 'Time to First byte',
'ts': 'Timestamp',
'tn': 'Transaction Name (Parent)',
's': 'Status',
'ResponseTime': 'Response Time',
'rc': 'Response Code',
'rm': 'Response Message',
'dt': 'Data Type',
'ResponseSize': 'Response Size',
'qps': 'Successful Transactions per second',
'ErrorsPerSecond': 'Errors per second',
'DataThroughput': 'Data Throughput'
}
self.sub_metric_units = {
'lt': 'ms',
'ResponseTime': 'ms',
'ResponseSize': 'bytes',
'qps': 'qps',
'DataThroughput': 'mbps',
'ErrorsPerSecond': 'qps'
}
self.calculated_stats = {}
self.aggregation_granularity = 'second'
self.calculated_percentiles = {}
self.summary_stats = defaultdict(dict)
self.summary_html_content_enabled = True
self.summary_charts = [self.label + '.Overall_Summary.div']
if not self.important_sub_metrics:
self.important_sub_metrics = important_sub_metrics_import['JMETER']
if other_options:
for (key, val) in other_options.iteritems():
setattr(self, key, val)
def test_metrics_with_summary_with_partial_error():
"""
Tests to verify that metric reports are generated for OK metrics if there are some metrics that are in error. Also a summary report should be created.
"""
global output_directory
global input_log_directory
global resource_path
rules = {}
metrics = get_two_metrics(output_directory, resource_path, rules)
files_list = [
os.path.join(input_log_directory, 'a.csv'),
os.path.join(input_log_directory, 'b.csv')
]
for metric in metrics:
metric.csv_files = files_list
metric.stats_files = files_list
metric.timeseries_csv_list = files_list
metric.important_stats_files = files_list
metric.percentiles_files = files_list
metrics.append(
Metric('MetricThree', 'TestThree.csv', 'HostnameOne', output_directory,
resource_path, 'MetricThree', None, None, rules, None, None))
aggregate_metrics = []
correlated_plots = []
rpt = Report(None,
output_directory,
resource_directory,
resource_path,
metrics + aggregate_metrics,
correlated_plots=correlated_plots)
rpt.generate()
assert naarad.utils.is_valid_file(
os.path.join(output_directory, 'MetricOne_report.html'))
assert naarad.utils.is_valid_file(
os.path.join(output_directory, 'MetricTwo_report.html'))
assert not naarad.utils.is_valid_file(
os.path.join(output_directory, 'MetricThree_report.html'))
assert naarad.utils.is_valid_file(
os.path.join(output_directory, 'report.html'))
assert naarad.utils.is_valid_file(
os.path.join(output_directory, 'summary_report.html'))
os.system('rm -rf tmp_report_test/*.*')
os.system('rm -rf tmp_report_test/resources/*.*')
def get_csv(self, column, func):
csv_file = Metric.get_csv(self, column + '.' + func)
return csv_file
def __init__(self, metric_type, infile_list, hostname, aggr_metrics, outdir, resource_path, label, ts_start, ts_end, rule_strings,
important_sub_metrics, anomaly_detection_metrics, **other_options):
Metric.__init__(self, metric_type, infile_list, hostname, aggr_metrics, outdir, resource_path, label, ts_start, ts_end, rule_strings,
important_sub_metrics, anomaly_detection_metrics)
if not self.important_sub_metrics:
self.important_sub_metrics = important_sub_metrics_import['GC']
self.sub_metrics = self.val_types
self.beginning_ts = None
self.beginning_date = None
for (key, val) in other_options.iteritems():
if key == 'gc-options' or key == 'sub_metrics':
self.sub_metrics = val.split()
else:
setattr(self, key, val)
self.sub_metric_description = {
'gen0': 'young gen collection time, excluding gc_prologue & gc_epilogue',
'gen0t': 'young gen collection time, including gc_prologue & gc_epilogue',
'gen0usr': '******',
'gen0sys': 'young gen collection time in cpu sys secs',
'gen0real': 'young gen collection time in elapsed secs',
'gen1i': 'train generation incremental collection',
'gen1t': 'old generation collection or full GC',
'cmsIM': 'CMS initial mark pause',
'cmsRM': 'CMS remark pause',
'cmsRS': 'CMS resize pause',
'GCPause': 'all stop-the-world GC pauses',
'cmsCM': 'CMS concurrent mark phase',
'cmsCP': 'CMS concurrent preclean phase',
'cmsCS': 'CMS concurrent sweep phase',
'cmsCR': 'CMS concurrent reset phase',
'alloc': 'object allocation in MB (approximate***)',
'promo': 'object promotion in MB (approximate***)',
'used0': 'young gen used memory size (before gc)',
'used1': 'old gen used memory size (before gc)',
'used': 'heap space used memory size (before gc) (excludes perm gen)',
'commit0': 'young gen committed memory size (after gc)',
'commit1': 'old gen committed memory size (after gc)',
'commit': 'heap committed memory size (after gc) (excludes perm gen)',
'apptime': 'amount of time application threads were running',
'safept': 'amount of time the VM spent at safepoints (app threads stopped)',
'used0AfterGC': 'young gen used memory size (after gc)',
'used1AfterGC': 'old gen used memory size (after gc)',
'usedAfterGC': 'heap space used memory size (after gc)',
'g1-pause-young': 'G1 Young GC Pause (seconds)',
'g1-pause-mixed': 'G1 Mixed GC Pause (seconds)',
'g1-pause-remark': 'G1 Remark Pause (seconds)',
'g1-pause-cleanup': 'G1 Cleanup Pause (seconds)',
'g1-pause-remark.ref-proc': 'G1 Remark: Reference Processing (seconds)',
'g1-pause-young.parallel': 'G1 Young GC Pause: Parallel Operations (ms)',
'g1-pause-young.parallel.gcworkers': 'G1 Young GC Pause: Number of Parallel GC Workers',
'g1-pause-young.parallel.gc-worker-start.avg': 'G1 Young GC Pause : Parallel : Avg Time spent in GC worker start (ms)',
'g1-pause-young.parallel.gc-worker-start.max': 'G1 Young GC Pause : Parallel : Max Time spent in GC worker start (ms)',
'g1-pause-young.parallel.ext-root-scanning.avg': 'G1 Young GC Pause: Avg Time spent in ext-root-scanning',
'g1-pause-young.parallel.ext-root-scanning.max': 'G1 Young GC Pause: Max Time spent in ext-root-scanning',
'g1-pause-young.parallel.update-rs.avg': 'G1 Young GC Pause: Parallel : Avg Time spent in updating Rsets',
'g1-pause-young.parallel.update-rs.max': 'G1 Young GC Pause: Parallel : Max Time spent in updating Rsets',
'g1-pause-young.parallel.update-rs.processed-buffers.avg': 'G1 Young GC Pause : Parallel : Update Rset : Avg number of processed buffers',
'g1-pause-young.parallel.update-rs.processed-buffers.max': 'G1 Young GC Pause : Parallel : Update Rset : Max number of processed buffers',
'g1-pause-young.parallel.scan-rs.avg': 'G1 Young GC Pause: Parallel : Avg Time spent in scanning Rsets',
'g1-pause-young.parallel.scan-rs.max': 'G1 Young GC Pause: Parallel : Max Time spent in scannning Rsets',
'g1-pause-young.parallel.object-copy-rs.avg': 'G1 Young GC Pause : Parallel : Avg Time spent in Object Copy',
'g1-pause-young.parallel.object-copy-rs.max': 'G1 Young GC Pause : Parallel : Max Time spent in Object Copy',
'g1-pause-young.parallel.termination.avg': 'G1 Young GC Pause : Parallel : Avg Time spent in termination',
'g1-pause-young.parallel.termination.max': 'G1 Young GC Pause : Parallel : Max Time spent in termination',
'g1-pause-young.parallel.gc-worker-other.avg': 'G1 Young GC Pause : Parallel : Avg Time spent in other',
'g1-pause-young.parallel.gc-worker-other.max': 'G1 Young GC Pause : Parallel : Max Time spent in other',
'g1-pause-young.parallel.gc-worker-total.avg': 'G1 Young GC Pause : Parallel : Avg Total time for GC worker',
'g1-pause-young.parallel.gc-worker-total.max': 'G1 Young GC Pause : Parallel : Max Total time for GC worker',
'g1-pause-young.parallel.gc-worker-end.avg': 'G1 Young GC Pause : Parallel : Avg Time for GC worker end',
'g1-pause-young.parallel.gc-worker-end.max': 'G1 Young GC Pause : Parallel : Max Time for GC worker end',
'g1-pause-young.code-root-fixup': 'G1 Young GC Pause : Time spent in code root fixup (ms)',
'g1-pause-young.clear-ct': 'G1 Young GC Pause: Time spent in clear ct (ms)',
'g1-pause-young.other': 'G1 Young GC Pause: Time spent in other (ms)',
'g1-pause-young.other.choose-cset': 'G1 Young GC Pause : Other : Time spent in choosing CSet (ms)',
'g1-pause-young.other.ref-proc': 'G1 Young GC Pause : Other : Time spent in reference processing (ms)',
'g1-pause-young.other.reg-enq': 'G1 Young GC Pause : Other : Time spent in reg-enq(ms)',
'g1-pause-young.other.free-cset': 'G1 Young GC Pause : Other : Time spent in processing free Cset(ms)',
'g1-pause-mixed.parallel': 'G1 Mixed GC Pause: Parallel Operations (ms)',
'g1-pause-mixed.parallel.gcworkers': 'G1 Mixed GC Pause: Number of Parallel GC Workers',
'g1-pause-mixed.parallel.gc-worker-start.avg': 'G1 Mixed GC Pause : Parallel : Avg Time spent in GC worker start (ms)',
'g1-pause-mixed.parallel.gc-worker-start.max': 'G1 Mixed GC Pause : Parallel : Max Time spent in GC worker start (ms)',
'g1-pause-mixed.parallel.ext-root-scanning.avg': 'G1 Mixed GC Pause: Avg Time spent in ext-root-scanning',
'g1-pause-mixed.parallel.ext-root-scanning.max': 'G1 Mixed GC Pause: Max Time spent in ext-root-scanning',
'g1-pause-mixed.parallel.update-rs.avg': 'G1 Mixed GC Pause: Parallel : Avg Time spent in updating Rsets',
'g1-pause-mixed.parallel.update-rs.max': 'G1 Mixed GC Pause: Parallel : Max Time spent in updating Rsets',
'g1-pause-mixed.parallel.update-rs.processed-buffers.avg': 'G1 Mixed GC Pause : Parallel : Update Rset : Avg number of processed buffers',
'g1-pause-mixed.parallel.update-rs.processed-buffers.max': 'G1 Mixed GC Pause : Parallel : Update Rset : Max number of processed buffers',
'g1-pause-mixed.parallel.scan-rs.avg': 'G1 Mixed GC Pause: Parallel : Avg Time spent in scanning Rsets',
'g1-pause-mixed.parallel.scan-rs.max': 'G1 Mixed GC Pause: Parallel : Max Time spent in scannning Rsets',
'g1-pause-mixed.parallel.object-copy-rs.avg': 'G1 Mixed GC Pause : Parallel : Avg Time spent in Object Copy',
'g1-pause-mixed.parallel.object-copy-rs.max': 'G1 Mixed GC Pause : Parallel : Max Time spent in Object Copy',
'g1-pause-mixed.parallel.termination.avg': 'G1 Mixed GC Pause : Parallel : Avg Time spent in termination',
'g1-pause-mixed.parallel.termination.max': 'G1 Mixed GC Pause : Parallel : Max Time spent in termination',
'g1-pause-mixed.parallel.gc-worker-other.avg': 'G1 Mixed GC Pause : Parallel : Avg Time spent in other',
'g1-pause-mixed.parallel.gc-worker-other.max': 'G1 Mixed GC Pause : Parallel : Max Time spent in other',
'g1-pause-mixed.parallel.gc-worker-total.avg': 'G1 Mixed GC Pause : Parallel : Avg Total time for GC worker',
'g1-pause-mixed.parallel.gc-worker-total.max': 'G1 Mixed GC Pause : Parallel : Max Total time for GC worker',
'g1-pause-mixed.parallel.gc-worker-end.avg': 'G1 Mixed GC Pause : Parallel : Avg Time for GC worker end',
'g1-pause-mixed.parallel.gc-worker-end.max': 'G1 Mixed GC Pause : Parallel : Max Time for GC worker end',
'g1-pause-mixed.code-root-fixup': 'G1 Mixed GC Pause : Time spent in code root fixup (ms)',
'g1-pause-mixed.clear-ct': 'G1 Mixed GC Pause: Time spent in clear ct (ms)',
'g1-pause-mixed.other': 'G1 Mixed GC Pause: Time spent in other (ms)',
'g1-pause-mixed.other.choose-cset': 'G1 Mixed GC Pause : Other : Time spent in choosing CSet (ms)',
'g1-pause-mixed.other.ref-proc': 'G1 Mixed GC Pause : Other : Time spent in reference processing (ms)',
'g1-pause-mixed.other.reg-enq': 'G1 Mixed GC Pause : Other : Time spent in reg-enq(ms)',
'g1-pause-mixed.other.free-cset': 'G1 Mixed GC Pause : Other : Time spent in processing free Cset(ms)',
'g1-eden-occupancy-before-gc': 'G1 Eden Occupancy (MB) (Before GC)',
'g1-eden-capacity-before-gc': 'G1 Eden Capacity (MB) (Before GC)',
'g1-eden-occupancy-after-gc': 'G1 Eden Occupancy (MB) (After GC)',
'g1-eden-capacity-after-gc': 'G1 Eden Capacity (MB) (After GC)',
'g1-survivor-before-gc': 'G1 Survivor Size (MB) (Before GC)',
'g1-survivor-after-gc': 'G1 Survivor Size (MB) (After GC)',
'g1-heap-occupancy-before-gc': 'G1 Heap Occupancy (MB) (Before GC)',
'g1-heap-capacity-before-gc': 'G1 Heap Capacity (MB) (Before GC)',
'g1-heap-occupancy-after-gc': 'G1 Heap Occupancy (MB) (After GC)',
'g1-heap-capacity-after-gc': 'G1 Heap Capacity (MB) (After GC)',
'g1-young-cpu.sys': 'G1 Young GC : sys cpu time (seconds)',
'g1-young-cpu.usr': '******',
'g1-young-cpu.real': 'G1 Young GC : elapsed time (seconds)',
'g1-mixed-cpu.usr': '******',
'g1-mixed-cpu.sys': 'G1 Mixed GC : sys cpu time (seconds)',
'g1-mixed-cpu.real': 'G1 Mixed GC : elapsed time (seconds)'
}
def get_csv(self, column, func):
csv_file = Metric.get_csv(self, column + "." + func)
return csv_file
def __init__(self, metric_type, infile, hostname, outdir, resource_path, label, ts_start, ts_end, rule_strings, **other_options):
Metric.__init__(self, metric_type, infile, hostname, outdir, resource_path, label, ts_start, ts_end, rule_strings)
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
def __init__(self, metric_type, infile, access, outdir, label, ts_start,
ts_end, **other_options):
Metric.__init__(self, metric_type, infile, access, outdir, label,
ts_start, ts_end)
for (key, val) in other_options.iteritems():
if key == 'gc-options':
self.gc_options = val.split()
else:
setattr(self, key, val)
self.metric_description = {
"appstop":
"approximate application stop times",
"gen0":
" young gen collection time, excluding gc_prologue & gc_epilogue",
"gen0t":
" young gen collection time, including gc_prologue & gc_epilogue",
"gen0usr":
"******",
"gen0sys":
" young gen collection time in cpu sys secs",
"gen1i":
" train generation incremental collection",
"gen1t":
" old generation collection/full GC",
"cmsIM":
" CMS initial mark pause",
"cmsRM":
" CMS remark pause",
"cmsRS":
" CMS resize pause",
"GC":
" all stop-the-world GC pauses",
"cmsCM":
" CMS concurrent mark phase",
"cmsCP":
" CMS concurrent preclean phase",
"cmsCS":
" CMS concurrent sweep phase",
"cmsCR":
" CMS concurrent reset phase",
"alloc":
" object allocation in MB (approximate***)",
"promo":
" object promotion in MB (approximate***)",
"used0":
" young gen used memory size (before gc)",
"used1":
" old gen used memory size (before gc)",
"used":
" heap space used memory size (before gc) (excludes perm gen)",
"commit0":
" young gen committed memory size (after gc)",
"commit1":
" old gen committed memory size (after gc)",
"commit":
" heap committed memory size (after gc) (excludes perm gen)",
"apptime":
" amount of time application threads were running",
"safept":
" amount of time the VM spent at safepoints (app threads stopped)"
}
def __init__(self, metric_type, infile, access, outdir, label, ts_start,
ts_end, **other_options):
Metric.__init__(self, metric_type, infile, access, outdir, label,
ts_start, ts_end)
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
def __init__(
self,
metric_type,
infile_list,
hostname,
aggr_metrics,
outdir,
resource_path,
label,
ts_start,
ts_end,
rule_strings,
important_sub_metrics,
anomaly_detection_metrics,
**other_options
):
Metric.__init__(
self,
metric_type,
infile_list,
hostname,
aggr_metrics,
outdir,
resource_path,
label,
ts_start,
ts_end,
rule_strings,
important_sub_metrics,
anomaly_detection_metrics,
)
if not self.important_sub_metrics:
self.important_sub_metrics = important_sub_metrics_import["GC"]
self.sub_metrics = self.val_types
self.beginning_ts = None
self.beginning_date = None
for (key, val) in other_options.iteritems():
if key == "gc-options" or key == "sub_metrics":
self.sub_metrics = val.split()
else:
setattr(self, key, val)
self.sub_metric_description = {
"gen0": "young gen collection time, excluding gc_prologue & gc_epilogue",
"gen0t": "young gen collection time, including gc_prologue & gc_epilogue",
"gen0usr": "******",
"gen0sys": "young gen collection time in cpu sys secs",
"gen0real": "young gen collection time in elapsed secs",
"gen1i": "train generation incremental collection",
"gen1t": "old generation collection or full GC",
"cmsIM": "CMS initial mark pause",
"cmsRM": "CMS remark pause",
"cmsRS": "CMS resize pause",
"GCPause": "all stop-the-world GC pauses",
"cmsCM": "CMS concurrent mark phase",
"cmsCP": "CMS concurrent preclean phase",
"cmsCS": "CMS concurrent sweep phase",
"cmsCR": "CMS concurrent reset phase",
"alloc": "object allocation in MB (approximate***)",
"promo": "object promotion in MB (approximate***)",
"used0": "young gen used memory size (before gc)",
"used1": "old gen used memory size (before gc)",
"used": "heap space used memory size (before gc) (excludes perm gen)",
"commit0": "young gen committed memory size (after gc)",
"commit1": "old gen committed memory size (after gc)",
"commit": "heap committed memory size (after gc) (excludes perm gen)",
"apptime": "amount of time application threads were running",
"safept": "amount of time the VM spent at safepoints (app threads stopped)",
"used0AfterGC": "young gen used memory size (after gc)",
"used1AfterGC": "old gen used memory size (after gc)",
"usedAfterGC": "heap space used memory size (after gc)",
"g1-pause-young": "G1 Young GC Pause (seconds)",
"g1-pause-mixed": "G1 Mixed GC Pause (seconds)",
"g1-pause-remark": "G1 Remark Pause (seconds)",
"g1-pause-cleanup": "G1 Cleanup Pause (seconds)",
"g1-pause-remark.ref-proc": "G1 Remark: Reference Processing (seconds)",
"g1-pause-young.parallel": "G1 Young GC Pause: Parallel Operations (ms)",
"g1-pause-young.parallel.gcworkers": "G1 Young GC Pause: Number of Parallel GC Workers",
"g1-pause-young.parallel.gc-worker-start.avg": "G1 Young GC Pause : Parallel : Avg Time spent in GC worker start (ms)",
"g1-pause-young.parallel.gc-worker-start.max": "G1 Young GC Pause : Parallel : Max Time spent in GC worker start (ms)",
"g1-pause-young.parallel.ext-root-scanning.avg": "G1 Young GC Pause: Avg Time spent in ext-root-scanning",
"g1-pause-young.parallel.ext-root-scanning.max": "G1 Young GC Pause: Max Time spent in ext-root-scanning",
"g1-pause-young.parallel.update-rs.avg": "G1 Young GC Pause: Parallel : Avg Time spent in updating Rsets",
"g1-pause-young.parallel.update-rs.max": "G1 Young GC Pause: Parallel : Max Time spent in updating Rsets",
"g1-pause-young.parallel.update-rs.processed-buffers.avg": "G1 Young GC Pause : Parallel : Update Rset : Avg number of processed buffers",
"g1-pause-young.parallel.update-rs.processed-buffers.max": "G1 Young GC Pause : Parallel : Update Rset : Max number of processed buffers",
"g1-pause-young.parallel.scan-rs.avg": "G1 Young GC Pause: Parallel : Avg Time spent in scanning Rsets",
"g1-pause-young.parallel.scan-rs.max": "G1 Young GC Pause: Parallel : Max Time spent in scannning Rsets",
"g1-pause-young.parallel.object-copy-rs.avg": "G1 Young GC Pause : Parallel : Avg Time spent in Object Copy",
"g1-pause-young.parallel.object-copy-rs.max": "G1 Young GC Pause : Parallel : Max Time spent in Object Copy",
"g1-pause-young.parallel.termination.avg": "G1 Young GC Pause : Parallel : Avg Time spent in termination",
"g1-pause-young.parallel.termination.max": "G1 Young GC Pause : Parallel : Max Time spent in termination",
"g1-pause-young.parallel.gc-worker-other.avg": "G1 Young GC Pause : Parallel : Avg Time spent in other",
"g1-pause-young.parallel.gc-worker-other.max": "G1 Young GC Pause : Parallel : Max Time spent in other",
"g1-pause-young.parallel.gc-worker-total.avg": "G1 Young GC Pause : Parallel : Avg Total time for GC worker",
"g1-pause-young.parallel.gc-worker-total.max": "G1 Young GC Pause : Parallel : Max Total time for GC worker",
"g1-pause-young.parallel.gc-worker-end.avg": "G1 Young GC Pause : Parallel : Avg Time for GC worker end",
"g1-pause-young.parallel.gc-worker-end.max": "G1 Young GC Pause : Parallel : Max Time for GC worker end",
"g1-pause-young.code-root-fixup": "G1 Young GC Pause : Time spent in code root fixup (ms)",
"g1-pause-young.clear-ct": "G1 Young GC Pause: Time spent in clear ct (ms)",
"g1-pause-young.other": "G1 Young GC Pause: Time spent in other (ms)",
"g1-pause-young.other.choose-cset": "G1 Young GC Pause : Other : Time spent in choosing CSet (ms)",
"g1-pause-young.other.ref-proc": "G1 Young GC Pause : Other : Time spent in reference processing (ms)",
"g1-pause-young.other.reg-enq": "G1 Young GC Pause : Other : Time spent in reg-enq(ms)",
"g1-pause-young.other.free-cset": "G1 Young GC Pause : Other : Time spent in processing free Cset(ms)",
"g1-pause-mixed.parallel": "G1 Mixed GC Pause: Parallel Operations (ms)",
"g1-pause-mixed.parallel.gcworkers": "G1 Mixed GC Pause: Number of Parallel GC Workers",
"g1-pause-mixed.parallel.gc-worker-start.avg": "G1 Mixed GC Pause : Parallel : Avg Time spent in GC worker start (ms)",
"g1-pause-mixed.parallel.gc-worker-start.max": "G1 Mixed GC Pause : Parallel : Max Time spent in GC worker start (ms)",
"g1-pause-mixed.parallel.ext-root-scanning.avg": "G1 Mixed GC Pause: Avg Time spent in ext-root-scanning",
"g1-pause-mixed.parallel.ext-root-scanning.max": "G1 Mixed GC Pause: Max Time spent in ext-root-scanning",
"g1-pause-mixed.parallel.update-rs.avg": "G1 Mixed GC Pause: Parallel : Avg Time spent in updating Rsets",
"g1-pause-mixed.parallel.update-rs.max": "G1 Mixed GC Pause: Parallel : Max Time spent in updating Rsets",
"g1-pause-mixed.parallel.update-rs.processed-buffers.avg": "G1 Mixed GC Pause : Parallel : Update Rset : Avg number of processed buffers",
"g1-pause-mixed.parallel.update-rs.processed-buffers.max": "G1 Mixed GC Pause : Parallel : Update Rset : Max number of processed buffers",
"g1-pause-mixed.parallel.scan-rs.avg": "G1 Mixed GC Pause: Parallel : Avg Time spent in scanning Rsets",
"g1-pause-mixed.parallel.scan-rs.max": "G1 Mixed GC Pause: Parallel : Max Time spent in scannning Rsets",
"g1-pause-mixed.parallel.object-copy-rs.avg": "G1 Mixed GC Pause : Parallel : Avg Time spent in Object Copy",
"g1-pause-mixed.parallel.object-copy-rs.max": "G1 Mixed GC Pause : Parallel : Max Time spent in Object Copy",
"g1-pause-mixed.parallel.termination.avg": "G1 Mixed GC Pause : Parallel : Avg Time spent in termination",
"g1-pause-mixed.parallel.termination.max": "G1 Mixed GC Pause : Parallel : Max Time spent in termination",
"g1-pause-mixed.parallel.gc-worker-other.avg": "G1 Mixed GC Pause : Parallel : Avg Time spent in other",
"g1-pause-mixed.parallel.gc-worker-other.max": "G1 Mixed GC Pause : Parallel : Max Time spent in other",
"g1-pause-mixed.parallel.gc-worker-total.avg": "G1 Mixed GC Pause : Parallel : Avg Total time for GC worker",
"g1-pause-mixed.parallel.gc-worker-total.max": "G1 Mixed GC Pause : Parallel : Max Total time for GC worker",
"g1-pause-mixed.parallel.gc-worker-end.avg": "G1 Mixed GC Pause : Parallel : Avg Time for GC worker end",
"g1-pause-mixed.parallel.gc-worker-end.max": "G1 Mixed GC Pause : Parallel : Max Time for GC worker end",
"g1-pause-mixed.code-root-fixup": "G1 Mixed GC Pause : Time spent in code root fixup (ms)",
"g1-pause-mixed.clear-ct": "G1 Mixed GC Pause: Time spent in clear ct (ms)",
"g1-pause-mixed.other": "G1 Mixed GC Pause: Time spent in other (ms)",
"g1-pause-mixed.other.choose-cset": "G1 Mixed GC Pause : Other : Time spent in choosing CSet (ms)",
"g1-pause-mixed.other.ref-proc": "G1 Mixed GC Pause : Other : Time spent in reference processing (ms)",
"g1-pause-mixed.other.reg-enq": "G1 Mixed GC Pause : Other : Time spent in reg-enq(ms)",
"g1-pause-mixed.other.free-cset": "G1 Mixed GC Pause : Other : Time spent in processing free Cset(ms)",
"g1-eden-occupancy-before-gc": "G1 Eden Occupancy (MB) (Before GC)",
"g1-eden-capacity-before-gc": "G1 Eden Capacity (MB) (Before GC)",
"g1-eden-occupancy-after-gc": "G1 Eden Occupancy (MB) (After GC)",
"g1-eden-capacity-after-gc": "G1 Eden Capacity (MB) (After GC)",
"g1-survivor-before-gc": "G1 Survivor Size (MB) (Before GC)",
"g1-survivor-after-gc": "G1 Survivor Size (MB) (After GC)",
"g1-heap-occupancy-before-gc": "G1 Heap Occupancy (MB) (Before GC)",
"g1-heap-capacity-before-gc": "G1 Heap Capacity (MB) (Before GC)",
"g1-heap-occupancy-after-gc": "G1 Heap Occupancy (MB) (After GC)",
"g1-heap-capacity-after-gc": "G1 Heap Capacity (MB) (After GC)",
"g1-young-cpu.sys": "G1 Young GC : sys cpu time (seconds)",
"g1-young-cpu.usr": "******",
"g1-young-cpu.real": "G1 Young GC : elapsed time (seconds)",
"g1-mixed-cpu.usr": "******",
"g1-mixed-cpu.sys": "G1 Mixed GC : sys cpu time (seconds)",
"g1-mixed-cpu.real": "G1 Mixed GC : elapsed time (seconds)",
}
def __init__(self, metric_type, infile, hostname, aggr_metrics, outdir, resource_path, label, ts_start, ts_end, rule_strings, important_sub_metrics,
anomaly_detection_metrics, **other_options):
Metric.__init__(self, metric_type, infile, hostname, aggr_metrics, outdir, resource_path, label, ts_start, ts_end, rule_strings, important_sub_metrics,
anomaly_detection_metrics)
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
def __init__ (self, metric_type, infile_list, hostname, outdir, resource_path, label, ts_start, ts_end, rule_strings,
important_sub_metrics, **other_options):
Metric.__init__(self, metric_type, infile_list, hostname, outdir, resource_path, label, ts_start, ts_end, rule_strings,
important_sub_metrics)
if not self.important_sub_metrics:
self.important_sub_metrics = important_sub_metrics_import['GC']
self.sub_metrics = self.val_types
self.beginning_ts = None
self.beginning_date = None
for (key, val) in other_options.iteritems():
if key == 'gc-options' or key == 'sub_metrics':
self.sub_metrics = val.split()
else:
setattr(self, key, val)
self.sub_metric_description = {
'gen0' : 'young gen collection time, excluding gc_prologue & gc_epilogue',
'gen0t' : 'young gen collection time, including gc_prologue & gc_epilogue',
'gen0usr' : 'young gen collection time in cpu user secs',
'gen0sys' : 'young gen collection time in cpu sys secs',
'gen0real' : 'young gen collection time in elapsed secs',
'gen1i' : 'train generation incremental collection',
'gen1t' : 'old generation collection/full GC',
'cmsIM' : 'CMS initial mark pause',
'cmsRM' : 'CMS remark pause',
'cmsRS' : 'CMS resize pause',
'GCPause' : 'all stop-the-world GC pauses',
'cmsCM' : 'CMS concurrent mark phase',
'cmsCP' : 'CMS concurrent preclean phase',
'cmsCS' : 'CMS concurrent sweep phase',
'cmsCR' : 'CMS concurrent reset phase',
'alloc' : 'object allocation in MB (approximate***)',
'promo' : 'object promotion in MB (approximate***)',
'used0' : 'young gen used memory size (before gc)',
'used1' : 'old gen used memory size (before gc)',
'used' : 'heap space used memory size (before gc) (excludes perm gen)',
'commit0' : 'young gen committed memory size (after gc)',
'commit1' : 'old gen committed memory size (after gc)',
'commit' : 'heap committed memory size (after gc) (excludes perm gen)',
'apptime' : 'amount of time application threads were running',
'safept' : 'amount of time the VM spent at safepoints (app threads stopped)',
'used0AfterGC' : 'young gen used memory size (after gc)',
'used1AfterGC' : 'old gen used memory size (after gc)',
'usedAfterGC' : 'heap space used memory size (after gc)',
'g1-pause-young' : 'G1 Young GC Pause (seconds)',
'g1-pause-mixed' : 'G1 Mixed GC Pause (seconds)',
'g1-pause-remark' : 'G1 Remark Pause (seconds)',
'g1-pause-cleanup' : 'G1 Cleanup Pause (seconds)',
'g1-pause-remark.ref-proc' : 'G1 Remark: Reference Processing (seconds)',
'g1-pause-young.parallel' : 'G1 Young GC Pause: Parallel Operations (ms)',
'g1-pause-young.parallel.gcworkers' : 'G1 Young GC Pause: Number of Parallel GC Workers',
'g1-pause-young.parallel.gc-worker-start.avg' : 'G1 Young GC Pause : Parallel : Avg Time spent in GC worker start (ms)',
'g1-pause-young.parallel.gc-worker-start.max' : 'G1 Young GC Pause : Parallel : Max Time spent in GC worker start (ms)',
'g1-pause-young.parallel.ext-root-scanning.avg' : 'G1 Young GC Pause: Avg Time spent in ext-root-scanning',
'g1-pause-young.parallel.ext-root-scanning.max' : 'G1 Young GC Pause: Max Time spent in ext-root-scanning',
'g1-pause-young.parallel.update-rs.avg' : 'G1 Young GC Pause: Parallel : Avg Time spent in updating Rsets',
'g1-pause-young.parallel.update-rs.max' : 'G1 Young GC Pause: Parallel : Max Time spent in updating Rsets',
'g1-pause-young.parallel.update-rs.processed-buffers.avg' : 'G1 Young GC Pause : Parallel : Update Rset : Avg number of processed buffers',
'g1-pause-young.parallel.update-rs.processed-buffers.max' : 'G1 Young GC Pause : Parallel : Update Rset : Max number of processed buffers',
'g1-pause-young.parallel.scan-rs.avg' : 'G1 Young GC Pause: Parallel : Avg Time spent in scanning Rsets',
'g1-pause-young.parallel.scan-rs.max' : 'G1 Young GC Pause: Parallel : Max Time spent in scannning Rsets',
'g1-pause-young.parallel.object-copy-rs.avg' : 'G1 Young GC Pause : Parallel : Avg Time spent in Object Copy',
'g1-pause-young.parallel.object-copy-rs.max' : 'G1 Young GC Pause : Parallel : Max Time spent in Object Copy',
'g1-pause-young.parallel.termination.avg' : 'G1 Young GC Pause : Parallel : Avg Time spent in termination',
'g1-pause-young.parallel.termination.max' : 'G1 Young GC Pause : Parallel : Max Time spent in termination',
'g1-pause-young.parallel.gc-worker-other.avg' : 'G1 Young GC Pause : Parallel : Avg Time spent in other',
'g1-pause-young.parallel.gc-worker-other.max' : 'G1 Young GC Pause : Parallel : Max Time spent in other',
'g1-pause-young.parallel.gc-worker-total.avg' : 'G1 Young GC Pause : Parallel : Avg Total time for GC worker',
'g1-pause-young.parallel.gc-worker-total.max' : 'G1 Young GC Pause : Parallel : Max Total time for GC worker',
'g1-pause-young.parallel.gc-worker-end.avg' : 'G1 Young GC Pause : Parallel : Avg Time for GC worker end',
'g1-pause-young.parallel.gc-worker-end.max' : 'G1 Young GC Pause : Parallel : Max Time for GC worker end',
'g1-pause-young.code-root-fixup' : 'G1 Young GC Pause : Time spent in code root fixup (ms)',
'g1-pause-young.clear-ct' : 'G1 Young GC Pause: Time spent in clear ct (ms)',
'g1-pause-young.other' : 'G1 Young GC Pause: Time spent in other (ms)',
'g1-pause-young.other.choose-cset' : 'G1 Young GC Pause : Other : Time spent in choosing CSet (ms)',
'g1-pause-young.other.ref-proc' : 'G1 Young GC Pause : Other : Time spent in reference processing (ms)',
'g1-pause-young.other.reg-enq' : 'G1 Young GC Pause : Other : Time spent in reg-enq(ms)',
'g1-pause-young.other.free-cset' : 'G1 Young GC Pause : Other : Time spent in processing free Cset(ms)',
'g1-pause-mixed.parallel' : 'G1 Mixed GC Pause: Parallel Operations (ms)',
'g1-pause-mixed.parallel.gcworkers' : 'G1 Mixed GC Pause: Number of Parallel GC Workers',
'g1-pause-mixed.parallel.gc-worker-start.avg' : 'G1 Mixed GC Pause : Parallel : Avg Time spent in GC worker start (ms)',
'g1-pause-mixed.parallel.gc-worker-start.max' : 'G1 Mixed GC Pause : Parallel : Max Time spent in GC worker start (ms)',
'g1-pause-mixed.parallel.ext-root-scanning.avg' : 'G1 Mixed GC Pause: Avg Time spent in ext-root-scanning',
'g1-pause-mixed.parallel.ext-root-scanning.max' : 'G1 Mixed GC Pause: Max Time spent in ext-root-scanning',
'g1-pause-mixed.parallel.update-rs.avg' : 'G1 Mixed GC Pause: Parallel : Avg Time spent in updating Rsets',
'g1-pause-mixed.parallel.update-rs.max' : 'G1 Mixed GC Pause: Parallel : Max Time spent in updating Rsets',
'g1-pause-mixed.parallel.update-rs.processed-buffers.avg' : 'G1 Mixed GC Pause : Parallel : Update Rset : Avg number of processed buffers',
'g1-pause-mixed.parallel.update-rs.processed-buffers.max' : 'G1 Mixed GC Pause : Parallel : Update Rset : Max number of processed buffers',
'g1-pause-mixed.parallel.scan-rs.avg' : 'G1 Mixed GC Pause: Parallel : Avg Time spent in scanning Rsets',
'g1-pause-mixed.parallel.scan-rs.max' : 'G1 Mixed GC Pause: Parallel : Max Time spent in scannning Rsets',
'g1-pause-mixed.parallel.object-copy-rs.avg' : 'G1 Mixed GC Pause : Parallel : Avg Time spent in Object Copy',
'g1-pause-mixed.parallel.object-copy-rs.max' : 'G1 Mixed GC Pause : Parallel : Max Time spent in Object Copy',
'g1-pause-mixed.parallel.termination.avg' : 'G1 Mixed GC Pause : Parallel : Avg Time spent in termination',
'g1-pause-mixed.parallel.termination.max' : 'G1 Mixed GC Pause : Parallel : Max Time spent in termination',
'g1-pause-mixed.parallel.gc-worker-other.avg' : 'G1 Mixed GC Pause : Parallel : Avg Time spent in other',
'g1-pause-mixed.parallel.gc-worker-other.max' : 'G1 Mixed GC Pause : Parallel : Max Time spent in other',
'g1-pause-mixed.parallel.gc-worker-total.avg' : 'G1 Mixed GC Pause : Parallel : Avg Total time for GC worker',
'g1-pause-mixed.parallel.gc-worker-total.max' : 'G1 Mixed GC Pause : Parallel : Max Total time for GC worker',
'g1-pause-mixed.parallel.gc-worker-end.avg' : 'G1 Mixed GC Pause : Parallel : Avg Time for GC worker end',
'g1-pause-mixed.parallel.gc-worker-end.max' : 'G1 Mixed GC Pause : Parallel : Max Time for GC worker end',
'g1-pause-mixed.code-root-fixup' : 'G1 Mixed GC Pause : Time spent in code root fixup (ms)',
'g1-pause-mixed.clear-ct' : 'G1 Mixed GC Pause: Time spent in clear ct (ms)',
'g1-pause-mixed.other' : 'G1 Mixed GC Pause: Time spent in other (ms)',
'g1-pause-mixed.other.choose-cset' : 'G1 Mixed GC Pause : Other : Time spent in choosing CSet (ms)',
'g1-pause-mixed.other.ref-proc' : 'G1 Mixed GC Pause : Other : Time spent in reference processing (ms)',
'g1-pause-mixed.other.reg-enq' : 'G1 Mixed GC Pause : Other : Time spent in reg-enq(ms)',
'g1-pause-mixed.other.free-cset' : 'G1 Mixed GC Pause : Other : Time spent in processing free Cset(ms)',
'g1-eden-occupancy-before-gc' : 'G1 Eden Occupancy (MB) (Before GC)',
'g1-eden-capacity-before-gc' : 'G1 Eden Capacity (MB) (Before GC)',
'g1-eden-occupancy-after-gc' : 'G1 Eden Occupancy (MB) (After GC)',
'g1-eden-capacity-after-gc' : 'G1 Efen Capacity (MB) (After GC)',
'g1-survivor-before-gc' : 'G1 Survivor Size (MB) (Before GC)',
'g1-survivor-after-gc' : 'G1 Survivor Size (MB) (After GC)',
'g1-heap-occupancy-before-gc' : 'G1 Heap Occupancy (MB) (Before GC)',
'g1-heap-capacity-before-gc' : 'G1 Heap Capacity (MB) (Before GC)',
'g1-heap-occupancy-after-gc' : 'G1 Heap Occupancy (MB) (After GC)',
'g1-heap-capacity-after-gc' : 'G1 Heap Capacity (MB) (After GC)',
'g1-young-cpu.sys' : 'G1 Young GC : sys cpu time (seconds)',
'g1-young-cpu.usr' : 'G1 Young GC : usr cpu time (seconds)',
'g1-young-cpu.real' : 'G1 Young GC : elapsed time (seconds)',
'g1-mixed-cpu.usr' : 'G1 Mixed GC : usr cpu time (seconds)',
'g1-mixed-cpu.sys' : 'G1 Mixed GC : sys cpu time (seconds)',
'g1-mixed-cpu.real' : 'G1 Mixed GC : elapsed time (seconds)'
}
def __init__(self, metric_type, infile, access, outdir, label, ts_start, ts_end, **other_options):
Metric.__init__(self, metric_type, infile, access, outdir, label, ts_start, ts_end)
self.options = None
self.devices = None
for (key, val) in other_options.iteritems():
setattr(self, key, val.split())
