def _update_metric_collection_settings(self, ladCfg):
"""
Update mdsd_config_xml_tree for Azure Portal metric collection. The mdsdCfg performanceCounters element contains
an array of metric definitions; this method passes each definition to its provider's AddMetric method, which is
responsible for configuring the provider to deliver the metric to mdsd and for updating the mdsd config as
required to expect the metric to arrive. This method also builds the necessary aggregation queries (from the
metrics.metricAggregation array) that grind the ingested data and push it to the WADmetric table.
:param ladCfg: ladCfg object from extension config
:return: None
"""
metrics = LadUtil.getPerformanceCounterCfgFromLadCfg(ladCfg)
if not metrics:
return
counter_to_table = {}
local_tables = set()
# Add each metric
for metric in metrics:
if metric['type'] == 'builtin':
local_table_name = BuiltIn.AddMetric(metric)
if local_table_name:
local_tables.add(local_table_name)
counter_to_table[
metric['counterSpecifier']] = local_table_name
# Finalize; update the mdsd config to be prepared to receive the metrics
BuiltIn.UpdateXML(self._mdsd_config_xml_tree)
# Aggregation is done by <LADQuery> within a <DerivedEvent>. If there are no alternate sinks, the DerivedQuery
# can send output directly to the WAD metrics table. If there *are* alternate sinks, have the LADQuery send
# output to a new local table, then arrange for additional derived queries to pull from that.
intervals = LadUtil.getAggregationPeriodsFromLadCfg(ladCfg)
sinks = LadUtil.getFeatureWideSinksFromLadCfg(ladCfg,
'performanceCounters')
for table_name in local_tables:
for aggregation_interval in intervals:
if sinks:
local_table_name = ProvUtil.MakeUniqueEventName(
'aggregationLocal')
self._add_derived_event(aggregation_interval,
table_name,
local_table_name,
'Local',
add_lad_query=True)
self._handle_alternate_sinks(aggregation_interval, sinks,
local_table_name)
else:
self._add_derived_event(
aggregation_interval,
table_name,
LadConfigAll._wad_table_name(aggregation_interval),
'Central',
add_lad_query=True)
def test_sample_rate(self):
try:
metric = BProvider.BuiltinMetric(self.basic_valid)
self.assertEqual(metric.sample_rate(), 30)
except Exception as ex:
self.fail(
"BuiltinMetric Constructor raised exception: {0}".format(ex))
dupe = self.basic_valid.copy()
del dupe['sampleRate']
try:
metric = BProvider.BuiltinMetric(dupe)
self.assertEqual(metric.sample_rate(), 15)
except Exception as ex:
self.fail(
"BuiltinMetric Constructor raised exception: {0}".format(ex))
def test_IsType(self):
try:
item = BProvider.BuiltinMetric(self.basic_valid)
self.assertTrue(item.is_type('builtin'))
except Exception as ex:
self.fail(
"BuiltinMetric Constructor raised exception: {0}".format(ex))
def test_two_and_two(self):
specs = [
{
"type": "builtin",
"class": "Processor",
"counter": "PercentIdleTime",
"counterSpecifier": "/builtin/Processor/PercentIdleTime",
"condition": "IsAggregate=TRUE",
"sampleRate": "PT30S",
},
{
"type": "builtin",
"class": "filesystem",
"counter": "Freespace",
"counterSpecifier": "/builtin/Filesystem/Freespace(/)",
"condition": "Name='/'",
},
{
"type": "builtin",
"class": "Processor",
"counter": "PercentProcessorTime",
"counterSpecifier": "/builtin/Processor/PercentProcessorTime",
"condition": "IsAggregate=TRUE",
"sampleRate": "PT30S",
},
{
"type": "builtin",
"class": "filesystem",
"counter": "Freespace",
"counterSpecifier": "/builtin/Filesystem/Freespace(/mnt)",
"condition": "Name=\"/mnt\"",
},
]
sink_names = set()
for spec in specs:
try:
sink = BProvider.AddMetric(spec)
self.assertIsNotNone(sink)
sink_names.add(sink)
except Exception as ex:
self.fail("AddMetric({0}) raised exception: {1}".format(
spec, ex))
self.assertEqual(len(sink_names), 3)
doc = ET.ElementTree(ET.fromstring(self.base_xml))
BProvider.UpdateXML(doc)
def test_Counter(self):
dupe = self.basic_valid.copy()
del dupe['counter']
self.assertRaises(ProvUtil.InvalidCounterSpecification,
BProvider.BuiltinMetric, dupe)
try:
metric = BProvider.BuiltinMetric(self.basic_valid)
self.assertEqual(metric.counter_name(), 'PercentIdleTime')
except Exception as ex:
self.fail(
"BuiltinMetric Constructor raised exception: {0}".format(ex))
try:
metric = BProvider.BuiltinMetric(self.mapped)
self.assertEqual(metric.counter_name(), 'FreeMegabytes')
except Exception as ex:
self.fail(
"BuiltinMetric Constructor raised exception: {0}".format(ex))
def test_Class(self):
dupe = self.basic_valid.copy()
del dupe['class']
self.assertRaises(ProvUtil.InvalidCounterSpecification,
BProvider.BuiltinMetric, dupe)
try:
metric = BProvider.BuiltinMetric(self.basic_valid)
self.assertEqual(metric.class_name(), 'processor')
except Exception as ex:
self.fail(
"BuiltinMetric Constructor raised exception: {0}".format(ex))
def test_label(self):
dupe = self.basic_valid.copy()
del dupe['counterSpecifier']
self.assertRaises(ProvUtil.InvalidCounterSpecification,
BProvider.BuiltinMetric, dupe)
try:
metric = BProvider.BuiltinMetric(self.basic_valid)
self.assertEqual(metric.label(),
'/builtin/Processor/PercentIdleTime')
except Exception as ex:
self.fail(
"BuiltinMetric Constructor raised exception: {0}".format(ex))
def test_condition(self):
dupe = self.basic_valid.copy()
del dupe['condition']
try:
metric = BProvider.BuiltinMetric(dupe)
self.assertIsNone(metric.condition())
except Exception as ex:
self.fail("BuiltinMetric Constructor (dupe) raised exception: {0}".
format(ex))
try:
metric = BProvider.BuiltinMetric(self.mapped)
self.assertEqual(metric.condition(), 'Name="/"')
except Exception as ex:
self.fail(
"BuiltinMetric Constructor (self.mapped) raised exception: {0}"
.format(ex))
try:
metric = BProvider.BuiltinMetric(self.basic_valid)
self.assertEqual(metric.condition(), 'IsAggregate=TRUE')
except Exception as ex:
self.fail(
"BuiltinMetric Constructor (self.basic_valid) raised exception: {0}"
.format(ex))
def test_lad_2_3_compatible_portal_public_settings(self):
"""
This is rather a utility function that attempts to generate a standard LAD 3.0 protected settings JSON string
for the Azure Portal charts experience. Unit, displayName, and condition are inferred/auto-filled from
a sample Azure Insights metric definitions JSON pulled from ACIS.
"""
pub_settings = {
"StorageAccount": "__DIAGNOSTIC_STORAGE_ACCOUNT__",
"ladCfg": {
"sampleRateInSeconds": 15,
"diagnosticMonitorConfiguration": {
"eventVolume": "Medium",
"metrics": {
"metricAggregation": [
{
"scheduledTransferPeriod": "PT1H"
},
{
"scheduledTransferPeriod": "PT1M"
}
],
"resourceId": "__VM_RESOURCE_ID__"
},
"performanceCounters": {
"performanceCounterConfiguration": []
},
"syslogEvents": {
"syslogEventConfiguration": {
'LOG_AUTH': 'LOG_DEBUG',
'LOG_AUTHPRIV': 'LOG_DEBUG',
'LOG_CRON': 'LOG_DEBUG',
'LOG_DAEMON': 'LOG_DEBUG',
'LOG_FTP': 'LOG_DEBUG',
'LOG_KERN': 'LOG_DEBUG',
'LOG_LOCAL0': 'LOG_DEBUG',
'LOG_LOCAL1': 'LOG_DEBUG',
'LOG_LOCAL2': 'LOG_DEBUG',
'LOG_LOCAL3': 'LOG_DEBUG',
'LOG_LOCAL4': 'LOG_DEBUG',
'LOG_LOCAL5': 'LOG_DEBUG',
'LOG_LOCAL6': 'LOG_DEBUG',
'LOG_LOCAL7': 'LOG_DEBUG',
'LOG_LPR': 'LOG_DEBUG',
'LOG_MAIL': 'LOG_DEBUG',
'LOG_NEWS': 'LOG_DEBUG',
'LOG_SYSLOG': 'LOG_DEBUG',
'LOG_USER': '******',
'LOG_UUCP': 'LOG_DEBUG'
}
}
}
}
}
each_perf_counter_cfg_template = {
"unit": "__TO_BE_FILLED__",
"type": "builtin",
"class": "__TO_BE_REPLACED_BY_CODE",
"counter": "__TO_BE_REPLACED_BY_CODE__",
"counterSpecifier": "__TO_BE_REPLACED_BY_CODE__",
"annotation": "__TO_BE_FILLED__", # Needs to be assigned a new instance to avoid shallow copy
# [
# {
# "locale": "en-us",
# "displayName": "__TO_BE_FILLED__"
# }
# ],
"condition": "__TO_BE_FILLED__"
}
perf_counter_cfg_list = pub_settings['ladCfg']['diagnosticMonitorConfiguration']['performanceCounters']['performanceCounterConfiguration']
units_and_names = self.extract_perf_counter_units_and_names_from_metrics_def_sample()
for class_name in BProvider._builtIns:
for lad_counter_name, scx_counter_name in BProvider._builtIns[class_name].iteritems():
perf_counter_cfg = dict(each_perf_counter_cfg_template)
perf_counter_cfg['class'] = class_name
perf_counter_cfg['counter'] = lad_counter_name
counter_specifier = '/builtin/{0}/{1}'.format(class_name, lad_counter_name)
perf_counter_cfg['counterSpecifier'] = counter_specifier
perf_counter_cfg['condition'] = BProvider.default_condition(class_name)
if not perf_counter_cfg['condition']:
del perf_counter_cfg['condition']
counter_specifier_with_scx_name = '/builtin/{0}/{1}'.format(class_name.title(), scx_counter_name)
if counter_specifier_with_scx_name in units_and_names:
perf_counter_cfg['unit'] = units_and_names[counter_specifier_with_scx_name]['unit']
perf_counter_cfg['annotation'] = [{
'displayName': units_and_names[counter_specifier_with_scx_name]['displayName'],
'locale': 'en-us'
}]
else:
# Use some ad hoc logic to auto-fill missing values (all from FileSystem class)
perf_counter_cfg['unit'] = self.inferred_unit_name_from_counter_name(scx_counter_name)
perf_counter_cfg['annotation'] = [{
'displayName': self.inferred_display_name_from_class_counter_names(class_name, scx_counter_name),
'locale': 'en-us'
}]
perf_counter_cfg_list.append(perf_counter_cfg)
actual = json.dumps(pub_settings, sort_keys=True, indent=2)
print actual
# Uncomment the following 2 lines when generating expected JSON file (of course after validating the actual)
#with open('lad_2_3_compatible_portal_pub_settings.json', 'w') as f:
# f.write(actual)
with open('lad_2_3_compatible_portal_pub_settings.json') as f:
expected = f.read()
self.assertEqual(json.dumps(json.loads(expected), sort_keys=True),
json.dumps(json.loads(actual), sort_keys=True))
to_be_filled = re.findall(r'"__.*?__"', actual)
self.assertEqual(2, len(to_be_filled))
self.assertIn('"__DIAGNOSTIC_STORAGE_ACCOUNT__"', to_be_filled)
self.assertIn('"__VM_RESOURCE_ID__"', to_be_filled)
def test_lad_2_3_compatible_portal_public_settings(self):
"""
This is rather a utility function that attempts to generate a standard LAD 3.0 protected settings JSON string
for the Azure Portal charts experience. Unit, displayName, and condition are inferred/auto-filled from
a sample Azure Insights metric definitions JSON pulled from ACIS.
"""
pub_settings = {
"StorageAccount": "__DIAGNOSTIC_STORAGE_ACCOUNT__",
"ladCfg": {
"sampleRateInSeconds": 15,
"diagnosticMonitorConfiguration": {
"eventVolume": "Medium",
"metrics": {
"metricAggregation": [{
"scheduledTransferPeriod": "PT1H"
}, {
"scheduledTransferPeriod": "PT1M"
}],
"resourceId":
"__VM_RESOURCE_ID__"
},
"performanceCounters": {
"performanceCounterConfiguration": []
},
"syslogEvents": {
"syslogEventConfiguration": {
'LOG_AUTH': 'LOG_DEBUG',
'LOG_AUTHPRIV': 'LOG_DEBUG',
'LOG_CRON': 'LOG_DEBUG',
'LOG_DAEMON': 'LOG_DEBUG',
'LOG_FTP': 'LOG_DEBUG',
'LOG_KERN': 'LOG_DEBUG',
'LOG_LOCAL0': 'LOG_DEBUG',
'LOG_LOCAL1': 'LOG_DEBUG',
'LOG_LOCAL2': 'LOG_DEBUG',
'LOG_LOCAL3': 'LOG_DEBUG',
'LOG_LOCAL4': 'LOG_DEBUG',
'LOG_LOCAL5': 'LOG_DEBUG',
'LOG_LOCAL6': 'LOG_DEBUG',
'LOG_LOCAL7': 'LOG_DEBUG',
'LOG_LPR': 'LOG_DEBUG',
'LOG_MAIL': 'LOG_DEBUG',
'LOG_NEWS': 'LOG_DEBUG',
'LOG_SYSLOG': 'LOG_DEBUG',
'LOG_USER': '******',
'LOG_UUCP': 'LOG_DEBUG'
}
}
}
}
}
each_perf_counter_cfg_template = {
"unit": "__TO_BE_FILLED__",
"type": "builtin",
"class": "__TO_BE_REPLACED_BY_CODE",
"counter": "__TO_BE_REPLACED_BY_CODE__",
"counterSpecifier": "__TO_BE_REPLACED_BY_CODE__",
"annotation":
"__TO_BE_FILLED__", # Needs to be assigned a new instance to avoid shallow copy
# [
# {
# "locale": "en-us",
# "displayName": "__TO_BE_FILLED__"
# }
# ],
"condition": "__TO_BE_FILLED__"
}
perf_counter_cfg_list = pub_settings['ladCfg'][
'diagnosticMonitorConfiguration']['performanceCounters'][
'performanceCounterConfiguration']
units_and_names = self.extract_perf_counter_units_and_names_from_metrics_def_sample(
)
for class_name in BProvider._builtIns:
for lad_counter_name, scx_counter_name in BProvider._builtIns[
class_name].iteritems():
perf_counter_cfg = dict(each_perf_counter_cfg_template)
perf_counter_cfg['class'] = class_name
perf_counter_cfg['counter'] = lad_counter_name
counter_specifier = '/builtin/{0}/{1}'.format(
class_name, lad_counter_name)
perf_counter_cfg['counterSpecifier'] = counter_specifier
perf_counter_cfg['condition'] = BProvider.default_condition(
class_name)
if not perf_counter_cfg['condition']:
del perf_counter_cfg['condition']
counter_specifier_with_scx_name = '/builtin/{0}/{1}'.format(
class_name.title(), scx_counter_name)
if counter_specifier_with_scx_name in units_and_names:
perf_counter_cfg['unit'] = units_and_names[
counter_specifier_with_scx_name]['unit']
perf_counter_cfg['annotation'] = [{
'displayName':
units_and_names[counter_specifier_with_scx_name]
['displayName'],
'locale':
'en-us'
}]
else:
# Use some ad hoc logic to auto-fill missing values (all from FileSystem class)
perf_counter_cfg[
'unit'] = self.inferred_unit_name_from_counter_name(
scx_counter_name)
perf_counter_cfg['annotation'] = [{
'displayName':
self.inferred_display_name_from_class_counter_names(
class_name, scx_counter_name),
'local':
'en-us'
}]
perf_counter_cfg_list.append(perf_counter_cfg)
actual = json.dumps(pub_settings, sort_keys=True, indent=2)
print actual
# Uncomment the following 2 lines when generating expected JSON file (of course after validating the actual)
#with open('lad_2_3_compatible_portal_pub_settings.json', 'w') as f:
# f.write(actual)
with open('lad_2_3_compatible_portal_pub_settings.json') as f:
expected = f.read()
self.assertEqual(json.dumps(json.loads(expected), sort_keys=True),
json.dumps(json.loads(actual), sort_keys=True))
to_be_filled = re.findall(r'"__.*?__"', actual)
self.assertEqual(2, len(to_be_filled))
self.assertIn('"__DIAGNOSTIC_STORAGE_ACCOUNT__"', to_be_filled)
self.assertIn('"__VM_RESOURCE_ID__"', to_be_filled)