def generate_all_configs(self):
"""
Generates configs for all components required by LAD.
Generates XML cfg file for mdsd, from JSON config settings (public & private).
Also generates rsyslog/syslog-ng configs corresponding to 'syslogEvents' or 'syslogCfg' setting.
Also generates fluentd's syslog/tail src configs and out_mdsd configs.
The rsyslog/syslog-ng and fluentd configs are not yet saved to files. They are available through
the corresponding getter methods of this class (get_fluentd_*_config(), get_*syslog*_config()).
Returns (True, '') if config was valid and proper xmlCfg.xml was generated.
Returns (False, '...') if config was invalid and the error message.
"""
# 1. Add DeploymentId (if available) to identity columns
if self._deployment_id:
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, "Management/Identity/IdentityComponent", "",
self._deployment_id, ["name", "DeploymentId"])
# 2. Use ladCfg to generate OMIQuery and LADQuery elements
lad_cfg = self._ladCfg()
if lad_cfg:
try:
self._update_metric_collection_settings(lad_cfg)
resource_id = self._ext_settings.get_resource_id()
if resource_id:
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Events/DerivedEvents/DerivedEvent/LADQuery',
'partitionKey', escape_nonalphanumerics(resource_id))
lad_query_instance_id = ""
uuid_for_instance_id = self._fetch_uuid()
if resource_id.find("providers/Microsoft.Compute/virtualMachineScaleSets") >= 0:
lad_query_instance_id = uuid_for_instance_id
self._set_xml_attr("instanceID", lad_query_instance_id, "Events/DerivedEvents/DerivedEvent/LADQuery")
# Set JsonBlob sink-related elements
self._add_obo_field(name='resourceId', value=resource_id)
self._add_obo_field(name='agentIdentityHash', value=uuid_for_instance_id)
except Exception as e:
self._logger_error("Failed to create portal config error:{0} {1}".format(e, traceback.format_exc()))
return False, 'Failed to create portal config from ladCfg (see extension error logs for more details)'
# 3. Generate config for perfCfg. Need to distinguish between non-AppInsights scenario and AppInsights scenario,
# so check if Application Insights key is present and pass it to the actual helper
# function (self._apply_perf_cfg()).
try:
self._apply_perf_cfg()
except Exception as e:
self._logger_error("Failed check for Application Insights key in LAD configuration with exception:{0}\n"
"Stacktrace: {1}".format(e, traceback.format_exc()))
return False, 'Failed to update perf counter config (see extension error logs for more details)'
# 4. Generate omsagent (fluentd) configs, rsyslog/syslog-ng config, and update corresponding mdsd config XML
try:
syslogEvents_setting = self._ext_settings.get_syslogEvents_setting()
fileLogs_setting = self._ext_settings.get_fileLogs_setting()
lad_logging_config_helper = LadLoggingConfig(syslogEvents_setting, fileLogs_setting, self._sink_configs,
self._pkey_path, self._cert_path, self._encrypt_secret)
mdsd_syslog_config = lad_logging_config_helper.get_mdsd_syslog_config()
mdsd_filelog_config = lad_logging_config_helper.get_mdsd_filelog_config()
copy_source_mdsdevent_eh_url_elems(self._mdsd_config_xml_tree, mdsd_syslog_config)
copy_source_mdsdevent_eh_url_elems(self._mdsd_config_xml_tree, mdsd_filelog_config)
self._fluentd_syslog_src_config = lad_logging_config_helper.get_fluentd_syslog_src_config()
self._fluentd_tail_src_config = lad_logging_config_helper.get_fluentd_filelog_src_config()
self._fluentd_out_mdsd_config = lad_logging_config_helper.get_fluentd_out_mdsd_config()
self._rsyslog_config = lad_logging_config_helper.get_rsyslog_config()
self._syslog_ng_config = lad_logging_config_helper.get_syslog_ng_config()
except Exception as e:
self._logger_error("Failed to create omsagent (fluentd), rsyslog/syslog-ng configs or to update "
"corresponding mdsd config XML. Error: {0}\nStacktrace: {1}"
.format(e, traceback.format_exc()))
return False, 'Failed to generate configs for fluentd, syslog, and mdsd ' \
'(see extension error logs for more details)'
# 5. Before starting to update the storage account settings, log extension's entire settings
# with secrets redacted, for diagnostic purpose.
self._ext_settings.log_ext_settings_with_secrets_redacted(self._logger_log, self._logger_error)
# 6. Actually update the storage account settings on mdsd config XML tree (based on extension's
# protectedSettings).
account = self._ext_settings.read_protected_config('storageAccountName').strip()
if not account:
return False, "Must specify storageAccountName"
if self._ext_settings.read_protected_config('storageAccountKey'):
return False, "The storageAccountKey protected setting is not supported and must not be used"
token = self._ext_settings.read_protected_config('storageAccountSasToken').strip()
if not token or token == '?':
return False, "Must specify storageAccountSasToken"
if '?' == token[0]:
token = token[1:]
endpoint = get_storage_endpoint_with_account(account,
self._ext_settings.read_protected_config('storageAccountEndPoint'))
self._update_account_settings(account, token, endpoint)
# 7. Update mdsd config XML's eventVolume attribute based on the logic specified in the helper.
self._set_event_volume(lad_cfg)
# 8. Finally generate mdsd config XML file out of the constructed XML tree object.
self._mdsd_config_xml_tree.write(os.path.join(self._ext_dir, 'xmlCfg.xml'))
return True, ""
def generate_all_configs(self):
"""
Generates configs for all components required by LAD.
Generates XML cfg file for mdsd, from JSON config settings (public & private).
Also generates rsyslog/syslog-ng configs corresponding to 'syslogEvents' or 'syslogCfg' setting.
Also generates fluentd's syslog/tail src configs and out_mdsd configs.
The rsyslog/syslog-ng and fluentd configs are not yet saved to files. They are available through
the corresponding getter methods of this class (get_fluentd_*_config(), get_*syslog*_config()).
Returns (True, '') if config was valid and proper xmlCfg.xml was generated.
Returns (False, '...') if config was invalid and the error message.
"""
# 1. Add DeploymentId (if available) to identity columns
if self._deployment_id:
XmlUtil.setXmlValue(self._mdsd_config_xml_tree,
"Management/Identity/IdentityComponent", "",
self._deployment_id, ["name", "DeploymentId"])
# 2. Generate telegraf, MetricsExtension, omsagent (fluentd) configs, rsyslog/syslog-ng config, and update corresponding mdsd config XML
try:
lad_cfg = self._ladCfg()
if not lad_cfg:
return False, 'Unable to find Ladcfg element. Failed to generate configs for fluentd, syslog, and mdsd ' \
'(see extension error logs for more details)'
syslogEvents_setting = self._ext_settings.get_syslogEvents_setting(
)
fileLogs_setting = self._ext_settings.get_fileLogs_setting()
lad_logging_config_helper = LadLoggingConfig(
syslogEvents_setting, fileLogs_setting, self._sink_configs,
self._pkey_path, self._cert_path, self._encrypt_secret)
mdsd_syslog_config = lad_logging_config_helper.get_mdsd_syslog_config(
self._ext_settings.read_protected_config(
'disableStorageAccount') == True)
mdsd_filelog_config = lad_logging_config_helper.get_mdsd_filelog_config(
)
copy_source_mdsdevent_eh_url_elems(self._mdsd_config_xml_tree,
mdsd_syslog_config)
copy_source_mdsdevent_eh_url_elems(self._mdsd_config_xml_tree,
mdsd_filelog_config)
self._fluentd_syslog_src_config = lad_logging_config_helper.get_fluentd_syslog_src_config(
)
self._fluentd_tail_src_config = lad_logging_config_helper.get_fluentd_filelog_src_config(
)
self._fluentd_out_mdsd_config = lad_logging_config_helper.get_fluentd_out_mdsd_config(
)
self._rsyslog_config = lad_logging_config_helper.get_rsyslog_config(
)
self._syslog_ng_config = lad_logging_config_helper.get_syslog_ng_config(
)
parsed_perf_settings = lad_logging_config_helper.parse_lad_perf_settings(
lad_cfg)
self._telegraf_config, self._telegraf_namespaces = telhandler.handle_config(
parsed_perf_settings, self._telegraf_me_url,
self._telegraf_mdsd_url, True)
#Handle the EH, JsonBlob and AzMonSink logic
self._update_metric_collection_settings(lad_cfg,
self._telegraf_namespaces)
mdsd_telegraf_config = lad_logging_config_helper.get_mdsd_telegraf_config(
self._telegraf_namespaces)
copy_source_mdsdevent_eh_url_elems(self._mdsd_config_xml_tree,
mdsd_telegraf_config)
resource_id = self._ext_settings.get_resource_id()
if resource_id:
# Set JsonBlob sink-related elements
uuid_for_instance_id = self._fetch_uuid()
self._add_obo_field(name='resourceId', value=resource_id)
self._add_obo_field(name='agentIdentityHash',
value=uuid_for_instance_id)
XmlUtil.setXmlValue(
self._mdsd_config_xml_tree,
'Events/DerivedEvents/DerivedEvent/LADQuery',
'partitionKey', escape_nonalphanumerics(resource_id))
lad_query_instance_id = ""
if resource_id.find(
"providers/Microsoft.Compute/virtualMachineScaleSets"
) >= 0:
lad_query_instance_id = uuid_for_instance_id
self._set_xml_attr(
"instanceID", lad_query_instance_id,
"Events/DerivedEvents/DerivedEvent/LADQuery")
else:
return False, 'Unable to find resource id in the config. Failed to generate configs for Metrics in mdsd ' \
'(see extension error logs for more details)'
#Only enable Metrics if AzMonSink is in the config
if self._enable_metrics_extension:
me_handler.setup_me(True)
except Exception as e:
self._logger_error(
"Failed to create omsagent (fluentd), rsyslog/syslog-ng configs, telegraf config or to update "
"corresponding mdsd config XML. Error: {0}\nStacktrace: {1}".
format(e, traceback.format_exc()))
return False, 'Failed to generate configs for fluentd, syslog, and mdsd; see extension.log for more details.'
# 3. Before starting to update the storage account settings, log extension's entire settings
# with secrets redacted, for diagnostic purpose.
self._ext_settings.log_ext_settings_with_secrets_redacted(
self._logger_log, self._logger_error)
# 4. Actually update the storage account settings on mdsd config XML tree (based on extension's
# protectedSettings).
account = self._ext_settings.read_protected_config(
'storageAccountName').strip()
if not account:
return False, "Configuration Error: Must specify storageAccountName in protected settings. For information on protected settings, " \
"visit https://docs.microsoft.com/en-us/azure/virtual-machines/extensions/diagnostics-linux#protected-settings."
if self._ext_settings.read_protected_config('storageAccountKey'):
return False, "Configuration Error: The storageAccountKey protected setting is deprecated in LAD 3.0 and cannot be used. " \
"Instead, use the storageAccountSasToken setting. For documentation of this setting and instructions for generating " \
"a SAS token, visit https://docs.microsoft.com/en-us/azure/virtual-machines/extensions/diagnostics-linux#protected-settings."
token = self._ext_settings.read_protected_config(
'storageAccountSasToken').strip()
if not token or token == '?':
return False, "Configuration Error: Must specify storageAccountSasToken in the protected settings. For documentation of this setting and instructions " \
"for generating a SAS token, visit https://docs.microsoft.com/en-us/azure/virtual-machines/extensions/diagnostics-linux#protected-settings."
if '?' == token[0]:
token = token[1:]
endpoints = get_storage_endpoints_with_account(
account,
self._ext_settings.read_protected_config('storageAccountEndPoint'))
self._update_account_settings(account, token, endpoints)
# 5. Update mdsd config XML's eventVolume attribute based on the logic specified in the helper.
self._set_event_volume(lad_cfg)
# 6. Finally generate mdsd config XML file out of the constructed XML tree object.
self._mdsd_config_xml_tree.write(
os.path.join(self._ext_dir, 'xmlCfg.xml'))
return True, ""
def generate_all_configs(self):
"""
Generates configs for all components required by LAD.
Generates XML cfg file for mdsd, from JSON config settings (public & private).
Also generates rsyslog/syslog-ng configs corresponding to 'syslogEvents' or 'syslogCfg' setting.
Also generates fluentd's syslog/tail src configs and out_mdsd configs.
The rsyslog/syslog-ng and fluentd configs are not yet saved to files. They are available through
the corresponding getter methods of this class (get_fluentd_*_config(), get_*syslog*_config()).
Returns (True, '') if config was valid and proper xmlCfg.xml was generated.
Returns (False, '...') if config was invalid and the error message.
"""
# 1. Add DeploymentId (if available) to identity columns
if self._deployment_id:
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, "Management/Identity/IdentityComponent", "",
self._deployment_id, ["name", "DeploymentId"])
# 2. Use ladCfg to generate OMIQuery and LADQuery elements
lad_cfg = self._ladCfg()
if lad_cfg:
try:
self._update_metric_collection_settings(lad_cfg)
resource_id = self._ext_settings.get_resource_id()
if resource_id:
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Events/DerivedEvents/DerivedEvent/LADQuery',
'partitionKey', escape_nonalphanumerics(resource_id))
lad_query_instance_id = ""
uuid_for_instance_id = self._fetch_uuid()
if resource_id.find("providers/Microsoft.Compute/virtualMachineScaleSets") >= 0:
lad_query_instance_id = uuid_for_instance_id
self._set_xml_attr("instanceID", lad_query_instance_id, "Events/DerivedEvents/DerivedEvent/LADQuery")
# Set JsonBlob sink-related elements
self._add_obo_field(name='resourceId', value=resource_id)
self._add_obo_field(name='agentIdentityHash', value=uuid_for_instance_id)
except Exception as e:
self._logger_error("Failed to create portal config error:{0} {1}".format(e, traceback.format_exc()))
return False, 'Failed to create portal config from ladCfg (see extension error logs for more details)'
# 3. Generate config for perfCfg. Need to distinguish between non-AppInsights scenario and AppInsights scenario,
# so check if Application Insights key is present and pass it to the actual helper
# function (self._apply_perf_cfg()).
try:
self._apply_perf_cfg()
except Exception as e:
self._logger_error("Failed check for Application Insights key in LAD configuration with exception:{0}\n"
"Stacktrace: {1}".format(e, traceback.format_exc()))
return False, 'Failed to update perf counter config (see extension error logs for more details)'
# 4. Generate omsagent (fluentd) configs, rsyslog/syslog-ng config, and update corresponding mdsd config XML
try:
syslogEvents_setting = self._ext_settings.get_syslogEvents_setting()
fileLogs_setting = self._ext_settings.get_fileLogs_setting()
lad_logging_config_helper = LadLoggingConfig(syslogEvents_setting, fileLogs_setting, self._sink_configs,
self._pkey_path, self._cert_path, self._encrypt_secret)
mdsd_syslog_config = lad_logging_config_helper.get_mdsd_syslog_config(self._ext_settings.read_protected_config('disableStorageAccount') == True)
mdsd_filelog_config = lad_logging_config_helper.get_mdsd_filelog_config()
copy_source_mdsdevent_eh_url_elems(self._mdsd_config_xml_tree, mdsd_syslog_config)
copy_source_mdsdevent_eh_url_elems(self._mdsd_config_xml_tree, mdsd_filelog_config)
self._fluentd_syslog_src_config = lad_logging_config_helper.get_fluentd_syslog_src_config()
self._fluentd_tail_src_config = lad_logging_config_helper.get_fluentd_filelog_src_config()
self._fluentd_out_mdsd_config = lad_logging_config_helper.get_fluentd_out_mdsd_config()
self._rsyslog_config = lad_logging_config_helper.get_rsyslog_config()
self._syslog_ng_config = lad_logging_config_helper.get_syslog_ng_config()
except Exception as e:
self._logger_error("Failed to create omsagent (fluentd), rsyslog/syslog-ng configs or to update "
"corresponding mdsd config XML. Error: {0}\nStacktrace: {1}"
.format(e, traceback.format_exc()))
return False, 'Failed to generate configs for fluentd, syslog, and mdsd ' \
'(see extension error logs for more details)'
# 5. Before starting to update the storage account settings, log extension's entire settings
# with secrets redacted, for diagnostic purpose.
self._ext_settings.log_ext_settings_with_secrets_redacted(self._logger_log, self._logger_error)
# 6. Actually update the storage account settings on mdsd config XML tree (based on extension's
# protectedSettings).
account = self._ext_settings.read_protected_config('storageAccountName').strip()
if not account:
return False, "Must specify storageAccountName"
if self._ext_settings.read_protected_config('storageAccountKey'):
return False, "The storageAccountKey protected setting is not supported and must not be used"
token = self._ext_settings.read_protected_config('storageAccountSasToken').strip()
if not token or token == '?':
return False, "Must specify storageAccountSasToken"
if '?' == token[0]:
token = token[1:]
endpoints = get_storage_endpoints_with_account(account,
self._ext_settings.read_protected_config('storageAccountEndPoint'))
self._update_account_settings(account, token, endpoints)
# 7. Update mdsd config XML's eventVolume attribute based on the logic specified in the helper.
self._set_event_volume(lad_cfg)
# 8. Finally generate mdsd config XML file out of the constructed XML tree object.
self._mdsd_config_xml_tree.write(os.path.join(self._ext_dir, 'xmlCfg.xml'))
return True, ""