def createAccountSettings(tree,account,key,endpoint,aikey=None):
XmlUtil.setXmlValue(tree,'Accounts/Account',"account",account,['isDefault','true'])
XmlUtil.setXmlValue(tree,'Accounts/Account',"key",key,['isDefault','true'])
XmlUtil.setXmlValue(tree,'Accounts/Account',"tableEndpoint",endpoint,['isDefault','true'])
if aikey:
AIUtil.createAccountElement(tree,aikey)
def _update_perf_counters_settings(self, omi_queries, for_app_insights=False):
"""
Update the mdsd XML tree with the OMI queries provided.
:param omi_queries: List of dictionaries specifying OMI queries and destination tables. E.g.:
[
{"query":"SELECT PercentAvailableMemory, AvailableMemory, UsedMemory, PercentUsedSwap FROM SCX_MemoryStatisticalInformation","table":"LinuxMemory"},
{"query":"SELECT PercentProcessorTime, PercentIOWaitTime, PercentIdleTime FROM SCX_ProcessorStatisticalInformation WHERE Name='_TOTAL'","table":"LinuxCpu"},
{"query":"SELECT AverageWriteTime,AverageReadTime,ReadBytesPerSecond,WriteBytesPerSecond FROM SCX_DiskDriveStatisticalInformation WHERE Name='_TOTAL'","table":"LinuxDisk"}
]
:param for_app_insights: Indicates whether we are updating perf counters settings for AppInsights.
AppInsights requires specific names, so we need this.
:return: None. The mdsd XML tree member is updated accordingly.
"""
assert self._mdsd_config_xml_tree is not None
if not omi_queries:
return
mdsd_omi_query_schema = """
<OMIQuery cqlQuery="" dontUsePerNDayTable="true" eventName="" omiNamespace="" priority="High" sampleRateInSeconds="" />
"""
for omi_query in omi_queries:
mdsd_omi_query_element = XmlUtil.createElement(mdsd_omi_query_schema)
mdsd_omi_query_element.set('cqlQuery', omi_query['query'])
mdsd_omi_query_element.set('eventName', omi_query['table'])
namespace = omi_query['namespace'] if 'namespace' in omi_query else 'root/scx'
mdsd_omi_query_element.set('omiNamespace', namespace)
if for_app_insights:
AIUtil.updateOMIQueryElement(mdsd_omi_query_element)
XmlUtil.addElement(xml=self._mdsd_config_xml_tree, path='Events/OMI', el=mdsd_omi_query_element, addOnlyOnce=True)
def createAccountSettings(tree, account, key, token, endpoint, aikey=None):
"""
Update the MDSD configuration Account element with Azure table storage properties.
Exactly one of (key, token) must be provided. If an aikey is passed, then add a new Account element for Application
Insights with the application insights key.
:param tree: The XML doc to be updated.
:param account: Storage account to which LAD should write data
:param key: Shared key secret for the storage account, if present
:param token: SAS token to access the storage account, if present
:param endpoint: Identifies the Azure instance (public or specific sovereign cloud) where the storage account is
:param aikey: Key for accessing AI, if present
"""
assert key or token, "Either key or token must be given."
def get_handler_cert_pkey_paths():
handler_settings = hutil.get_handler_settings()
thumbprint = handler_settings['protectedSettingsCertThumbprint']
cert_path = waagent.LibDir + '/' + thumbprint + '.crt'
pkey_path = waagent.LibDir + '/' + thumbprint + '.prv'
return cert_path, pkey_path
def encrypt_secret_with_cert(cert_path, secret):
encrypted_secret_tmp_file_path = os.path.join(WorkDir, "mdsd_secret.bin")
cmd = "echo -n '{0}' | openssl smime -encrypt -outform DER -out {1} {2}"
cmd_to_run = cmd.format(secret, encrypted_secret_tmp_file_path, cert_path)
ret_status, ret_msg = RunGetOutput(cmd_to_run, should_log=False)
if ret_status is not 0:
hutil.error("Encrypting storage secret failed with the following message: " + ret_msg)
return None
with open(encrypted_secret_tmp_file_path, 'rb') as f:
encrypted_secret = f.read()
os.remove(encrypted_secret_tmp_file_path)
return binascii.b2a_hex(encrypted_secret).upper()
handler_cert_path, handler_pkey_path = get_handler_cert_pkey_paths()
if key:
key = encrypt_secret_with_cert(handler_cert_path, key)
XmlUtil.setXmlValue(tree, 'Accounts/Account', "account", account, ['isDefault', 'true'])
XmlUtil.setXmlValue(tree, 'Accounts/Account', "key", key, ['isDefault', 'true'])
XmlUtil.setXmlValue(tree, 'Accounts/Account', "decryptKeyPath", handler_pkey_path, ['isDefault', 'true'])
XmlUtil.setXmlValue(tree, 'Accounts/Account', "tableEndpoint", endpoint, ['isDefault', 'true'])
XmlUtil.removeElement(tree, 'Accounts', 'SharedAccessSignature')
else: # token
token = encrypt_secret_with_cert(handler_cert_path, token)
XmlUtil.setXmlValue(tree, 'Accounts/SharedAccessSignature', "account", account, ['isDefault', 'true'])
XmlUtil.setXmlValue(tree, 'Accounts/SharedAccessSignature', "key", token, ['isDefault', 'true'])
XmlUtil.setXmlValue(tree, 'Accounts/SharedAccessSignature', "decryptKeyPath", handler_pkey_path,
['isDefault', 'true'])
XmlUtil.setXmlValue(tree, 'Accounts/SharedAccessSignature', "tableEndpoint", endpoint, ['isDefault', 'true'])
XmlUtil.removeElement(tree, 'Accounts', 'Account')
if aikey:
AIUtil.createAccountElement(tree, aikey)
def createPerfSettngs(tree,perfs,forAI=False):
if not perfs:
return
for perf in perfs:
perfElement = XmlUtil.createElement(perfSchema)
perfElement.set('cqlQuery',perf['query'])
perfElement.set('eventName',perf['table'])
namespace="root/scx"
if perf.has_key('namespace'):
namespace=perf['namespace']
perfElement.set('omiNamespace',namespace)
if forAI:
AIUtil.updateOMIQueryElement(perfElement)
XmlUtil.addElement(tree,'Events/OMI',perfElement,["omitag","perf"])
def createPerfSettngs(tree, perfs, forAI=False):
if not perfs:
return
for perf in perfs:
perfElement = XmlUtil.createElement(perfSchema)
perfElement.set("cqlQuery", perf["query"])
perfElement.set("eventName", perf["table"])
namespace = "root/scx"
if perf.has_key("namespace"):
namespace = perf["namespace"]
perfElement.set("omiNamespace", namespace)
if forAI:
AIUtil.updateOMIQueryElement(perfElement)
XmlUtil.addElement(tree, "Events/OMI", perfElement, ["omitag", "perf"])
def _update_account_settings(self, account, key, token, endpoint, aikey=None):
"""
Update the MDSD configuration Account element with Azure table storage properties.
Exactly one of (key, token) must be provided. If an aikey is passed, then add a new Account element for Application
Insights with the application insights key.
:param account: Storage account to which LAD should write data
:param key: Shared key secret for the storage account, if present
:param token: SAS token to access the storage account, if present
:param endpoint: Identifies the Azure instance (public or specific sovereign cloud) where the storage account is
:param aikey: Key for accessing AI, if present
"""
assert key or token, "Either key or token must be given."
assert self._mdsd_config_xml_tree is not None
handler_cert_path, handler_pkey_path = self._get_handler_cert_pkey_paths(self._ext_settings.get_handler_settings())
if key:
key = self._encrypt_secret_with_cert(handler_cert_path, key)
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/Account',
"account", account, ['isDefault', 'true'])
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/Account',
"key", key, ['isDefault', 'true'])
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/Account',
"decryptKeyPath", handler_pkey_path, ['isDefault', 'true'])
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/Account',
"tableEndpoint", endpoint, ['isDefault', 'true'])
XmlUtil.removeElement(self._mdsd_config_xml_tree, 'Accounts', 'SharedAccessSignature')
else: # token
token = self._encrypt_secret_with_cert(handler_cert_path, token)
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/SharedAccessSignature',
"account", account, ['isDefault', 'true'])
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/SharedAccessSignature',
"key", token, ['isDefault', 'true'])
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/SharedAccessSignature',
"decryptKeyPath", handler_pkey_path, ['isDefault', 'true'])
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/SharedAccessSignature',
"tableEndpoint", endpoint, ['isDefault', 'true'])
XmlUtil.removeElement(self._mdsd_config_xml_tree, 'Accounts', 'Account')
if aikey:
AIUtil.createAccountElement(self._mdsd_config_xml_tree, aikey)
try:
resourceId = getResourceId()
if resourceId:
createPortalSettings(mdsdCfg,escape(resourceId))
instanceID=""
if resourceId.find("providers/Microsoft.Compute/virtualMachineScaleSets") >=0:
instanceID = readUUID();
config(mdsdCfg,"instanceID",instanceID,"Events/DerivedEvents/DerivedEvent/LADQuery")
except Exception, e:
hutil.error("Failed to create portal config error:{0} {1}".format(e,traceback.format_exc()))
# Check if Application Insights key is present in ladCfg
ladCfg = readPublicConfig('ladCfg')
try:
aikey = AIUtil.tryGetAiKey(ladCfg)
if aikey:
hutil.log("Application Insights key found.")
else:
hutil.log("Application Insights key not found.")
except Exception, e:
hutil.error("Failed check for Application Insights key in LAD configuration with exception:{0} {1}".format(e,traceback.format_exc()))
generatePerformanceCounterConfiguration(mdsdCfg,aikey != None)
syslogCfg = getSyslogCfg()
fileCfg = getFileCfg()
#fileCfg = [{"file":"/var/log/waagent.log","table":"waagent"},{"file":"/var/log/waagent2.log","table":"waagent3"}]
try:
if fileCfg:
syslogCfg = createEventFileSettings(mdsdCfg,fileCfg)+syslogCfg
def configSettings():
'''
Generates XML cfg file for mdsd, from JSON config settings (public & private).
Returns (True, '') if config was valid and proper xmlCfg.xml was generated.
Returns (False, '...') if config was invalid and the error message.
'''
mdsdCfgstr = readPublicConfig('mdsdCfg')
if not mdsdCfgstr:
with open(os.path.join(WorkDir, './mdsdConfig.xml.template'),"r") as defaulCfg:
mdsdCfgstr = defaulCfg.read()
else:
mdsdCfgstr = base64.b64decode(mdsdCfgstr)
mdsdCfg = ET.ElementTree()
mdsdCfg._setroot(XmlUtil.createElement(mdsdCfgstr))
# update deployment id
deployment_id = get_deployment_id()
XmlUtil.setXmlValue(mdsdCfg, "Management/Identity/IdentityComponent", "", deployment_id, ["name", "DeploymentId"])
try:
resourceId = getResourceId()
if resourceId:
createPortalSettings(mdsdCfg,escape(resourceId))
instanceID=""
if resourceId.find("providers/Microsoft.Compute/virtualMachineScaleSets") >=0:
instanceID = readUUID()
config(mdsdCfg,"instanceID",instanceID,"Events/DerivedEvents/DerivedEvent/LADQuery")
except Exception as e:
hutil.error("Failed to create portal config error:{0} {1}".format(e,traceback.format_exc()))
# Check if Application Insights key is present in ladCfg
ladCfg = readPublicConfig('ladCfg')
try:
aikey = AIUtil.tryGetAiKey(ladCfg)
if aikey:
hutil.log("Application Insights key found.")
else:
hutil.log("Application Insights key not found.")
except Exception as e:
hutil.error("Failed check for Application Insights key in LAD configuration with exception:{0} {1}".format(e,traceback.format_exc()))
generatePerformanceCounterConfiguration(mdsdCfg,aikey != None)
syslogCfg = getSyslogCfg()
fileCfg = getFileCfg()
#fileCfg = [{"file":"/var/log/waagent.log","table":"waagent"},{"file":"/var/log/waagent2.log","table":"waagent3"}]
try:
if fileCfg:
syslogCfg = createEventFileSettings(mdsdCfg,fileCfg)+syslogCfg
with open(omfileconfig,'w') as hfile:
hfile.write(syslogCfg)
except Exception as e:
hutil.error("Failed to create syslog_file config error:{0} {1}".format(e,traceback.format_exc()))
account = readPrivateConfig('storageAccountName')
if not account:
return False, "Empty storageAccountName"
key = readPrivateConfig('storageAccountKey')
if not key:
return False, "Empty storageAccountKey"
endpoint = readPrivateConfig('endpoint')
if not endpoint:
endpoint = 'table.core.windows.net'
endpoint = 'https://'+account+"."+endpoint
createAccountSettings(mdsdCfg,account,key,endpoint,aikey)
# Check and add new syslog RouteEvent for Application Insights.
if aikey:
AIUtil.createSyslogRouteEventElement(mdsdCfg)
setEventVolume(mdsdCfg,ladCfg)
config(mdsdCfg,"sampleRateInSeconds","60","Events/OMI/OMIQuery")
mdsdCfg.write(os.path.join(WorkDir, './xmlCfg.xml'))
return True, ""
def generate_mdsd_rsyslog_configs(self):
"""
Generates XML cfg file for mdsd, from JSON config settings (public & private).
Also generates rsyslog imfile conf file from the 'fileCfg' setting.
Returns (True, '') if config was valid and proper xmlCfg.xml was generated.
Returns (False, '...') if config was invalid and the error message.
"""
# 1. Get the mdsd config XML tree base.
# - 1st priority is from the extension setting's 'mdsdCfg' value.
# Note that we have never used this option.
# - 2nd priority is to use the provided XML template stored in <ext_dir>/mdsdConfig.xml.template.
mdsd_cfg_str = self._ext_settings.get_mdsd_cfg()
if not mdsd_cfg_str:
with open(os.path.join(self._ext_dir, './mdsdConfig.xml.template'), "r") as f:
mdsd_cfg_str = f.read()
self._mdsd_config_xml_tree = ET.ElementTree()
self._mdsd_config_xml_tree._setroot(XmlUtil.createElement(mdsd_cfg_str))
# 2. Add DeploymentId (if available) to identity columns
if self._deployment_id:
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, "Management/Identity/IdentityComponent", "",
self._deployment_id, ["name", "DeploymentId"])
# 2.1. Apply resourceId attribute to LADQuery elements
try:
resource_id = self._ext_settings.get_resource_id()
if resource_id:
escaped_resource_id_str = escape_nonalphanumerics(resource_id)
self._add_portal_settings(escaped_resource_id_str)
instanceID = ""
if resource_id.find("providers/Microsoft.Compute/virtualMachineScaleSets") >= 0:
instanceID = read_uuid(self._run_command)
self._set_xml_attr("instanceID", instanceID, "Events/DerivedEvents/DerivedEvent/LADQuery")
except Exception as e:
self._logger_error("Failed to create portal config error:{0} {1}".format(e, traceback.format_exc()))
# 3. Update perf counter config. Need to distinguish between non-AppInsights scenario and AppInsights scenario,
# so check if Application Insights key is present in ladCfg first, and pass it to the actual helper
# function (self._apply_perf_cfgs()).
lad_cfg = self._ext_settings.read_public_config('ladCfg')
do_ai = False
aikey = None
try:
aikey = AIUtil.tryGetAiKey(lad_cfg)
if aikey:
self._logger_log("Application Insights key found.")
do_ai = True
else:
self._logger_log("Application Insights key not found.")
except Exception as e:
self._logger_error("Failed check for Application Insights key in LAD configuration with exception:{0}\n"
"Stacktrace: {1}".format(e, traceback.format_exc()))
self._apply_perf_cfgs(do_ai)
# 4. Generate rsyslog imfile config. It's unclear why non-imfile config stuff (self._get_syslog_config())
# is appended to imfileconfig as well. That part has never been used, as far as I remember, and will
# definitely need to change later.
syslog_cfg = self._ext_settings.get_syslog_config()
file_cfg = self._ext_settings.get_file_monitoring_config()
# fileCfg = [{"file":"/var/log/waagent.log","table":"waagent"},{"file":"/var/log/waagent2.log","table":"waagent3"}]
try:
if file_cfg:
syslog_cfg = self._update_and_get_file_monitoring_settings(file_cfg) + syslog_cfg
with open(self._imfile_config_filename, 'w') as imfile_config_file:
imfile_config_file.write(syslog_cfg)
except Exception as e:
self._logger_error("Failed to create rsyslog imfile config. Error:{0}\n"
"Stacktrace: {1}".format(e, traceback.format_exc()))
# 5. Before starting to update the storage account settings, log extension's protected settings'
# keys only (except well-known values), for diagnostic purpose. This is mainly to make sure that
# the extension's Json settings include a correctly entered 'storageEndpoint'.
self._ext_settings.log_protected_settings_keys(self._logger_log, self._logger_error)
# 6. Actually update the storage account settings on mdsd config XML tree (based on extension's
# protectedSettings).
account = self._ext_settings.read_protected_config('storageAccountName')
if not account:
return False, "Empty storageAccountName"
key = self._ext_settings.read_protected_config('storageAccountKey')
token = self._ext_settings.read_protected_config('storageAccountSasToken')
if not key and not token:
return False, "Neither storageAccountKey nor storageAccountSasToken is given"
if key and token:
return False, "Either storageAccountKey or storageAccountSasToken (but not both) should be given"
endpoint = get_storage_endpoint_with_account(account,
self._ext_settings.read_protected_config('storageAccountEndPoint'))
self._update_account_settings(account, key, token, endpoint, aikey)
# 7. Check and add new syslog RouteEvent for Application Insights.
if aikey:
AIUtil.createSyslogRouteEventElement(self._mdsd_config_xml_tree)
# 8. Update mdsd config XML's eventVolume attribute based on the logic specified in the helper.
self._set_event_volume(lad_cfg)
# 9. Update mdsd config XML's sampleRateInSeconds attribute with default '60'
self._set_xml_attr("sampleRateInSeconds", "60", "Events/OMI/OMIQuery")
# 10. Finally generate mdsd config XML file out of the constructed XML tree object.
self._mdsd_config_xml_tree.write(os.path.join(self._ext_dir, './xmlCfg.xml'))
return True, ""
def configSettings():
"""
Generates XML cfg file for mdsd, from JSON config settings (public & private).
Returns (True, '') if config was valid and proper xmlCfg.xml was generated.
Returns (False, '...') if config was invalid and the error message.
"""
mdsdCfgstr = readPublicConfig('mdsdCfg')
if not mdsdCfgstr:
with open(os.path.join(WorkDir, './mdsdConfig.xml.template'), "r") as defaulCfg:
mdsdCfgstr = defaulCfg.read()
else:
mdsdCfgstr = base64.b64decode(mdsdCfgstr)
mdsdCfg = ET.ElementTree()
mdsdCfg._setroot(XmlUtil.createElement(mdsdCfgstr))
# Add DeploymentId (if available) to identity columns
deployment_id = get_deployment_id()
if deployment_id:
XmlUtil.setXmlValue(mdsdCfg, "Management/Identity/IdentityComponent", "", deployment_id,
["name", "DeploymentId"])
try:
resourceId = getResourceId()
if resourceId:
createPortalSettings(mdsdCfg, escape_nonalphanumerics(resourceId))
instanceID = ""
if resourceId.find("providers/Microsoft.Compute/virtualMachineScaleSets") >= 0:
instanceID = read_uuid(waagent.RunGetOutput)
config(mdsdCfg, "instanceID", instanceID, "Events/DerivedEvents/DerivedEvent/LADQuery")
except Exception as e:
hutil.error("Failed to create portal config error:{0} {1}".format(e, traceback.format_exc()))
# Check if Application Insights key is present in ladCfg
ladCfg = readPublicConfig('ladCfg')
do_ai = False
aikey = None
try:
aikey = AIUtil.tryGetAiKey(ladCfg)
if aikey:
hutil.log("Application Insights key found.")
do_ai = True
else:
hutil.log("Application Insights key not found.")
except Exception as e:
msg = "Failed check for Application Insights key in LAD configuration with exception:{0} {1}"
hutil.error(msg.format(e, traceback.format_exc()))
generatePerformanceCounterConfiguration(mdsdCfg, do_ai)
syslogCfg = getSyslogCfg()
fileCfg = getFileCfg()
# fileCfg = [{"file":"/var/log/waagent.log","table":"waagent"},{"file":"/var/log/waagent2.log","table":"waagent3"}]
try:
if fileCfg:
syslogCfg = createEventFileSettings(mdsdCfg, fileCfg) + syslogCfg
with open(omfileconfig, 'w') as hfile:
hfile.write(syslogCfg)
except Exception as e:
hutil.error("Failed to create syslog_file config error:{0} {1}".format(e, traceback.format_exc()))
log_private_settings_keys(private_settings, hutil.log, hutil.error)
account = readPrivateConfig('storageAccountName')
if not account:
return False, "Empty storageAccountName"
key = readPrivateConfig('storageAccountKey')
token = readPrivateConfig('storageAccountSasToken')
if not key and not token:
return False, "Neither storageAccountKey nor storageAccountSasToken is given"
if key and token:
return False, "Either storageAccountKey or storageAccountSasToken (but not both) should be given"
endpoint = get_storage_endpoint_with_account(account, readPrivateConfig('storageAccountEndPoint'))
createAccountSettings(mdsdCfg, account, key, token, endpoint, aikey)
# Check and add new syslog RouteEvent for Application Insights.
if aikey:
AIUtil.createSyslogRouteEventElement(mdsdCfg)
setEventVolume(mdsdCfg, ladCfg)
config(mdsdCfg, "sampleRateInSeconds", "60", "Events/OMI/OMIQuery")
mdsdCfg.write(os.path.join(WorkDir, './xmlCfg.xml'))
return True, ""