def createListLayoutCode(self, listMetadata, index, listView):
baseListItemMetadata = listMetadata.mListItemMetadatas[0]
# document = XmlUtil.createDocument()
rootView = RectView(baseListItemMetadata.bound, None)
for rectView in baseListItemMetadata.baseViews:
rootView.addChild(rectView)
for rectView in baseListItemMetadata.additionalViews:
rootView.addChild(rectView)
rootView.mChildren.sort(
key=cmp_to_key(RectUtil.getTopBottomComparator()))
rootView.mChildren.sort(
key=cmp_to_key(RectUtil.getLeftRightComparator()))
rootElement = XmlUtil.createRoot(self.mDipCalculator,
FRAMELAYOUT_ELEMENT, rootView,
self.mColorWriter)
_map = {}
self.addChildrenLayout(rootElement, rootView, listView.x, listView.y,
_map)
rectViews = RectUtil.toRects(rootView)
rectViews.remove(rootView)
XmlUtil.writeDocument(
rootView, self.mOutProjectFolder + Constants.DEFAULT_LAYOUT_PATH +
"/" + Constants.DEFAULT_LAYOUT_LIST_PREFIX + index + ".xml")
def configSettings():
mdsdCfgstr = readPublicConfig('mdsdCfg')
if not mdsdCfgstr :
with open (os.path.join(WorkDir, './mdsdConfig.xml.template'),"r") as defaulCfg:
mdsdCfgstr = defaulCfg.read()
else:
mdsdCfgstr = base64.b64decode(mdsdCfgstr)
mdsdCfg = ET.ElementTree()
mdsdCfg._setroot(XmlUtil.createElement(mdsdCfgstr))
# update deployment id
deployment_id = get_deployment_id()
XmlUtil.setXmlValue(mdsdCfg, "Management/Identity/IdentityComponent", "", deployment_id, ["name", "DeploymentId"])
try:
resourceId = getResourceId()
if resourceId:
createPortalSettings(mdsdCfg,escape(resourceId))
instanceID=""
if resourceId.find("providers/Microsoft.Compute/virtualMachineScaleSets") >=0:
instanceID = readUUID();
config(mdsdCfg,"instanceID",instanceID,"Events/DerivedEvents/DerivedEvent/LADQuery")
except Exception, e:
hutil.error("Failed to create portal config error:{0} {1}".format(e,traceback.format_exc()))
def _set_event_volume(self, lad_cfg):
"""
Set event volume in mdsd config. Check if desired event volume is specified,
first in ladCfg then in public config. If in neither then default to Medium.
:param lad_cfg: 'ladCfg' Json object to look up for the event volume setting.
:return: None. The mdsd config XML tree's eventVolume attribute is directly updated.
:rtype: str
"""
assert self._mdsd_config_xml_tree is not None
event_volume = LadUtil.getEventVolumeFromLadCfg(lad_cfg)
if event_volume:
self._logger_log("Event volume found in ladCfg: " + event_volume)
else:
event_volume = self._ext_settings.read_public_config("eventVolume")
if event_volume:
self._logger_log("Event volume found in public config: " +
event_volume)
else:
event_volume = "Medium"
self._logger_log(
"Event volume not found in config. Using default value: " +
event_volume)
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, "Management",
"eventVolume", event_volume)
def _update_account_settings(self, account, token, endpoint):
"""
Update the MDSD configuration Account element with Azure table storage properties.
Exactly one of (key, token) must be provided.
:param account: Storage account to which LAD should write data
:param token: SAS token to access the storage account
:param endpoint: Identifies the Azure instance (public or specific sovereign cloud) where the storage account is
"""
assert token, "Token must be given."
assert self._mdsd_config_xml_tree is not None
token = self._encrypt_secret_with_cert(token)
assert token, "Could not encrypt token"
XmlUtil.setXmlValue(self._mdsd_config_xml_tree,
'Accounts/SharedAccessSignature', "account",
account, ['isDefault', 'true'])
XmlUtil.setXmlValue(self._mdsd_config_xml_tree,
'Accounts/SharedAccessSignature', "key", token,
['isDefault', 'true'])
XmlUtil.setXmlValue(self._mdsd_config_xml_tree,
'Accounts/SharedAccessSignature', "decryptKeyPath",
self._pkey_path, ['isDefault', 'true'])
XmlUtil.setXmlValue(self._mdsd_config_xml_tree,
'Accounts/SharedAccessSignature', "tableEndpoint",
endpoint, ['isDefault', 'true'])
XmlUtil.removeElement(self._mdsd_config_xml_tree, 'Accounts',
'Account')
def _update_perf_counters_settings(self, omi_queries, for_app_insights=False):
"""
Update the mdsd XML tree with the OMI queries provided.
:param omi_queries: List of dictionaries specifying OMI queries and destination tables. E.g.:
[
{"query":"SELECT PercentAvailableMemory, AvailableMemory, UsedMemory, PercentUsedSwap FROM SCX_MemoryStatisticalInformation","table":"LinuxMemory"},
{"query":"SELECT PercentProcessorTime, PercentIOWaitTime, PercentIdleTime FROM SCX_ProcessorStatisticalInformation WHERE Name='_TOTAL'","table":"LinuxCpu"},
{"query":"SELECT AverageWriteTime,AverageReadTime,ReadBytesPerSecond,WriteBytesPerSecond FROM SCX_DiskDriveStatisticalInformation WHERE Name='_TOTAL'","table":"LinuxDisk"}
]
:param for_app_insights: Indicates whether we are updating perf counters settings for AppInsights.
AppInsights requires specific names, so we need this.
:return: None. The mdsd XML tree member is updated accordingly.
"""
assert self._mdsd_config_xml_tree is not None
if not omi_queries:
return
mdsd_omi_query_schema = """
<OMIQuery cqlQuery="" dontUsePerNDayTable="true" eventName="" omiNamespace="" priority="High" sampleRateInSeconds="" />
"""
for omi_query in omi_queries:
mdsd_omi_query_element = XmlUtil.createElement(mdsd_omi_query_schema)
mdsd_omi_query_element.set('cqlQuery', omi_query['query'])
mdsd_omi_query_element.set('eventName', omi_query['table'])
namespace = omi_query['namespace'] if 'namespace' in omi_query else 'root/scx'
mdsd_omi_query_element.set('omiNamespace', namespace)
if for_app_insights:
AIUtil.updateOMIQueryElement(mdsd_omi_query_element)
XmlUtil.addElement(xml=self._mdsd_config_xml_tree, path='Events/OMI', el=mdsd_omi_query_element, addOnlyOnce=True)
def _add_element_from_element(self, path, xml_elem, add_only_once=True):
"""
Add an XML fragment to the mdsd config document in accordance with path
:param str path: Where to add the fragment
:param ElementTree xml_elem: An ElementTree object XML fragment that should be added to the path.
:param bool add_only_once: Indicates whether to perform the addition only to the first match of the path.
"""
XmlUtil.addElement(xml=self._mdsd_config_xml_tree, path=path, el=xml_elem, addOnlyOnce=add_only_once)
def _add_element_from_element(self, path, xml_elem, add_only_once=True):
"""
Add an XML fragment to the mdsd config document in accordance with path
:param str path: Where to add the fragment
:param ElementTree xml_elem: An ElementTree object XML fragment that should be added to the path.
:param bool add_only_once: Indicates whether to perform the addition only to the first match of the path.
"""
XmlUtil.addElement(xml=self._mdsd_config_xml_tree, path=path, el=xml_elem, addOnlyOnce=add_only_once)
def _add_element_from_string(self, path, xml_string, add_only_once=True):
"""
Add an XML fragment to the mdsd config document in accordance with path
:param str path: Where to add the fragment
:param str xml_string: A string containing the XML element to add
:param bool add_only_once: Indicates whether to perform the addition only to the first match of the path.
"""
XmlUtil.addElement(xml=self._mdsd_config_xml_tree, path=path, el=ET.fromstring(xml_string),
addOnlyOnce=add_only_once)
def _add_element_from_string(self, path, xml_string, add_only_once=True):
"""
Add an XML fragment to the mdsd config document in accordance with path
:param str path: Where to add the fragment
:param str xml_string: A string containing the XML element to add
:param bool add_only_once: Indicates whether to perform the addition only to the first match of the path.
"""
XmlUtil.addElement(xml=self._mdsd_config_xml_tree, path=path, el=ET.fromstring(xml_string),
addOnlyOnce=add_only_once)
def getResourceId():
ladCfg = readPublicConfig('ladCfg')
resourceId = LadUtil.getResourceIdFromLadCfg(ladCfg)
if not resourceId:
encodedXmlCfg = readPublicConfig('xmlCfg').strip()
if encodedXmlCfg:
xmlCfg = base64.b64decode(encodedXmlCfg)
resourceId = XmlUtil.getXmlValue(XmlUtil.createElement(xmlCfg),'diagnosticMonitorConfiguration/metrics','resourceId')
# Azure portal uses xmlCfg which contains WadCfg which is pascal case, Currently we will support both casing and deprecate one later
if not resourceId:
resourceId = XmlUtil.getXmlValue(XmlUtil.createElement(xmlCfg),'DiagnosticMonitorConfiguration/Metrics','resourceId')
return resourceId
def anotateIntenal(self, root, elementDataMap):
top = int(
XmlUtil.getDipValue(root, Constants.ATTRIBUTE_LAYOUT_MARGIN_TOP))
left = int(
XmlUtil.getDipValue(root, Constants.ATTRIBUTE_LAYOUT_MARGIN_LEFT))
width = int(XmlUtil.getDipValue(root,
Constants.ATTRIBUTE_LAYOUT_WIDTH))
height = int(
XmlUtil.getDipValue(root, Constants.ATTRIBUTE_LAYOUT_HEIGHT))
data = ElementData(top, left, width, height)
elementDataMap[root] = data
def setEventVolume(mdsdCfg,ladCfg):
eventVolume = LadUtil.getEventVolumeFromLadCfg(ladCfg)
if eventVolume:
hutil.log("Event volume found in ladCfg: " + eventVolume)
else:
eventVolume = readPublicConfig("eventVolume")
if eventVolume:
hutil.log("Event volume found in public config: " + eventVolume)
else:
eventVolume = "Medium"
hutil.log("Event volume not found in config. Using default value: " + eventVolume)
XmlUtil.setXmlValue(mdsdCfg,"Management","eventVolume",eventVolume)
def createPerfSettngs(tree, perfs, forAI=False):
if not perfs:
return
for perf in perfs:
perfElement = XmlUtil.createElement(perfSchema)
perfElement.set("cqlQuery", perf["query"])
perfElement.set("eventName", perf["table"])
namespace = "root/scx"
if perf.has_key("namespace"):
namespace = perf["namespace"]
perfElement.set("omiNamespace", namespace)
if forAI:
AIUtil.updateOMIQueryElement(perfElement)
XmlUtil.addElement(tree, "Events/OMI", perfElement, ["omitag", "perf"])
def createEventFileSettings(tree, files):
fileid = 0
sysconfig = """
$ModLoad imfile
"""
if not files:
return
for file in files:
fileid = fileid + 1
eventSourceElement = XmlUtil.createElement(eventSourceSchema)
XmlUtil.setXmlValue(eventSourceElement, 'RouteEvent', 'eventName',
file["table"])
eventSourceElement.set('source', 'ladfile' + str(fileid))
XmlUtil.addElement(tree, 'Events/MdsdEvents', eventSourceElement)
sourceElement = XmlUtil.createElement(sourceSchema)
sourceElement.set('name', 'ladfile' + str(fileid))
XmlUtil.addElement(tree, 'Sources', sourceElement)
syslog_config = syslogEventSourceConfig.replace('#FILE#', file['file'])
syslog_config = syslog_config.replace('#STATFILE#',
file['file'].replace("/", "-"))
syslog_config = syslog_config.replace('#FILETAG#',
'ladfile' + str(fileid))
sysconfig += syslog_config
return sysconfig
def _add_derived_event(self, interval, source, event_name, store_type, add_lad_query=False):
"""
Add a <DerivedEvent> element to the configuration
:param str interval: Interval at which this DerivedEvent should be run
:param str source: Local table from which this DerivedEvent should pull
:param str event_name: Destination table to which this DerivedEvent should push
:param str store_type: The storage type of the destination table, e.g. Local, Central, JsonBlob
:param bool add_lad_query: True if a <LadQuery> subelement should be added to this <DerivedEvent> element
"""
derived_event = mxt.derived_event.format(interval=interval, source=source, target=event_name, type=store_type)
element = ET.fromstring(derived_event)
if add_lad_query:
XmlUtil.addElement(element, ".", ET.fromstring(mxt.lad_query))
self._add_element_from_element('Events/DerivedEvents', element)
def _add_derived_event(self, interval, source, event_name, store_type, add_lad_query=False):
"""
Add a <DerivedEvent> element to the configuration
:param str interval: Interval at which this DerivedEvent should be run
:param str source: Local table from which this DerivedEvent should pull
:param str event_name: Destination table to which this DerivedEvent should push
:param str store_type: The storage type of the destination table, e.g. Local, Central, JsonBlob
:param bool add_lad_query: True if a <LadQuery> subelement should be added to this <DerivedEvent> element
"""
derived_event = mxt.derived_event.format(interval=interval, source=source, target=event_name, type=store_type)
element = ET.fromstring(derived_event)
if add_lad_query:
XmlUtil.addElement(element, ".", ET.fromstring(mxt.lad_query))
self._add_element_from_element('Events/DerivedEvents', element)
def createPerfSettngs(tree,perfs,forAI=False):
if not perfs:
return
for perf in perfs:
perfElement = XmlUtil.createElement(perfSchema)
perfElement.set('cqlQuery',perf['query'])
perfElement.set('eventName',perf['table'])
namespace="root/scx"
if perf.has_key('namespace'):
namespace=perf['namespace']
perfElement.set('omiNamespace',namespace)
if forAI:
AIUtil.updateOMIQueryElement(perfElement)
XmlUtil.addElement(tree,'Events/OMI',perfElement,["omitag","perf"])
def _set_xml_attr(self, key, value, xml_path, selector=[]):
"""
Set XML attribute on the element specified with xml_path.
:param key: The attribute name to set on the XML element.
:param value: The default value to be set, if there's no public config for that attribute.
:param xml_path: The path of the XML element(s) to which the attribute is applied.
:param selector: Selector for finding the actual XML element (see XmlUtil.setXmlValue)
:return: None. Change is directly applied to mdsd_config_xml_tree XML member object.
"""
assert self._mdsd_config_xml_tree is not None
v = self._ext_settings.read_public_config(key)
if not v:
v = value
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, xml_path, key, v, selector)
def _set_xml_attr(self, key, value, xml_path, selector=[]):
"""
Set XML attribute on the element specified with xml_path.
:param key: The attribute name to set on the XML element.
:param value: The default value to be set, if there's no public config for that attribute.
:param xml_path: The path of the XML element(s) to which the attribute is applied.
:param selector: Selector for finding the actual XML element (see XmlUtil.setXmlValue)
:return: None. Change is directly applied to mdsd_config_xml_tree XML member object.
"""
assert self._mdsd_config_xml_tree is not None
v = self._ext_settings.read_public_config(key)
if not v:
v = value
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, xml_path, key, v, selector)
def generate_omi_query_xml_elem(omi_query, sink=()):
"""
Helper for generating OMI event XML element
:param omi_query: Python dictionary object for the raw OMI query specified as a LAD 3.0 perfCfg array item
:param sink: (name, type) tuple for this OMI query. Not specified implies default XTable sink
:return: An XML element object for this OMI event that should be added to the mdsd XML cfg tree
"""
omi_xml_schema = """
<OMIQuery cqlQuery="" dontUsePerNDayTable="true" eventName="" omiNamespace="" priority="High" sampleRateInSeconds="" storeType="" />
""" if sink else """
<OMIQuery cqlQuery="" dontUsePerNDayTable="true" eventName="" omiNamespace="" priority="High" sampleRateInSeconds="" />
"""
xml_elem = XmlUtil.createElement(omi_xml_schema)
xml_elem.set('cqlQuery', omi_query['query'])
xml_elem.set('eventName', sink[0] if sink else omi_query['table'])
# Default OMI namespace is 'root/scx'
xml_elem.set(
'omiNamespace', omi_query['namespace']
if 'namespace' in omi_query else 'root/scx')
# Default query frequency is 300 seconds
xml_elem.set(
'sampleRateInSeconds',
str(omi_query['frequency'])
if 'frequency' in omi_query else '300')
if sink:
xml_elem.set('storeType',
'local' if sink[1] == 'EventHub' else sink[1])
return xml_elem
def createDocument(self):
# self.writeHierarchyLog("Un Categorized")
self.pruneBasic(self.mRootView)
# self.writeHierarchyLog("Categorized")
# self.pruneToCreateGroupText(self.mRootView)
# self.pruneToCreateListItemsHasDrawable(self.mRootView, self.mDrawableMap.values(), 0)
# self.pruneToCreateListItemsHasViewsSameSize(self.mRootView, self.mDrawableMap.values(), self.mDefaultAlignThreshold)
for i in range(len(self.mListViews)):
listMetadata = self.mListViews[i]
self.updateListContent(listMetadata, i)
_map = {}
# self.updateColorBackground(self.mRootView)
rootElement = XmlUtil.createRoot(self.mDipCalculator,
LayoutHelper.FRAMELAYOUT_ELEMENT,
self.mRootView, self.mColorWriter)
self.addChildrenLayout(rootElement, self.mRootView, 0, 0, _map)
return rootElement
def _update_and_get_file_monitoring_settings(self, files):
"""
Update mdsd config's file monitoring config. Also creates/returns rsyslog imfile config.
All the operations are based on the input param files, which is a Json-deserialized dictionary
corresponding the following Json array example:
[
{"file":"/var/log/a.log", "table":"aLog"},
{"file":"/var/log/b.log", "table":"bLog"}
]
:param files: Array of dictionaries deserialized from the 'fileCfg' Json config (example as above)
:return: rsyslog omfile module config file content
"""
assert self._mdsd_config_xml_tree is not None
if not files:
return ''
file_id = 0
imfile_config = """
$ModLoad imfile
"""
mdsd_event_source_schema = """
<MdsdEventSource source="ladfile">
<RouteEvent dontUsePerNDayTable="true" eventName="" priority="High"/>
</MdsdEventSource>
"""
mdsd_source_schema = """
<Source name="ladfile1" schema="ladfile" />
"""
imfile_per_file_config_template = """
$InputFileName #FILE#
$InputFileTag #FILETAG#
$InputFileFacility local6
$InputFileStateFile syslog-stat#STATFILE#
$InputFileSeverity debug
$InputRunFileMonitor
"""
for item in files:
file_id += 1
mdsd_event_source_element = XmlUtil.createElement(mdsd_event_source_schema)
XmlUtil.setXmlValue(mdsd_event_source_element, 'RouteEvent', 'eventName', item["table"])
mdsd_event_source_element.set('source', 'ladfile'+str(file_id))
XmlUtil.addElement(self._mdsd_config_xml_tree, 'Events/MdsdEvents', mdsd_event_source_element)
mdsd_source_element = XmlUtil.createElement(mdsd_source_schema)
mdsd_source_element.set('name', 'ladfile'+str(file_id))
XmlUtil.addElement(self._mdsd_config_xml_tree, 'Sources', mdsd_source_element)
imfile_per_file_config = imfile_per_file_config_template.replace('#FILE#', item['file'])
imfile_per_file_config = imfile_per_file_config.replace('#STATFILE#', item['file'].replace("/","-"))
imfile_per_file_config = imfile_per_file_config.replace('#FILETAG#', 'ladfile'+str(file_id))
imfile_config += imfile_per_file_config
return imfile_config
def get_resource_id(self):
"""
Try to get resourceId from LadCfg. If not present, try to fetch from xmlCfg.
"""
lad_cfg = self.read_public_config('ladCfg')
resource_id = LadUtil.getResourceIdFromLadCfg(lad_cfg)
if not resource_id:
encoded_xml_cfg = self.read_public_config('xmlCfg').strip()
if encoded_xml_cfg:
xml_cfg = base64.b64decode(encoded_xml_cfg)
resource_id = XmlUtil.getXmlValue(XmlUtil.createElement(xml_cfg),
'diagnosticMonitorConfiguration/metrics', 'resourceId')
# Azure portal uses xmlCfg which contains WadCfg which is pascal case
# Currently we will support both casing and deprecate one later
if not resource_id:
resource_id = XmlUtil.getXmlValue(XmlUtil.createElement(xml_cfg),
'DiagnosticMonitorConfiguration/Metrics', 'resourceId')
return resource_id
def setEventVolume(mdsdCfg, ladCfg):
"""
Set event volume in mdsd config. Check if desired event volume is specified, first in ladCfg then in public config.
:param mdsdCfg: The XML document holding the mdsd config
:param ladCfg: The ladCfg part of the public config
:return: The event volume. If not set anywhere in public config, return "Medium"
:rtype: str
"""
eventVolume = LadUtil.getEventVolumeFromLadCfg(ladCfg)
if eventVolume:
hutil.log("Event volume found in ladCfg: " + eventVolume)
else:
eventVolume = readPublicConfig("eventVolume")
if eventVolume:
hutil.log("Event volume found in public config: " + eventVolume)
else:
eventVolume = "Medium"
hutil.log("Event volume not found in config. Using default value: " + eventVolume)
XmlUtil.setXmlValue(mdsdCfg, "Management", "eventVolume", eventVolume)
def _set_event_volume(self, lad_cfg):
"""
Set event volume in mdsd config. Check if desired event volume is specified,
first in ladCfg then in public config. If in neither then default to Medium.
:param lad_cfg: 'ladCfg' Json object to look up for the event volume setting.
:return: None. The mdsd config XML tree's eventVolume attribute is directly updated.
:rtype: str
"""
assert self._mdsd_config_xml_tree is not None
event_volume = LadUtil.getEventVolumeFromLadCfg(lad_cfg)
if event_volume:
self._logger_log("Event volume found in ladCfg: " + event_volume)
else:
event_volume = self._ext_settings.read_public_config("eventVolume")
if event_volume:
self._logger_log("Event volume found in public config: " + event_volume)
else:
event_volume = "Medium"
self._logger_log("Event volume not found in config. Using default value: " + event_volume)
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, "Management", "eventVolume", event_volume)
def UpdateXML(doc):
"""
Add to the mdsd XML the minimal set of OMI queries which will retrieve the metrics requested via AddMetric(). This
provider doesn't need any configuration external to mdsd; if it did, that would be generated here as well.
:param doc: XML document object to be updated
:return: None
"""
global _metrics, _eventNames, _omiClassName
for group in _metrics:
(class_name, condition_clause, sample_rate) = group
if not condition_clause:
condition_clause = default_condition(class_name)
columns = []
mappings = []
for metric in _metrics[group]:
omi_name = metric.counter_name()
scale = _scaling[class_name][omi_name]
columns.append(omi_name)
mappings.append('<MapName name="{0}" {1}>{2}</MapName>'.format(omi_name, scale, metric.label()))
column_string = ','.join(columns)
if condition_clause:
cql_query = quoteattr("SELECT {0} FROM {1} WHERE {2}".format(column_string,
_omiClassName[class_name], condition_clause))
else:
cql_query = quoteattr("SELECT {0} FROM {1}".format(column_string, _omiClassName[class_name]))
query = '''
<OMIQuery cqlQuery={qry} eventName={evname} omiNamespace="root/scx" sampleRateInSeconds="{rate}" storeType="local">
<Unpivot columnName="CounterName" columnValue="Value" columns={columns}>
{mappings}
</Unpivot>
</OMIQuery>'''.format(
qry=cql_query,
evname=quoteattr(_eventNames[group]),
columns=quoteattr(column_string),
rate=sample_rate,
mappings='\n '.join(mappings)
)
XmlUtil.addElement(doc, 'Events/OMI', ET.fromstring(query))
return
def createPortalSettings(tree, resourceId):
portal_config = ET.ElementTree()
portal_config.parse(os.path.join(WorkDir, "portal.xml.template"))
XmlUtil.setXmlValue(portal_config, "./DerivedEvents/DerivedEvent/LADQuery",
"partitionKey", resourceId)
XmlUtil.addElement(tree, 'Events', portal_config._root.getchildren()[0])
XmlUtil.addElement(tree, 'Events', portal_config._root.getchildren()[1])
def UpdateXML(doc):
"""
Add to the mdsd XML the minimal set of OMI queries which will retrieve the metrics requested via AddMetric(). This
provider doesn't need any configuration external to mdsd; if it did, that would be generated here as well.
:param doc: XML document object to be updated
:return: None
"""
global _metrics, _eventNames, _omiClassName
for group in _metrics:
(class_name, condition_clause, sample_rate) = group
if not condition_clause:
condition_clause = default_condition(class_name)
columns = []
mappings = []
for metric in _metrics[group]:
omi_name = metric.counter_name()
scale = _scaling[class_name][omi_name]
columns.append(omi_name)
mappings.append('<MapName name="{0}" {1}>{2}</MapName>'.format(
omi_name, scale, metric.label()))
column_string = ','.join(columns)
if condition_clause:
cql_query = quoteattr("SELECT {0} FROM {1} WHERE {2}".format(
column_string, _omiClassName[class_name], condition_clause))
else:
cql_query = quoteattr("SELECT {0} FROM {1}".format(
column_string, _omiClassName[class_name]))
query = '''
<OMIQuery cqlQuery={qry} eventName={evname} omiNamespace="root/scx" sampleRateInSeconds="{rate}" storeType="local">
<Unpivot columnName="CounterName" columnValue="Value" columns={columns}>
{mappings}
</Unpivot>
</OMIQuery>'''.format(qry=cql_query,
evname=quoteattr(_eventNames[group]),
columns=quoteattr(column_string),
rate=sample_rate,
mappings='\n '.join(mappings))
XmlUtil.addElement(doc, 'Events/OMI', ET.fromstring(query))
return
def createAccountSettings(tree,account,key,endpoint,aikey=None):
XmlUtil.setXmlValue(tree,'Accounts/Account',"account",account,['isDefault','true'])
XmlUtil.setXmlValue(tree,'Accounts/Account',"key",key,['isDefault','true'])
XmlUtil.setXmlValue(tree,'Accounts/Account',"tableEndpoint",endpoint,['isDefault','true'])
if aikey:
AIUtil.createAccountElement(tree,aikey)
def _update_account_settings(self, account, token, endpoint):
"""
Update the MDSD configuration Account element with Azure table storage properties.
Exactly one of (key, token) must be provided.
:param account: Storage account to which LAD should write data
:param token: SAS token to access the storage account
:param endpoint: Identifies the Azure instance (public or specific sovereign cloud) where the storage account is
"""
assert token, "Token must be given."
assert self._mdsd_config_xml_tree is not None
token = self._encrypt_secret_with_cert(token)
assert token, "Could not encrypt token"
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/SharedAccessSignature',
"account", account, ['isDefault', 'true'])
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/SharedAccessSignature',
"key", token, ['isDefault', 'true'])
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/SharedAccessSignature',
"decryptKeyPath", self._pkey_path, ['isDefault', 'true'])
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/SharedAccessSignature',
"tableEndpoint", endpoint, ['isDefault', 'true'])
XmlUtil.removeElement(self._mdsd_config_xml_tree, 'Accounts', 'Account')
def updateMargin(root, attributeLayoutPaddingName, attributeLayoutMarginName) :
# @SuppressWarnings("unchecked")
elements = [elem for elem in root.iter()]
if (len(elements) == 0) :
return
# Check if all children have margin dip value
for node in elements:
element = Element (node)
if (not XmlUtil.hasDipValue(element, attributeLayoutMarginName)) :
return
paddingValue = 0
if (XmlUtil.hasDipValue(root, attributeLayoutPaddingName)) :
paddingValue = XmlUtil.getDipValue(root, attributeLayoutPaddingName)
# find min margin
minMargin = sys.maxint
for node in elements:
element = Element(node)
minMargin = min(minMargin, XmlUtil.getDipValue(element,attributeLayoutMarginName))
# we add more margin to the parent
paddingValue += paddingValue + minMargin
if (paddingValue == 0) :
return
root.addAttribute(attributeLayoutPaddingName, paddingValue + Constants.UNIT_DIP)
# we subtract margin from children
for node in elements:
element = Element(node)
root.set(attributeLayoutMarginName, XmlUtil.getDipValue(element, attributeLayoutMarginName) - minMargin + Constants.UNIT_DIP)
def generateProject(rectViews, projectFolder, templateFolder, canvas_width, projectName="SketchToUI"):
CANVAS_WIDTH = canvas_width
dipCalulator = SketchDipCalculator(CANVAS_WIDTH, CANVAS_HEIGHT)
mOutProjectFolder = os.path.join(projectFolder,projectName )
rawRects = jsonToRect(rectViews,dipCalulator)
# return
rootView = createHierachy(rawRects, CANVAS_WIDTH,CANVAS_HEIGHT)
mProjectName = generateProjectName(projectName)
SketchProjectGenerator.setup(projectFolder, templateFolder)
# mDrawableWriter = DrawableWriter(file_extension, mOutProjectFolder)
creator = SketchLayoutCreator(rootView, mProjectName, mOutProjectFolder,dipCalulator)
# create layout
layoutDocument = creator.createDocument()
layoutFilter = SketchLayoutFilter()
#
anotateMap = layoutFilter.anotate(layoutDocument)
layoutFilter = SketchRelativeLayoutFilter()
layoutFilter.doFilter(layoutDocument, anotateMap)
layoutFilter = SketchLayoutFilter()
layoutFilter.doFilter(layoutDocument, anotateMap)
mainXmlPath = os.path.join(mOutProjectFolder,"app", "src","main", "res", "layout", "activity_main.xml")
# write to xml
XmlUtil.writeDocumentxml(layoutDocument,mainXmlPath)
# write style
# styleWriter = creator.mStyleWriter
# styleDocument = styleWriter.mRoot
# styleDocumentPath = os.path.join(mOutProjectFolder, "app", "src","main","res","values","styles.xml")
# XmlUtil.writeDocumentxml(styleDocument, styleDocumentPath)
#
#write to color file
colorWriter = creator.mColorWriter
colorDocument = colorWriter.mRoot
colorDocumentPath = os.path.join(mOutProjectFolder, "app", "src","main","res","values","colors.xml")
XmlUtil.writeDocumentxml(colorDocument, colorDocumentPath)
# write to string file
stringWriter = creator.mWriter
resourceDocument = stringWriter.mRoot
stringDocumentPath = os.path.join(mOutProjectFolder, "app", "src","main","res","values","strings.xml")
XmlUtil.writeDocumentxml(resourceDocument, stringDocumentPath)
SketchProjectGenerator.prepareProject(projectFolder, projectName)
return
def createDocument(self):
_map = {}
# create root element
rootElement = XmlUtil.createSketchRoot(
self.mDipCalculator, LayoutHelper.FRAMELAYOUT_ELEMENT,
self.mRootView, self.mColorWriter)
# To fix the style color
color = ColorUtil.cColortoInt(CColor.Black)
self.mColorWriter.addResource(selectColorMode(color))
color = ColorUtil.cColortoInt(CColor.Cyan)
self.mColorWriter.addResource(selectColorMode(color))
# add childeren to the layout
self.addChildrenLayout(rootElement, self.mRootView, 0, 0, _map)
return rootElement
def _add_portal_settings(self, resource_id):
"""
Update mdsd_config_xml_tree for Azure Portal metric collection setting.
It's basically applying the resource_id as the partitionKey attribute of LADQuery elements.
:param resource_id: ARM rerousce ID to provide as partitionKey in LADQuery elements
:return: None
"""
assert self._mdsd_config_xml_tree is not None
portal_config = ET.ElementTree()
portal_config.parse(os.path.join(self._ext_dir, 'portal.xml.template'))
XmlUtil.setXmlValue(portal_config, './DerivedEvents/DerivedEvent/LADQuery', 'partitionKey', resource_id)
XmlUtil.addElement(self._mdsd_config_xml_tree, 'Events', portal_config._root.getchildren()[0])
XmlUtil.addElement(self._mdsd_config_xml_tree, 'Events', portal_config._root.getchildren()[1])
def generate_omi_query_xml_elem(omi_query, sink=()):
"""
Helper for generating OMI event XML element
:param omi_query: Python dictionary object for the raw OMI query specified as a LAD 3.0 perfCfg array item
:param sink: (name, type) tuple for this OMI query. Not specified implies default XTable sink
:return: An XML element object for this OMI event that should be added to the mdsd XML cfg tree
"""
omi_xml_schema = """
<OMIQuery cqlQuery="" dontUsePerNDayTable="true" eventName="" omiNamespace="" priority="High" sampleRateInSeconds="" storeType="" />
""" if sink else """
<OMIQuery cqlQuery="" dontUsePerNDayTable="true" eventName="" omiNamespace="" priority="High" sampleRateInSeconds="" />
"""
xml_elem = XmlUtil.createElement(omi_xml_schema)
xml_elem.set('cqlQuery', omi_query['query'])
xml_elem.set('eventName', sink[0] if sink else omi_query['table'])
# Default OMI namespace is 'root/scx'
xml_elem.set('omiNamespace', omi_query['namespace'] if 'namespace' in omi_query else 'root/scx')
# Default query frequency is 300 seconds
xml_elem.set('sampleRateInSeconds', str(omi_query['frequency']) if 'frequency' in omi_query else '300')
if sink:
xml_elem.set('storeType', 'local' if sink[1] == 'EventHub' else sink[1])
return xml_elem
def createEventFileSettings(tree, files):
fileid = 0
sysconfig = """
$ModLoad imfile
"""
if not files:
return
for file in files:
fileid = fileid + 1
eventSourceElement = XmlUtil.createElement(eventSourceSchema)
XmlUtil.setXmlValue(eventSourceElement, "RouteEvent", "eventName", file["table"])
eventSourceElement.set("source", "ladfile" + str(fileid))
XmlUtil.addElement(tree, "Events/MdsdEvents", eventSourceElement)
sourceElement = XmlUtil.createElement(sourceSchema)
sourceElement.set("name", "ladfile" + str(fileid))
XmlUtil.addElement(tree, "Sources", sourceElement)
syslog_config = syslogEventSourceConfig.replace("#FILE#", file["file"])
syslog_config = syslog_config.replace("#STATFILE#", file["file"].replace("/", "-"))
syslog_config = syslog_config.replace("#FILETAG#", "ladfile" + str(fileid))
sysconfig += syslog_config
return sysconfig
def createEventFileSettings(tree,files):
fileid = 0;
sysconfig = """
$ModLoad imfile
"""
if not files:
return
for file in files:
fileid=fileid+1
eventSourceElement = XmlUtil.createElement(eventSourceSchema)
XmlUtil.setXmlValue(eventSourceElement,'RouteEvent','eventName',file["table"])
eventSourceElement.set('source','ladfile'+str(fileid))
XmlUtil.addElement(tree,'Events/MdsdEvents',eventSourceElement)
sourceElement = XmlUtil.createElement(sourceSchema)
sourceElement.set('name','ladfile'+str(fileid))
XmlUtil.addElement(tree,'Sources',sourceElement)
syslog_config = syslogEventSourceConfig.replace('#FILE#',file['file'])
syslog_config = syslog_config.replace('#STATFILE#',file['file'].replace("/","-"))
syslog_config = syslog_config.replace('#FILETAG#','ladfile'+str(fileid))
sysconfig+=syslog_config
return sysconfig
def generate_all_configs(self):
"""
Generates configs for all components required by LAD.
Generates XML cfg file for mdsd, from JSON config settings (public & private).
Also generates rsyslog/syslog-ng configs corresponding to 'syslogEvents' or 'syslogCfg' setting.
Also generates fluentd's syslog/tail src configs and out_mdsd configs.
The rsyslog/syslog-ng and fluentd configs are not yet saved to files. They are available through
the corresponding getter methods of this class (get_fluentd_*_config(), get_*syslog*_config()).
Returns (True, '') if config was valid and proper xmlCfg.xml was generated.
Returns (False, '...') if config was invalid and the error message.
"""
# 1. Add DeploymentId (if available) to identity columns
if self._deployment_id:
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, "Management/Identity/IdentityComponent", "",
self._deployment_id, ["name", "DeploymentId"])
# 2. Use ladCfg to generate OMIQuery and LADQuery elements
lad_cfg = self._ladCfg()
if lad_cfg:
try:
self._update_metric_collection_settings(lad_cfg)
resource_id = self._ext_settings.get_resource_id()
if resource_id:
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Events/DerivedEvents/DerivedEvent/LADQuery',
'partitionKey', escape_nonalphanumerics(resource_id))
lad_query_instance_id = ""
uuid_for_instance_id = self._fetch_uuid()
if resource_id.find("providers/Microsoft.Compute/virtualMachineScaleSets") >= 0:
lad_query_instance_id = uuid_for_instance_id
self._set_xml_attr("instanceID", lad_query_instance_id, "Events/DerivedEvents/DerivedEvent/LADQuery")
# Set JsonBlob sink-related elements
self._add_obo_field(name='resourceId', value=resource_id)
self._add_obo_field(name='agentIdentityHash', value=uuid_for_instance_id)
except Exception as e:
self._logger_error("Failed to create portal config error:{0} {1}".format(e, traceback.format_exc()))
return False, 'Failed to create portal config from ladCfg (see extension error logs for more details)'
# 3. Generate config for perfCfg. Need to distinguish between non-AppInsights scenario and AppInsights scenario,
# so check if Application Insights key is present and pass it to the actual helper
# function (self._apply_perf_cfg()).
try:
self._apply_perf_cfg()
except Exception as e:
self._logger_error("Failed check for Application Insights key in LAD configuration with exception:{0}\n"
"Stacktrace: {1}".format(e, traceback.format_exc()))
return False, 'Failed to update perf counter config (see extension error logs for more details)'
# 4. Generate omsagent (fluentd) configs, rsyslog/syslog-ng config, and update corresponding mdsd config XML
try:
syslogEvents_setting = self._ext_settings.get_syslogEvents_setting()
fileLogs_setting = self._ext_settings.get_fileLogs_setting()
lad_logging_config_helper = LadLoggingConfig(syslogEvents_setting, fileLogs_setting, self._sink_configs,
self._pkey_path, self._cert_path, self._encrypt_secret)
mdsd_syslog_config = lad_logging_config_helper.get_mdsd_syslog_config()
mdsd_filelog_config = lad_logging_config_helper.get_mdsd_filelog_config()
copy_source_mdsdevent_eh_url_elems(self._mdsd_config_xml_tree, mdsd_syslog_config)
copy_source_mdsdevent_eh_url_elems(self._mdsd_config_xml_tree, mdsd_filelog_config)
self._fluentd_syslog_src_config = lad_logging_config_helper.get_fluentd_syslog_src_config()
self._fluentd_tail_src_config = lad_logging_config_helper.get_fluentd_filelog_src_config()
self._fluentd_out_mdsd_config = lad_logging_config_helper.get_fluentd_out_mdsd_config()
self._rsyslog_config = lad_logging_config_helper.get_rsyslog_config()
self._syslog_ng_config = lad_logging_config_helper.get_syslog_ng_config()
except Exception as e:
self._logger_error("Failed to create omsagent (fluentd), rsyslog/syslog-ng configs or to update "
"corresponding mdsd config XML. Error: {0}\nStacktrace: {1}"
.format(e, traceback.format_exc()))
return False, 'Failed to generate configs for fluentd, syslog, and mdsd ' \
'(see extension error logs for more details)'
# 5. Before starting to update the storage account settings, log extension's entire settings
# with secrets redacted, for diagnostic purpose.
self._ext_settings.log_ext_settings_with_secrets_redacted(self._logger_log, self._logger_error)
# 6. Actually update the storage account settings on mdsd config XML tree (based on extension's
# protectedSettings).
account = self._ext_settings.read_protected_config('storageAccountName').strip()
if not account:
return False, "Must specify storageAccountName"
if self._ext_settings.read_protected_config('storageAccountKey'):
return False, "The storageAccountKey protected setting is not supported and must not be used"
token = self._ext_settings.read_protected_config('storageAccountSasToken').strip()
if not token or token == '?':
return False, "Must specify storageAccountSasToken"
if '?' == token[0]:
token = token[1:]
endpoint = get_storage_endpoint_with_account(account,
self._ext_settings.read_protected_config('storageAccountEndPoint'))
self._update_account_settings(account, token, endpoint)
# 7. Update mdsd config XML's eventVolume attribute based on the logic specified in the helper.
self._set_event_volume(lad_cfg)
# 8. Finally generate mdsd config XML file out of the constructed XML tree object.
self._mdsd_config_xml_tree.write(os.path.join(self._ext_dir, 'xmlCfg.xml'))
return True, ""
def createAccountElement(mdsdCfg, aikey):
aiAccountElement = XmlUtil.createElement("<Account moniker=\"appinsights\" appInsightsKey=\"" + aikey + "\"/>");
XmlUtil.addElement(mdsdCfg,'Accounts',aiAccountElement)
def config(xmltree,key,value,xmlpath,selector=[]):
v = readPublicConfig(key)
if not v:
v = value
XmlUtil.setXmlValue(xmltree,xmlpath,key,v,selector)
def createSyslogRouteEventElement(mdsdCfg):
aiSyslogRouteEvent = XmlUtil.createElement("<RouteEvent eventName=\"aiSyslogRouteEvent\" priority=\"High\" account=\"appinsights\" storeType=\"appinsights\"/>");
XmlUtil.addElement(mdsdCfg,'Events/MdsdEvents/MdsdEventSource',aiSyslogRouteEvent,['source','syslog'])
def addChildrenLayout(self, element, rectView, parentLeft, parentTop,
rectViewElementInfoMap):
# Setting background
if (self.useTransparentBackground(rectView)):
XmlUtil.addBackgroundColor(element,
ColorUtil.toInt(0, 255, 255, 255),
self.mColorWriter)
# elif (RectViewTypes.isContanerView(rectView)) :
# We always want to genenate background regard less of the respect
# ratioand Environment.get().getValue(Environment.KEY_KEEP_ASPECT_RATIO) == Boolean.TRUE
# We will not do this if ratio is between input image and output is the same
# bound = ImageUtil.getImageFromRect(self.mImage, rectView)
# # We remove children here using 4 layer channel So we have to make sure that when we save it we will not Adding anymore layer
# newImageBackground = ImageUtil.removeChildren(bound, rectView)
# drawableId = self.mDrawableWriter.addResourceDirectly(newImageBackground, rectView)
# XmlUtil.addBackroundImage(element, drawableId)
else:
XmlUtil.addBackgroundColor(element, rectView.mColor,
self.mColorWriter)
for childRectView in rectView.mChildren:
_id = ""
# list view has it own index
if (childRectView.mType == RectViewTypes.VIEW_TYPE_LIST):
_id = childRectView.mListInfo.xmlId
else:
_id = self.getId(LayoutHelper.FRAMELAYOUT_ELEMENT,
childRectView)
marginLeft = childRectView.x - parentLeft
marginTop = childRectView.y - parentTop
childElement = None
if (self.useTransparentBackground(childRectView)):
childElement = XmlUtil.addElement(
self.mDipCalculator, element,
self.getElementTypeForRect(childRectView), childRectView,
marginLeft, marginTop, _id)
else:
childElement = XmlUtil.addElement(
self.mDipCalculator, element,
self.getElementTypeForRect(childRectView), childRectView,
marginLeft, marginTop, _id, self.mColorWriter)
rectViewElementInfoMap[childRectView] = ElementInfo(
childElement, _id)
self.addChildrenLayout(childElement, childRectView,
childRectView.x, childRectView.y,
rectViewElementInfoMap)
# image view
if (rectView.mType == RectViewTypes.VIEW_TYPE_IMAGE):
drawableId = self.interestedIcons.get(rectView.mImageInfo.iconInfo)
element.tag = Constants.ELEMENT_IMAGE_VIEW
XmlUtil.addImageDrawable(element, drawableId)
# override attributes
XmlUtil.removeAttribute(element, Constants.ATTRIBUTE_BACKGROUND)
_id = self.getId(Constants.ELEMENT_IMAGE_VIEW, rectView)
XmlUtil.addId(element, _id)
XmlUtil.addScaleType(element, "fitXY")
rectViewElementInfoMap[rectView] = ElementInfo(element, _id)
elif (rectView.mType == RectViewTypes.VIEW_TYPE_TEXT):
textWrapper = rectView.mTextInfo.textWrapper
stringId = self.mWriter.addResource(textWrapper.text)
rectView.mTextInfo._id = stringId
element.tag = Constants.ELEMENT_TEXT_VIEW
if (Environment.getValue(
Environment.KEY_TEXT_WIDTH_WRAP_CONTENT) == True):
XmlUtil.addSize(self.mDipCalculator, element,
Constants.ATTRIBUTE_WRAP_CONTENT,
rectView.height)
else:
XmlUtil.addSize(self.mDipCalculator, element, rectView.width,
rectView.height)
XmlUtil.addBackgroundColor(element, rectView.mColor,
self.mColorWriter)
element.set(Constants.ATTRIBUTE_TEXT,
XmlUtil.getReferenceResourceId(stringId))
_id = self.getId(Constants.ELEMENT_TEXT_VIEW, rectView)
XmlUtil.addId(element, _id)
textAttributes = textWrapper.getTextAttributes(
self.mOcr, rectView.height)
XmlUtil.addTextColor(element, rectView.textColor,
self.mColorWriter)
stypeId = self.mStyleWriter.addResource(textAttributes)
element.set(Constants.ATTRIBUTE_STYLE,
XmlUtil.getReferenceStyleId(stypeId))
rectViewElementInfoMap[rectView] = ElementInfo(element, _id)
def createPortalSettings(tree,resourceId):
portal_config = ET.ElementTree()
portal_config.parse(os.path.join(WorkDir, "portal.xml.template"))
XmlUtil.setXmlValue(portal_config,"./DerivedEvents/DerivedEvent/LADQuery","partitionKey",resourceId)
XmlUtil.addElement(tree,'Events',portal_config._root.getchildren()[0])
XmlUtil.addElement(tree,'Events',portal_config._root.getchildren()[1])
def configSettings():
'''
Generates XML cfg file for mdsd, from JSON config settings (public & private).
Returns (True, '') if config was valid and proper xmlCfg.xml was generated.
Returns (False, '...') if config was invalid and the error message.
'''
mdsdCfgstr = readPublicConfig('mdsdCfg')
if not mdsdCfgstr:
with open(os.path.join(WorkDir, './mdsdConfig.xml.template'),"r") as defaulCfg:
mdsdCfgstr = defaulCfg.read()
else:
mdsdCfgstr = base64.b64decode(mdsdCfgstr)
mdsdCfg = ET.ElementTree()
mdsdCfg._setroot(XmlUtil.createElement(mdsdCfgstr))
# update deployment id
deployment_id = get_deployment_id()
XmlUtil.setXmlValue(mdsdCfg, "Management/Identity/IdentityComponent", "", deployment_id, ["name", "DeploymentId"])
try:
resourceId = getResourceId()
if resourceId:
createPortalSettings(mdsdCfg,escape(resourceId))
instanceID=""
if resourceId.find("providers/Microsoft.Compute/virtualMachineScaleSets") >=0:
instanceID = readUUID()
config(mdsdCfg,"instanceID",instanceID,"Events/DerivedEvents/DerivedEvent/LADQuery")
except Exception as e:
hutil.error("Failed to create portal config error:{0} {1}".format(e,traceback.format_exc()))
# Check if Application Insights key is present in ladCfg
ladCfg = readPublicConfig('ladCfg')
try:
aikey = AIUtil.tryGetAiKey(ladCfg)
if aikey:
hutil.log("Application Insights key found.")
else:
hutil.log("Application Insights key not found.")
except Exception as e:
hutil.error("Failed check for Application Insights key in LAD configuration with exception:{0} {1}".format(e,traceback.format_exc()))
generatePerformanceCounterConfiguration(mdsdCfg,aikey != None)
syslogCfg = getSyslogCfg()
fileCfg = getFileCfg()
#fileCfg = [{"file":"/var/log/waagent.log","table":"waagent"},{"file":"/var/log/waagent2.log","table":"waagent3"}]
try:
if fileCfg:
syslogCfg = createEventFileSettings(mdsdCfg,fileCfg)+syslogCfg
with open(omfileconfig,'w') as hfile:
hfile.write(syslogCfg)
except Exception as e:
hutil.error("Failed to create syslog_file config error:{0} {1}".format(e,traceback.format_exc()))
account = readPrivateConfig('storageAccountName')
if not account:
return False, "Empty storageAccountName"
key = readPrivateConfig('storageAccountKey')
if not key:
return False, "Empty storageAccountKey"
endpoint = readPrivateConfig('endpoint')
if not endpoint:
endpoint = 'table.core.windows.net'
endpoint = 'https://'+account+"."+endpoint
createAccountSettings(mdsdCfg,account,key,endpoint,aikey)
# Check and add new syslog RouteEvent for Application Insights.
if aikey:
AIUtil.createSyslogRouteEventElement(mdsdCfg)
setEventVolume(mdsdCfg,ladCfg)
config(mdsdCfg,"sampleRateInSeconds","60","Events/OMI/OMIQuery")
mdsdCfg.write(os.path.join(WorkDir, './xmlCfg.xml'))
return True, ""
def _update_account_settings(self, account, key, token, endpoint, aikey=None):
"""
Update the MDSD configuration Account element with Azure table storage properties.
Exactly one of (key, token) must be provided. If an aikey is passed, then add a new Account element for Application
Insights with the application insights key.
:param account: Storage account to which LAD should write data
:param key: Shared key secret for the storage account, if present
:param token: SAS token to access the storage account, if present
:param endpoint: Identifies the Azure instance (public or specific sovereign cloud) where the storage account is
:param aikey: Key for accessing AI, if present
"""
assert key or token, "Either key or token must be given."
assert self._mdsd_config_xml_tree is not None
handler_cert_path, handler_pkey_path = self._get_handler_cert_pkey_paths(self._ext_settings.get_handler_settings())
if key:
key = self._encrypt_secret_with_cert(handler_cert_path, key)
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/Account',
"account", account, ['isDefault', 'true'])
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/Account',
"key", key, ['isDefault', 'true'])
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/Account',
"decryptKeyPath", handler_pkey_path, ['isDefault', 'true'])
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/Account',
"tableEndpoint", endpoint, ['isDefault', 'true'])
XmlUtil.removeElement(self._mdsd_config_xml_tree, 'Accounts', 'SharedAccessSignature')
else: # token
token = self._encrypt_secret_with_cert(handler_cert_path, token)
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/SharedAccessSignature',
"account", account, ['isDefault', 'true'])
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/SharedAccessSignature',
"key", token, ['isDefault', 'true'])
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/SharedAccessSignature',
"decryptKeyPath", handler_pkey_path, ['isDefault', 'true'])
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, 'Accounts/SharedAccessSignature',
"tableEndpoint", endpoint, ['isDefault', 'true'])
XmlUtil.removeElement(self._mdsd_config_xml_tree, 'Accounts', 'Account')
if aikey:
AIUtil.createAccountElement(self._mdsd_config_xml_tree, aikey)
def generate_all_configs(self):
"""
Generates configs for all components required by LAD.
Generates XML cfg file for mdsd, from JSON config settings (public & private).
Also generates rsyslog/syslog-ng configs corresponding to 'syslogEvents' or 'syslogCfg' setting.
Also generates fluentd's syslog/tail src configs and out_mdsd configs.
The rsyslog/syslog-ng and fluentd configs are not yet saved to files. They are available through
the corresponding getter methods of this class (get_fluentd_*_config(), get_*syslog*_config()).
Returns (True, '') if config was valid and proper xmlCfg.xml was generated.
Returns (False, '...') if config was invalid and the error message.
"""
# 1. Add DeploymentId (if available) to identity columns
if self._deployment_id:
XmlUtil.setXmlValue(self._mdsd_config_xml_tree,
"Management/Identity/IdentityComponent", "",
self._deployment_id, ["name", "DeploymentId"])
# 2. Generate telegraf, MetricsExtension, omsagent (fluentd) configs, rsyslog/syslog-ng config, and update corresponding mdsd config XML
try:
lad_cfg = self._ladCfg()
if not lad_cfg:
return False, 'Unable to find Ladcfg element. Failed to generate configs for fluentd, syslog, and mdsd ' \
'(see extension error logs for more details)'
syslogEvents_setting = self._ext_settings.get_syslogEvents_setting(
)
fileLogs_setting = self._ext_settings.get_fileLogs_setting()
lad_logging_config_helper = LadLoggingConfig(
syslogEvents_setting, fileLogs_setting, self._sink_configs,
self._pkey_path, self._cert_path, self._encrypt_secret)
mdsd_syslog_config = lad_logging_config_helper.get_mdsd_syslog_config(
self._ext_settings.read_protected_config(
'disableStorageAccount') == True)
mdsd_filelog_config = lad_logging_config_helper.get_mdsd_filelog_config(
)
copy_source_mdsdevent_eh_url_elems(self._mdsd_config_xml_tree,
mdsd_syslog_config)
copy_source_mdsdevent_eh_url_elems(self._mdsd_config_xml_tree,
mdsd_filelog_config)
self._fluentd_syslog_src_config = lad_logging_config_helper.get_fluentd_syslog_src_config(
)
self._fluentd_tail_src_config = lad_logging_config_helper.get_fluentd_filelog_src_config(
)
self._fluentd_out_mdsd_config = lad_logging_config_helper.get_fluentd_out_mdsd_config(
)
self._rsyslog_config = lad_logging_config_helper.get_rsyslog_config(
)
self._syslog_ng_config = lad_logging_config_helper.get_syslog_ng_config(
)
parsed_perf_settings = lad_logging_config_helper.parse_lad_perf_settings(
lad_cfg)
self._telegraf_config, self._telegraf_namespaces = telhandler.handle_config(
parsed_perf_settings, self._telegraf_me_url,
self._telegraf_mdsd_url, True)
#Handle the EH, JsonBlob and AzMonSink logic
self._update_metric_collection_settings(lad_cfg,
self._telegraf_namespaces)
mdsd_telegraf_config = lad_logging_config_helper.get_mdsd_telegraf_config(
self._telegraf_namespaces)
copy_source_mdsdevent_eh_url_elems(self._mdsd_config_xml_tree,
mdsd_telegraf_config)
resource_id = self._ext_settings.get_resource_id()
if resource_id:
# Set JsonBlob sink-related elements
uuid_for_instance_id = self._fetch_uuid()
self._add_obo_field(name='resourceId', value=resource_id)
self._add_obo_field(name='agentIdentityHash',
value=uuid_for_instance_id)
XmlUtil.setXmlValue(
self._mdsd_config_xml_tree,
'Events/DerivedEvents/DerivedEvent/LADQuery',
'partitionKey', escape_nonalphanumerics(resource_id))
lad_query_instance_id = ""
if resource_id.find(
"providers/Microsoft.Compute/virtualMachineScaleSets"
) >= 0:
lad_query_instance_id = uuid_for_instance_id
self._set_xml_attr(
"instanceID", lad_query_instance_id,
"Events/DerivedEvents/DerivedEvent/LADQuery")
else:
return False, 'Unable to find resource id in the config. Failed to generate configs for Metrics in mdsd ' \
'(see extension error logs for more details)'
#Only enable Metrics if AzMonSink is in the config
if self._enable_metrics_extension:
me_handler.setup_me(True)
except Exception as e:
self._logger_error(
"Failed to create omsagent (fluentd), rsyslog/syslog-ng configs, telegraf config or to update "
"corresponding mdsd config XML. Error: {0}\nStacktrace: {1}".
format(e, traceback.format_exc()))
return False, 'Failed to generate configs for fluentd, syslog, and mdsd; see extension.log for more details.'
# 3. Before starting to update the storage account settings, log extension's entire settings
# with secrets redacted, for diagnostic purpose.
self._ext_settings.log_ext_settings_with_secrets_redacted(
self._logger_log, self._logger_error)
# 4. Actually update the storage account settings on mdsd config XML tree (based on extension's
# protectedSettings).
account = self._ext_settings.read_protected_config(
'storageAccountName').strip()
if not account:
return False, "Configuration Error: Must specify storageAccountName in protected settings. For information on protected settings, " \
"visit https://docs.microsoft.com/en-us/azure/virtual-machines/extensions/diagnostics-linux#protected-settings."
if self._ext_settings.read_protected_config('storageAccountKey'):
return False, "Configuration Error: The storageAccountKey protected setting is deprecated in LAD 3.0 and cannot be used. " \
"Instead, use the storageAccountSasToken setting. For documentation of this setting and instructions for generating " \
"a SAS token, visit https://docs.microsoft.com/en-us/azure/virtual-machines/extensions/diagnostics-linux#protected-settings."
token = self._ext_settings.read_protected_config(
'storageAccountSasToken').strip()
if not token or token == '?':
return False, "Configuration Error: Must specify storageAccountSasToken in the protected settings. For documentation of this setting and instructions " \
"for generating a SAS token, visit https://docs.microsoft.com/en-us/azure/virtual-machines/extensions/diagnostics-linux#protected-settings."
if '?' == token[0]:
token = token[1:]
endpoints = get_storage_endpoints_with_account(
account,
self._ext_settings.read_protected_config('storageAccountEndPoint'))
self._update_account_settings(account, token, endpoints)
# 5. Update mdsd config XML's eventVolume attribute based on the logic specified in the helper.
self._set_event_volume(lad_cfg)
# 6. Finally generate mdsd config XML file out of the constructed XML tree object.
self._mdsd_config_xml_tree.write(
os.path.join(self._ext_dir, 'xmlCfg.xml'))
return True, ""
def generate_mdsd_rsyslog_configs(self):
"""
Generates XML cfg file for mdsd, from JSON config settings (public & private).
Also generates rsyslog imfile conf file from the 'fileCfg' setting.
Returns (True, '') if config was valid and proper xmlCfg.xml was generated.
Returns (False, '...') if config was invalid and the error message.
"""
# 1. Get the mdsd config XML tree base.
# - 1st priority is from the extension setting's 'mdsdCfg' value.
# Note that we have never used this option.
# - 2nd priority is to use the provided XML template stored in <ext_dir>/mdsdConfig.xml.template.
mdsd_cfg_str = self._ext_settings.get_mdsd_cfg()
if not mdsd_cfg_str:
with open(os.path.join(self._ext_dir, './mdsdConfig.xml.template'), "r") as f:
mdsd_cfg_str = f.read()
self._mdsd_config_xml_tree = ET.ElementTree()
self._mdsd_config_xml_tree._setroot(XmlUtil.createElement(mdsd_cfg_str))
# 2. Add DeploymentId (if available) to identity columns
if self._deployment_id:
XmlUtil.setXmlValue(self._mdsd_config_xml_tree, "Management/Identity/IdentityComponent", "",
self._deployment_id, ["name", "DeploymentId"])
# 2.1. Apply resourceId attribute to LADQuery elements
try:
resource_id = self._ext_settings.get_resource_id()
if resource_id:
escaped_resource_id_str = escape_nonalphanumerics(resource_id)
self._add_portal_settings(escaped_resource_id_str)
instanceID = ""
if resource_id.find("providers/Microsoft.Compute/virtualMachineScaleSets") >= 0:
instanceID = read_uuid(self._run_command)
self._set_xml_attr("instanceID", instanceID, "Events/DerivedEvents/DerivedEvent/LADQuery")
except Exception as e:
self._logger_error("Failed to create portal config error:{0} {1}".format(e, traceback.format_exc()))
# 3. Update perf counter config. Need to distinguish between non-AppInsights scenario and AppInsights scenario,
# so check if Application Insights key is present in ladCfg first, and pass it to the actual helper
# function (self._apply_perf_cfgs()).
lad_cfg = self._ext_settings.read_public_config('ladCfg')
do_ai = False
aikey = None
try:
aikey = AIUtil.tryGetAiKey(lad_cfg)
if aikey:
self._logger_log("Application Insights key found.")
do_ai = True
else:
self._logger_log("Application Insights key not found.")
except Exception as e:
self._logger_error("Failed check for Application Insights key in LAD configuration with exception:{0}\n"
"Stacktrace: {1}".format(e, traceback.format_exc()))
self._apply_perf_cfgs(do_ai)
# 4. Generate rsyslog imfile config. It's unclear why non-imfile config stuff (self._get_syslog_config())
# is appended to imfileconfig as well. That part has never been used, as far as I remember, and will
# definitely need to change later.
syslog_cfg = self._ext_settings.get_syslog_config()
file_cfg = self._ext_settings.get_file_monitoring_config()
# fileCfg = [{"file":"/var/log/waagent.log","table":"waagent"},{"file":"/var/log/waagent2.log","table":"waagent3"}]
try:
if file_cfg:
syslog_cfg = self._update_and_get_file_monitoring_settings(file_cfg) + syslog_cfg
with open(self._imfile_config_filename, 'w') as imfile_config_file:
imfile_config_file.write(syslog_cfg)
except Exception as e:
self._logger_error("Failed to create rsyslog imfile config. Error:{0}\n"
"Stacktrace: {1}".format(e, traceback.format_exc()))
# 5. Before starting to update the storage account settings, log extension's protected settings'
# keys only (except well-known values), for diagnostic purpose. This is mainly to make sure that
# the extension's Json settings include a correctly entered 'storageEndpoint'.
self._ext_settings.log_protected_settings_keys(self._logger_log, self._logger_error)
# 6. Actually update the storage account settings on mdsd config XML tree (based on extension's
# protectedSettings).
account = self._ext_settings.read_protected_config('storageAccountName')
if not account:
return False, "Empty storageAccountName"
key = self._ext_settings.read_protected_config('storageAccountKey')
token = self._ext_settings.read_protected_config('storageAccountSasToken')
if not key and not token:
return False, "Neither storageAccountKey nor storageAccountSasToken is given"
if key and token:
return False, "Either storageAccountKey or storageAccountSasToken (but not both) should be given"
endpoint = get_storage_endpoint_with_account(account,
self._ext_settings.read_protected_config('storageAccountEndPoint'))
self._update_account_settings(account, key, token, endpoint, aikey)
# 7. Check and add new syslog RouteEvent for Application Insights.
if aikey:
AIUtil.createSyslogRouteEventElement(self._mdsd_config_xml_tree)
# 8. Update mdsd config XML's eventVolume attribute based on the logic specified in the helper.
self._set_event_volume(lad_cfg)
# 9. Update mdsd config XML's sampleRateInSeconds attribute with default '60'
self._set_xml_attr("sampleRateInSeconds", "60", "Events/OMI/OMIQuery")
# 10. Finally generate mdsd config XML file out of the constructed XML tree object.
self._mdsd_config_xml_tree.write(os.path.join(self._ext_dir, './xmlCfg.xml'))
return True, ""
