def collect_tags(self, infrastructure_data):
# type: (InfrastructureData) -> ResourceTags
"""
Fetch the all tags, build tags for each monitored resources and store all of that into the tags_cache.
"""
if not self.api_rest:
return {}
# In order to be more efficient in tag collection, the infrastructure data is filtered as much as possible.
# All filters are applied except the ones based on tags of course.
resource_filters_without_tags = [f for f in self._config.resource_filters if not isinstance(f, TagFilter)]
filtered_infra_data = {
mor: props
for mor, props in iteritems(infrastructure_data)
if isinstance(mor, tuple(self._config.collected_resource_types))
and is_resource_collected_by_filters(mor, infrastructure_data, resource_filters_without_tags)
}
t0 = Timer()
mors_list = list(filtered_infra_data.keys())
try:
mor_tags = self.api_rest.get_resource_tags_for_mors(mors_list)
except Exception as e:
self.log.error("Failed to collect tags: %s", e)
return {}
self.gauge(
'datadog.vsphere.query_tags.time',
t0.total(),
tags=self._config.base_tags,
raw=True,
hostname=self._hostname,
)
return mor_tags
def test_is_realtime_resource_collected_by_filters(realtime_instance):
realtime_instance['resource_filters'] = [
{'resource': 'vm', 'property': 'name', 'patterns': [r'^\$VM5$', r'^VM4-2\d$']},
{'resource': 'vm', 'property': 'inventory_path', 'patterns': [u'\\/D\xe4tacenter\\/vm\\/m.*']},
{'resource': 'vm', 'property': 'hostname', 'patterns': [r'10\.0\.0\.103']},
{'resource': 'vm', 'property': 'guest_hostname', 'patterns': [r'ubuntu-test']},
{'resource': 'vm', 'property': 'tag', 'patterns': [r'env:production']},
{'resource': 'host', 'property': 'name', 'patterns': [r'10\.0\.0\.103'], 'type': 'blacklist'},
]
realtime_instance['collect_tags'] = True
collected_resources = [
'VM2-1',
'$VM3-2',
'$VM5',
'10.0.0.101',
'10.0.0.102',
'10.0.0.104',
u'VM1-6ê',
'VM3-1',
'VM4-20',
'migrationTest',
]
check = VSphereCheck('vsphere', {}, [realtime_instance])
formatted_filters = check.config.resource_filters
infra = MockedAPI(realtime_instance).get_infrastructure()
resources = [m for m in infra if m.__class__ in (vim.VirtualMachine, vim.HostSystem)]
VM2_1 = next(r for r in resources if infra.get(r).get('name') == 'VM2-1')
check.infrastructure_cache.set_all_tags({vim.VirtualMachine: {VM2_1._moId: ['env:production', 'tag:2']}})
for resource in resources:
is_collected = infra.get(resource).get('name') in collected_resources
assert (
is_resource_collected_by_filters(
resource, infra, formatted_filters, check.infrastructure_cache.get_mor_tags(resource)
)
== is_collected
)
def refresh_infrastructure_cache(self):
# type: () -> None
"""Fetch the complete infrastructure, generate tags for each monitored resources and store all of that
into the infrastructure_cache. It also computes the resource `hostname` property to be used when submitting
metrics for this mor."""
self.log.debug("Refreshing the infrastructure cache...")
t0 = Timer()
infrastructure_data = self.api.get_infrastructure()
self.gauge(
"datadog.vsphere.refresh_infrastructure_cache.time",
t0.total(),
tags=self.config.base_tags,
raw=True,
hostname=self._hostname,
)
self.log.debug("Infrastructure cache refreshed in %.3f seconds.",
t0.total())
self.log.debug("Infrastructure cache: %s", infrastructure_data)
all_tags = {}
if self.config.should_collect_tags:
all_tags = self.collect_tags(infrastructure_data)
self.infrastructure_cache.set_all_tags(all_tags)
for mor, properties in iteritems(infrastructure_data):
if not isinstance(mor, tuple(
self.config.collected_resource_types)):
# Do nothing for the resource types we do not collect
continue
if not is_resource_collected_by_filters(
mor, infrastructure_data, self.config.resource_filters,
self.infrastructure_cache.get_mor_tags(mor)):
# The resource does not match the specified whitelist/blacklist patterns.
continue
mor_name = to_string(properties.get("name", "unknown"))
mor_type_str = MOR_TYPE_AS_STRING[type(mor)]
hostname = None
tags = []
if isinstance(mor, vim.VirtualMachine):
power_state = properties.get("runtime.powerState")
if power_state != vim.VirtualMachinePowerState.poweredOn:
# Skipping because the VM is not powered on
# TODO: Sometimes VM are "poweredOn" but "disconnected" and thus have no metrics
self.log.debug("Skipping VM %s in state %s", mor_name,
to_string(power_state))
continue
# Hosts are not considered as parents of the VMs they run, we use the `runtime.host` property
# to get the name of the ESXi host
runtime_host = properties.get("runtime.host")
runtime_host_props = infrastructure_data[
runtime_host] if runtime_host else {}
runtime_hostname = to_string(
runtime_host_props.get("name", "unknown"))
tags.append('vsphere_host:{}'.format(runtime_hostname))
if self.config.use_guest_hostname:
hostname = properties.get("guest.hostName", mor_name)
else:
hostname = mor_name
elif isinstance(mor, vim.HostSystem):
hostname = mor_name
else:
tags.append('vsphere_{}:{}'.format(mor_type_str, mor_name))
tags.extend(get_parent_tags_recursively(mor, infrastructure_data))
tags.append('vsphere_type:{}'.format(mor_type_str))
# Attach tags from fetched attributes.
tags.extend(properties.get('attributes', []))
mor_payload = {"tags": tags} # type: Dict[str, Any]
if hostname:
mor_payload['hostname'] = hostname
self.infrastructure_cache.set_mor_props(mor, mor_payload)
