def collect_tags(self, infrastructure_data): # type: (InfrastructureData) -> ResourceTags """ Fetch the all tags, build tags for each monitored resources and store all of that into the tags_cache. """ if not self.api_rest: return {} # In order to be more efficient in tag collection, the infrastructure data is filtered as much as possible. # All filters are applied except the ones based on tags of course. resource_filters_without_tags = [f for f in self._config.resource_filters if not isinstance(f, TagFilter)] filtered_infra_data = { mor: props for mor, props in iteritems(infrastructure_data) if isinstance(mor, tuple(self._config.collected_resource_types)) and is_resource_collected_by_filters(mor, infrastructure_data, resource_filters_without_tags) } t0 = Timer() mors_list = list(filtered_infra_data.keys()) try: mor_tags = self.api_rest.get_resource_tags_for_mors(mors_list) except Exception as e: self.log.error("Failed to collect tags: %s", e) return {} self.gauge( 'datadog.vsphere.query_tags.time', t0.total(), tags=self._config.base_tags, raw=True, hostname=self._hostname, ) return mor_tags
def test_is_realtime_resource_collected_by_filters(realtime_instance): realtime_instance['resource_filters'] = [ {'resource': 'vm', 'property': 'name', 'patterns': [r'^\$VM5$', r'^VM4-2\d$']}, {'resource': 'vm', 'property': 'inventory_path', 'patterns': [u'\\/D\xe4tacenter\\/vm\\/m.*']}, {'resource': 'vm', 'property': 'hostname', 'patterns': [r'10\.0\.0\.103']}, {'resource': 'vm', 'property': 'guest_hostname', 'patterns': [r'ubuntu-test']}, {'resource': 'vm', 'property': 'tag', 'patterns': [r'env:production']}, {'resource': 'host', 'property': 'name', 'patterns': [r'10\.0\.0\.103'], 'type': 'blacklist'}, ] realtime_instance['collect_tags'] = True collected_resources = [ 'VM2-1', '$VM3-2', '$VM5', '10.0.0.101', '10.0.0.102', '10.0.0.104', u'VM1-6ê', 'VM3-1', 'VM4-20', 'migrationTest', ] check = VSphereCheck('vsphere', {}, [realtime_instance]) formatted_filters = check.config.resource_filters infra = MockedAPI(realtime_instance).get_infrastructure() resources = [m for m in infra if m.__class__ in (vim.VirtualMachine, vim.HostSystem)] VM2_1 = next(r for r in resources if infra.get(r).get('name') == 'VM2-1') check.infrastructure_cache.set_all_tags({vim.VirtualMachine: {VM2_1._moId: ['env:production', 'tag:2']}}) for resource in resources: is_collected = infra.get(resource).get('name') in collected_resources assert ( is_resource_collected_by_filters( resource, infra, formatted_filters, check.infrastructure_cache.get_mor_tags(resource) ) == is_collected )
def refresh_infrastructure_cache(self): # type: () -> None """Fetch the complete infrastructure, generate tags for each monitored resources and store all of that into the infrastructure_cache. It also computes the resource `hostname` property to be used when submitting metrics for this mor.""" self.log.debug("Refreshing the infrastructure cache...") t0 = Timer() infrastructure_data = self.api.get_infrastructure() self.gauge( "datadog.vsphere.refresh_infrastructure_cache.time", t0.total(), tags=self.config.base_tags, raw=True, hostname=self._hostname, ) self.log.debug("Infrastructure cache refreshed in %.3f seconds.", t0.total()) self.log.debug("Infrastructure cache: %s", infrastructure_data) all_tags = {} if self.config.should_collect_tags: all_tags = self.collect_tags(infrastructure_data) self.infrastructure_cache.set_all_tags(all_tags) for mor, properties in iteritems(infrastructure_data): if not isinstance(mor, tuple( self.config.collected_resource_types)): # Do nothing for the resource types we do not collect continue if not is_resource_collected_by_filters( mor, infrastructure_data, self.config.resource_filters, self.infrastructure_cache.get_mor_tags(mor)): # The resource does not match the specified whitelist/blacklist patterns. continue mor_name = to_string(properties.get("name", "unknown")) mor_type_str = MOR_TYPE_AS_STRING[type(mor)] hostname = None tags = [] if isinstance(mor, vim.VirtualMachine): power_state = properties.get("runtime.powerState") if power_state != vim.VirtualMachinePowerState.poweredOn: # Skipping because the VM is not powered on # TODO: Sometimes VM are "poweredOn" but "disconnected" and thus have no metrics self.log.debug("Skipping VM %s in state %s", mor_name, to_string(power_state)) continue # Hosts are not considered as parents of the VMs they run, we use the `runtime.host` property # to get the name of the ESXi host runtime_host = properties.get("runtime.host") runtime_host_props = infrastructure_data[ runtime_host] if runtime_host else {} runtime_hostname = to_string( runtime_host_props.get("name", "unknown")) tags.append('vsphere_host:{}'.format(runtime_hostname)) if self.config.use_guest_hostname: hostname = properties.get("guest.hostName", mor_name) else: hostname = mor_name elif isinstance(mor, vim.HostSystem): hostname = mor_name else: tags.append('vsphere_{}:{}'.format(mor_type_str, mor_name)) tags.extend(get_parent_tags_recursively(mor, infrastructure_data)) tags.append('vsphere_type:{}'.format(mor_type_str)) # Attach tags from fetched attributes. tags.extend(properties.get('attributes', [])) mor_payload = {"tags": tags} # type: Dict[str, Any] if hostname: mor_payload['hostname'] = hostname self.infrastructure_cache.set_mor_props(mor, mor_payload)