def _cache_morlist_process_atomic(self, instance, mor): t = Timer() i_key = self._instance_key(instance) server_instance = self._get_server_instance(instance) perfManager = server_instance.content.perfManager self.log.debug( "job_atomic: Querying available metrics" " for MOR {0} (type={1})".format(mor['mor'], mor['mor_type']) ) available_metrics = perfManager.QueryAvailablePerfMetric( mor['mor'], intervalId=REAL_TIME_INTERVAL) mor['metrics'] = self._compute_needed_metrics(instance, available_metrics) mor_name = str(mor['mor']) if mor_name in self.morlist[i_key]: self.morlist[i_key][mor_name]['metrics'] = mor['metrics'] else: self.morlist[i_key][mor_name] = mor self.morlist[i_key][mor_name]['last_seen'] = time.time() self.histogram('datamonitor.agent.vsphere.morlist_process_atomic.time', t.total())
def _cache_metrics_metadata(self, instance): """ Get from the server instance, all the performance counters metadata meaning name/group/description... attached with the corresponding ID """ ### <TEST-INSTRUMENTATION> t = Timer() ### </TEST-INSTRUMENTATION> i_key = self._instance_key(instance) self.log.info( "Warming metrics metadata cache for instance {0}".format(i_key)) server_instance = self._get_server_instance(instance) perfManager = server_instance.content.perfManager new_metadata = {} for counter in perfManager.perfCounter: d = dict( name="%s.%s" % (counter.groupInfo.key, counter.nameInfo.key), unit=counter.unitInfo.key, instance_tag='instance' # FIXME: replace by what we want to tag! ) new_metadata[counter.key] = d self.cache_times[i_key][METRICS_METADATA][LAST] = time.time() self.log.info( "Finished metadata collection for instance {0}".format(i_key)) # Reset metadata self.metrics_metadata[i_key] = new_metadata ### <TEST-INSTRUMENTATION> self.histogram('datadog.agent.vsphere.metric_metadata_collection.time', t.total())
def _cache_morlist_process_atomic(self, instance, mor): """ Process one item of the self.morlist_raw list by querying the available metrics for this MOR and then putting it in self.morlist """ ### <TEST-INSTRUMENTATION> t = Timer() ### </TEST-INSTRUMENTATION> i_key = self._instance_key(instance) server_instance = self._get_server_instance(instance) perfManager = server_instance.content.perfManager self.log.debug( "job_atomic: Querying available metrics" " for MOR {0} (type={1})".format(mor['mor'], mor['mor_type']) ) available_metrics = perfManager.QueryAvailablePerfMetric( mor['mor'], intervalId=REAL_TIME_INTERVAL) mor['metrics'] = self._compute_needed_metrics(instance, available_metrics) mor_name = str(mor['mor']) if mor_name in self.morlist[i_key]: # Was already here last iteration self.morlist[i_key][mor_name]['metrics'] = mor['metrics'] else: self.morlist[i_key][mor_name] = mor self.morlist[i_key][mor_name]['last_seen'] = time.time() ### <TEST-INSTRUMENTATION> self.histogram('datadog.agent.vsphere.morlist_process_atomic.time', t.total())
def _cache_morlist_process_atomic(self, instance, mor): """ Process one item of the self.morlist_raw list by querying the available metrics for this MOR and then putting it in self.morlist """ ### <TEST-INSTRUMENTATION> t = Timer() ### </TEST-INSTRUMENTATION> i_key = self._instance_key(instance) server_instance = self._get_server_instance(instance) perfManager = server_instance.content.perfManager self.log.debug("job_atomic: Querying available metrics" " for MOR {0} (type={1})".format( mor['mor'], mor['mor_type'])) available_metrics = perfManager.QueryAvailablePerfMetric( mor['mor'], intervalId=REAL_TIME_INTERVAL) mor['metrics'] = self._compute_needed_metrics(instance, available_metrics) mor_name = str(mor['mor']) if mor_name in self.morlist[i_key]: # Was already here last iteration self.morlist[i_key][mor_name]['metrics'] = mor['metrics'] else: self.morlist[i_key][mor_name] = mor self.morlist[i_key][mor_name]['last_seen'] = time.time() ### <TEST-INSTRUMENTATION> self.histogram('datadog.agent.vsphere.morlist_process_atomic.time', t.total())
def _cache_metrics_metadata(self, instance): """ Get from the server instance, all the performance counters metadata meaning name/group/description... attached with the corresponding ID """ ### <TEST-INSTRUMENTATION> t = Timer() ### </TEST-INSTRUMENTATION> i_key = self._instance_key(instance) self.log.info("Warming metrics metadata cache for instance {0}".format(i_key)) server_instance = self._get_server_instance(instance) perfManager = server_instance.content.perfManager new_metadata = {} for counter in perfManager.perfCounter: d = dict( name = "%s.%s" % (counter.groupInfo.key, counter.nameInfo.key), unit = counter.unitInfo.key, instance_tag = 'instance' # FIXME: replace by what we want to tag! ) new_metadata[counter.key] = d self.cache_times[i_key][METRICS_METADATA][LAST] = time.time() self.log.info("Finished metadata collection for instance {0}".format(i_key)) # Reset metadata self.metrics_metadata[i_key] = new_metadata ### <TEST-INSTRUMENTATION> self.histogram('datadog.agent.vsphere.metric_metadata_collection.time', t.total())
def _collect_metrics_atomic(self, instance, mor): """ Task that collects the metrics listed in the morlist for one MOR """ ### <TEST-INSTRUMENTATION> t = Timer() ### </TEST-INSTRUMENTATION> i_key = self._instance_key(instance) server_instance = self._get_server_instance(instance) perfManager = server_instance.content.perfManager query = vim.PerformanceManager.QuerySpec(maxSample=1, entity=mor['mor'], metricId=mor['metrics'], intervalId=20, format='normal') results = perfManager.QueryPerf(querySpec=[query]) if results: for result in results[0].value: if result.id.counterId not in self.metrics_metadata[i_key]: self.log.debug("Skipping this metric value, because there is no metadata about it") continue instance_name = result.id.instance or "none" value = self._transform_value(instance, result.id.counterId, result.value[0]) self.gauge("vsphere.%s" % self.metrics_metadata[i_key][result.id.counterId]['name'], value, hostname=mor['hostname'], tags=['instance:%s' % instance_name] ) ### <TEST-INSTRUMENTATION> self.histogram('datadog.agent.vsphere.metric_colection.time', t.total())
def _collect_metrics_atomic(self, instance, mor): """ Task that collects the metrics listed in the morlist for one MOR """ ### <TEST-INSTRUMENTATION> t = Timer() ### </TEST-INSTRUMENTATION> i_key = self._instance_key(instance) server_instance = self._get_server_instance(instance) perfManager = server_instance.content.perfManager query = vim.PerformanceManager.QuerySpec(maxSample=1, entity=mor['mor'], metricId=mor['metrics'], intervalId=20, format='normal') results = perfManager.QueryPerf(querySpec=[query]) if results: for result in results[0].value: if result.id.counterId not in self.metrics_metadata[i_key]: self.log.debug("Skipping this metric value, because there is no metadata about it") continue instance_name = result.id.instance or "none" value = self._transform_value(instance, result.id.counterId, result.value[0]) # Metric types are absolute, delta, and rate if ALL_METRICS[self.metrics_metadata[i_key][result.id.counterId]['name']]['s_type'] == 'rate': record_metric = self.rate else: record_metric = self.gauge ip = "unknown" content = server_instance.RetrieveContent() for child in content.rootFolder.childEntity: if hasattr(child, 'vmFolder'): datacenter = child vmFolder = datacenter.vmFolder vmList = vmFolder.childEntity for vm in vmList: if isinstance(vm, vim.VirtualMachine): ip = vm.summary.guest.ipAddress self.log.info("Get VM ip {} by VMtools".format(ip)) if ip != "unknown" and ip != "None": record_metric( "vsphere.%s" % self.metrics_metadata[i_key][result.id.counterId]['name'], value, hostname=mor['hostname'], tags=['instance:%s' % instance_name, 'ip:%s' % ip, 'type:VM'] ) else: record_metric( "vsphere.%s" % self.metrics_metadata[i_key][result.id.counterId]['name'], value, hostname=mor['hostname'], tags=['instance:%s' % instance_name, "type:VM"] ) ### <TEST-INSTRUMENTATION> self.histogram('datamonitor.agent.vsphere.metric_colection.time', t.total())
def _collect_metrics_atomic(self, instance, mor): """ Task that collects the metrics listed in the morlist for one MOR """ ### <TEST-INSTRUMENTATION> t = Timer() ### </TEST-INSTRUMENTATION> i_key = self._instance_key(instance) server_instance = self._get_server_instance(instance) perfManager = server_instance.content.perfManager query = vim.PerformanceManager.QuerySpec(maxSample=1, entity=mor['mor'], metricId=mor['metrics'], intervalId=mor['interval'], format='normal') results = perfManager.QueryPerf(querySpec=[query]) if results: for result in results[0].value: if result.id.counterId not in self.metrics_metadata[i_key]: self.log.debug( "Skipping this metric value, because there is no metadata about it" ) continue instance_name = result.id.instance or "none" value = self._transform_value(instance, result.id.counterId, result.value[0]) # Metric types are absolute, delta, and rate metric_name = self.metrics_metadata[i_key][ result.id.counterId]['name'] if metric_name not in ALL_METRICS: self.log.debug(u"Skipping unknown `%s` metric.", metric_name) continue tags = ['instance:%s' % instance_name] if not mor['hostname']: # no host tags available tags.extend(mor['tags']) # vsphere "rates" should be submitted as gauges (rate is # precomputed). self.gauge("vsphere.%s" % metric_name, value, hostname=mor['hostname'], tags=['instance:%s' % instance_name]) ### <TEST-INSTRUMENTATION> self.histogram('datadog.agent.vsphere.metric_colection.time', t.total())
def _collect_metrics_atomic(self, instance, mor): """ Task that collects the metrics listed in the morlist for one MOR """ ### <TEST-INSTRUMENTATION> t = Timer() ### </TEST-INSTRUMENTATION> i_key = self._instance_key(instance) server_instance = self._get_server_instance(instance) perfManager = server_instance.content.perfManager query = vim.PerformanceManager.QuerySpec(maxSample=1, entity=mor['mor'], metricId=mor['metrics'], intervalId=mor['interval'], format='normal') results = perfManager.QueryPerf(querySpec=[query]) if results: for result in results[0].value: if result.id.counterId not in self.metrics_metadata[i_key]: self.log.debug("Skipping this metric value, because there is no metadata about it") continue instance_name = result.id.instance or "none" value = self._transform_value(instance, result.id.counterId, result.value[0]) # Metric types are absolute, delta, and rate metric_name = self.metrics_metadata[i_key][result.id.counterId]['name'] if metric_name not in ALL_METRICS: self.log.debug(u"Skipping unknown `%s` metric.", metric_name) continue tags = ['instance:%s' % instance_name] if not mor['hostname']: # no host tags available tags.extend(mor['tags']) # vsphere "rates" should be submitted as gauges (rate is # precomputed). self.gauge( "vsphere.%s" % metric_name, value, hostname=mor['hostname'], tags=['instance:%s' % instance_name] ) ### <TEST-INSTRUMENTATION> self.histogram('datadog.agent.vsphere.metric_colection.time', t.total())
def _cache_metrics_metadata(self, instance): t = Timer() i_key = self._instance_key(instance) self.log.info("Warming metrics metadata cache for instance {0}".format(i_key)) server_instance = self._get_server_instance(instance) perfManager = server_instance.content.perfManager new_metadata = {} for counter in perfManager.perfCounter: d = dict( name="%s.%s" % (counter.groupInfo.key, counter.nameInfo.key), unit=counter.unitInfo.key, instance_tag='instance' ) new_metadata[counter.key] = d self.cache_times[i_key][METRICS_METADATA][LAST] = time.time() self.log.info("Finished metadata collection for instance {0}".format(i_key)) self.metrics_metadata[i_key] = new_metadata self.histogram('datamonitor.agent.vsphere.metric_metadata_collection.time', t.total())
def _cache_morlist_raw_atomic(self, i_key, obj_type, obj, tags, regexes=None): t = Timer() self.log.debug("job_atomic: Exploring MOR {0} (type={1})".format(obj, obj_type)) tags_copy = deepcopy(tags) if obj_type == 'rootFolder': for datacenter in obj.childEntity: if not hasattr(datacenter, 'hostFolder'): continue self.pool.apply_async( self._cache_morlist_raw_atomic, args=(i_key, 'datacenter', datacenter, tags_copy, regexes) ) elif obj_type == 'datacenter': dc_tag = "vsphere_datacenter:%s" % obj.name tags_copy.append(dc_tag) for compute_resource in obj.hostFolder.childEntity: if not hasattr(compute_resource, 'host'): continue self.pool.apply_async( self._cache_morlist_raw_atomic, args=(i_key, 'compute_resource', compute_resource, tags_copy, regexes) ) elif obj_type == 'compute_resource': if obj.__class__ == vim.ClusterComputeResource: cluster_tag = "vsphere_cluster:%s" % obj.name tags_copy.append(cluster_tag) for host in obj.host: if not hasattr(host, 'vm'): continue self.pool.apply_async( self._cache_morlist_raw_atomic, args=(i_key, 'host', host, tags_copy, regexes) ) elif obj_type == 'host': if regexes and regexes.get('host_include') is not None: match = re.search(regexes['host_include'], obj.name) if not match: self.log.debug(u"Filtered out VM {0} because of host_include_only_regex".format(obj.name)) return watched_mor = dict(mor_type='host', mor=obj, hostname=obj.name, tags=tags_copy + ['vsphere_type:host']) self.morlist_raw[i_key].append(watched_mor) host_tag = "vsphere_host:%s" % obj.name tags_copy.append(host_tag) for vm in obj.vm: if vm.runtime.powerState != 'poweredOn': continue self.pool.apply_async( self._cache_morlist_raw_atomic, args=(i_key, 'vm', vm, tags_copy, regexes) ) elif obj_type == 'vm': if regexes and regexes.get('vm_include') is not None: match = re.search(regexes['vm_include'], obj.name) if not match: self.log.debug(u"Filtered out VM {0} because of vm_include_only_regex".format(obj.name)) return watched_mor = dict(mor_type='vm', mor=obj, hostname=obj.name, tags=tags_copy + ['vsphere_type:vm']) self.morlist_raw[i_key].append(watched_mor) self.histogram('datamonitor.agent.vsphere.morlist_raw_atomic.time', t.total())
def _cache_morlist_raw_atomic(self, i_key, obj_type, obj, tags, regexes=None, include_only_marked=False): """ Compute tags for a single node in the vCenter rootFolder and queue other such jobs for children nodes. Usual hierarchy: rootFolder - datacenter1 - compute_resource1 == cluster - host1 - host2 - host3 - compute_resource2 - host5 - vm1 - vm2 If it's a node we want to query metric for, queue it in self.morlist_raw that will be processed by another job. """ ### <TEST-INSTRUMENTATION> t = Timer() self.log.debug("job_atomic: Exploring MOR {0} (type={1})".format( obj, obj_type)) ### </TEST-INSTRUMENTATION> tags_copy = deepcopy(tags) if obj_type == 'rootFolder': for datacenter in obj.childEntity: # Skip non-datacenter if not hasattr(datacenter, 'hostFolder'): continue self.pool.apply_async(self._cache_morlist_raw_atomic, args=(i_key, 'datacenter', datacenter, tags_copy, regexes, include_only_marked)) elif obj_type == 'datacenter': dc_tag = "vsphere_datacenter:%s" % obj.name tags_copy.append(dc_tag) for compute_resource in obj.hostFolder.childEntity: # Skip non-compute resource if not hasattr(compute_resource, 'host'): continue self.pool.apply_async(self._cache_morlist_raw_atomic, args=(i_key, 'compute_resource', compute_resource, tags_copy, regexes, include_only_marked)) elif obj_type == 'compute_resource': if obj.__class__ == vim.ClusterComputeResource: cluster_tag = "vsphere_cluster:%s" % obj.name tags_copy.append(cluster_tag) for host in obj.host: # Skip non-host if not hasattr(host, 'vm'): continue self.pool.apply_async(self._cache_morlist_raw_atomic, args=(i_key, 'host', host, tags_copy, regexes, include_only_marked)) elif obj_type == 'host': if regexes and regexes.get('host_include') is not None: match = re.search(regexes['host_include'], obj.name) if not match: self.log.debug( u"Filtered out VM {0} because of host_include_only_regex" .format(obj.name)) return watched_mor = dict(mor_type='host', mor=obj, hostname=obj.name, tags=tags_copy + ['vsphere_type:host']) self.morlist_raw[i_key].append(watched_mor) host_tag = "vsphere_host:%s" % obj.name tags_copy.append(host_tag) for vm in obj.vm: if vm.runtime.powerState != 'poweredOn': continue self.pool.apply_async(self._cache_morlist_raw_atomic, args=(i_key, 'vm', vm, tags_copy, regexes, include_only_marked)) elif obj_type == 'vm': if regexes and regexes.get('vm_include') is not None: match = re.search(regexes['vm_include'], obj.name) if not match: self.log.debug( u"Filtered out VM {0} because of vm_include_only_regex" .format(obj.name)) return # Also, if include_only_marked is true, then check if there exists a # custom field with the value DatadogMonitored if include_only_marked: monitored = False for field in obj.customValue: if field.value == VM_MONITORING_FLAG: monitored = True break # we shall monitor if not monitored: self.log.debug( u"Filtered out VM {0} because of include_only_marked". format(obj.name)) return watched_mor = dict(mor_type='vm', mor=obj, hostname=obj.name, tags=tags_copy + ['vsphere_type:vm']) self.morlist_raw[i_key].append(watched_mor) ### <TEST-INSTRUMENTATION> self.histogram('datadog.agent.vsphere.morlist_raw_atomic.time', t.total())
def _cache_morlist_raw_atomic(self, i_key, obj_type, obj, tags, regexes=None): """ Compute tags for a single node in the vCenter rootFolder and queue other such jobs for children nodes. Usual hierarchy: rootFolder - datacenter1 - compute_resource1 == cluster - host1 - host2 - host3 - compute_resource2 - host5 - vm1 - vm2 If it's a node we want to query metric for, queue it in self.morlist_raw that will be processed by another job. """ ### <TEST-INSTRUMENTATION> t = Timer() self.log.debug("job_atomic: Exploring MOR {0} (type={1})".format(obj, obj_type)) ### </TEST-INSTRUMENTATION> tags_copy = deepcopy(tags) if obj_type == 'rootFolder': for datacenter in obj.childEntity: # Skip non-datacenter if not hasattr(datacenter, 'hostFolder'): continue self.pool.apply_async( self._cache_morlist_raw_atomic, args=(i_key, 'datacenter', datacenter, tags_copy, regexes) ) elif obj_type == 'datacenter': dc_tag = "vsphere_datacenter:%s" % obj.name tags_copy.append(dc_tag) for compute_resource in obj.hostFolder.childEntity: # Skip non-compute resource if not hasattr(compute_resource, 'host'): continue self.pool.apply_async( self._cache_morlist_raw_atomic, args=(i_key, 'compute_resource', compute_resource, tags_copy, regexes) ) elif obj_type == 'compute_resource': if obj.__class__ == vim.ClusterComputeResource: cluster_tag = "vsphere_cluster:%s" % obj.name tags_copy.append(cluster_tag) for host in obj.host: # Skip non-host if not hasattr(host, 'vm'): continue self.pool.apply_async( self._cache_morlist_raw_atomic, args=(i_key, 'host', host, tags_copy, regexes) ) elif obj_type == 'host': if regexes and regexes.get('host_include') is not None: match = re.search(regexes['host_include'], obj.name) if not match: self.log.debug(u"Filtered out VM {0} because of host_include_only_regex".format(obj.name)) return watched_mor = dict(mor_type='host', mor=obj, hostname=obj.name, tags=tags_copy+['vsphere_type:host']) self.morlist_raw[i_key].append(watched_mor) host_tag = "vsphere_host:%s" % obj.name tags_copy.append(host_tag) for vm in obj.vm: if vm.runtime.powerState != 'poweredOn': continue self.pool.apply_async( self._cache_morlist_raw_atomic, args=(i_key, 'vm', vm, tags_copy, regexes) ) elif obj_type == 'vm': if regexes and regexes.get('vm_include') is not None: match = re.search(regexes['vm_include'], obj.name) if not match: self.log.debug(u"Filtered out VM {0} because of vm_include_only_regex".format(obj.name)) return watched_mor = dict(mor_type='vm', mor=obj, hostname=obj.name, tags=tags_copy+['vsphere_type:vm']) self.morlist_raw[i_key].append(watched_mor) ### <TEST-INSTRUMENTATION> self.histogram('datadog.agent.vsphere.morlist_raw_atomic.time', t.total())
def _cache_morlist_raw_atomic(self, i_key, obj_type, obj, tags): """ Compute tags for a single node in the vCenter rootFolder and queue other such jobs for children nodes. Usual hierarchy: rootFolder - datacenter1 - compute_resource1 == cluster - host1 - host2 - host3 - compute_resource2 - host5 - vm1 - vm2 If it's a node we want to query metric for, queue it in self.morlist_raw that will be processed by another job. """ ### <TEST-INSTRUMENTATION> t = Timer() self.log.debug("job_atomic: Exploring MOR {0} (type={1})".format( obj, obj_type)) ### </TEST-INSTRUMENTATION> tags_copy = deepcopy(tags) if obj_type == 'rootFolder': for datacenter in obj.childEntity: # Skip non-datacenter if not hasattr(datacenter, 'hostFolder'): continue self.pool.apply_async(self._cache_morlist_raw_atomic, args=(i_key, 'datacenter', datacenter, tags_copy)) elif obj_type == 'datacenter': dc_tag = "vsphere_datacenter:%s" % obj.name tags_copy.append(dc_tag) for compute_resource in obj.hostFolder.childEntity: # Skip non-compute resource if not hasattr(compute_resource, 'host'): continue self.pool.apply_async(self._cache_morlist_raw_atomic, args=(i_key, 'compute_resource', compute_resource, tags_copy)) elif obj_type == 'compute_resource': if obj.__class__ == vim.ClusterComputeResource: cluster_tag = "vsphere_cluster:%s" % obj.name tags_copy.append(cluster_tag) for host in obj.host: # Skip non-host if not hasattr(host, 'vm'): continue self.pool.apply_async(self._cache_morlist_raw_atomic, args=(i_key, 'host', host, tags_copy)) elif obj_type == 'host': watched_mor = dict(mor_type='host', mor=obj, hostname=obj.name, tags=tags_copy + ['vsphere_type:host']) self.morlist_raw[i_key].append(watched_mor) host_tag = "vsphere_host:%s" % obj.name tags_copy.append(host_tag) for vm in obj.vm: if vm.runtime.powerState != 'poweredOn': continue self.pool.apply_async(self._cache_morlist_raw_atomic, args=(i_key, 'vm', vm, tags_copy)) elif obj_type == 'vm': watched_mor = dict(mor_type='vm', mor=obj, hostname=obj.name, tags=tags_copy + ['vsphere_type:vm']) self.morlist_raw[i_key].append(watched_mor) ### <TEST-INSTRUMENTATION> self.histogram('datadog.agent.vsphere.morlist_raw_atomic.time', t.total())