def __init__(self, gpus, frequency, metrics, dcgmPath=None):
"""
Parameters
----------
gpus : list of GPUDevice
The gpus to be monitored
frequency : int
Sampling frequency for the metric
metrics : list
List of Record types to monitor
dcgmPath : str (optional)
DCGM installation path
"""
super().__init__(frequency, metrics)
structs._dcgmInit(dcgmPath)
dcgm_agent.dcgmInit()
self._gpus = gpus
# Start DCGM in the embedded mode to use the shared library
self.dcgm_handle = dcgm_handle = dcgm_agent.dcgmStartEmbedded(
structs.DCGM_OPERATION_MODE_MANUAL)
# Create DCGM monitor group
self.group_id = dcgm_agent.dcgmGroupCreate(dcgm_handle,
structs.DCGM_GROUP_EMPTY,
"triton-monitor")
# Add the GPUs to the group
for gpu in self._gpus:
dcgm_agent.dcgmGroupAddDevice(dcgm_handle, self.group_id,
gpu.device_id())
frequency = int(self._frequency * 1000)
fields = []
try:
for metric in metrics:
fields.append(self.model_analyzer_to_dcgm_field[metric])
except KeyError:
dcgm_agent.dcgmShutdown()
raise TritonModelAnalyzerException(
f'{metric} is not supported by Model Analyzer DCGM Monitor')
self.dcgm_field_group_id = dcgm_agent.dcgmFieldGroupCreate(
dcgm_handle, fields, 'triton-monitor')
self.group_watcher = dcgm_field_helpers.DcgmFieldGroupWatcher(
dcgm_handle, self.group_id, self.dcgm_field_group_id.value,
structs.DCGM_OPERATION_MODE_MANUAL, frequency, 3600, 0, 0)
def __init__(self, frequency, tags, dcgmPath=None):
"""
Parameters
----------
frequency : int
Sampling frequency for the metric
tags : list
List of Record types to monitor
dcgmPath : str (optional)
DCGM installation path
"""
super().__init__(frequency, tags)
structs._dcgmInit(dcgmPath)
dcgm_agent.dcgmInit()
# Start DCGM in the embedded mode to use the shared library
self.dcgm_handle = dcgm_handle = dcgm_agent.dcgmStartEmbedded(
structs.DCGM_OPERATION_MODE_MANUAL)
# Create DCGM monitor group
self.group_id = dcgm_agent.dcgmGroupCreate(dcgm_handle,
structs.DCGM_GROUP_EMPTY,
"triton-monitor")
# Add the GPUs to the group
for gpu in self._gpus:
dcgm_agent.dcgmGroupAddDevice(dcgm_handle, self.group_id,
gpu.device_id())
frequency = int(self._frequency * 1000)
fields = []
for tag in tags:
if tag in self.MODEL_ANALYZER_TO_DCGM_FIELD:
dcgm_field = self.MODEL_ANALYZER_TO_DCGM_FIELD[tag]
fields.append(dcgm_field)
else:
raise TritonModelAnalyzerException(
f'{tag} is not supported by Model Analyzer DCGM Monitor')
self.dcgm_field_group_id = dcgm_agent.dcgmFieldGroupCreate(
dcgm_handle, fields, 'triton-monitor')
self.group_watcher = dcgm_field_helpers.DcgmFieldGroupWatcher(
dcgm_handle, self.group_id, self.dcgm_field_group_id.value,
structs.DCGM_OPERATION_MODE_MANUAL, frequency, 3600, 0, 0)