def agent_worker_function(handle, groupId, groupInfo, status_handle): NUM_ITERATIONS = 5 count = 0 while True: dcgm_agent.dcgmUpdateAllFields(handle, 1) ## Get the current configuration for the group config_values = dcgm_agent.dcgmConfigGet(handle, groupId, dcgm_structs.DCGM_CONFIG_CURRENT_STATE, groupInfo.count, status_handle) ## Since this is a group operation, Check for the status codes if any of the property failed helper_investigate_status(status_handle) dcgm_agent.dcgmStatusClear(status_handle) ## Display current configuration for the group for x in xrange(0,groupInfo.count): print "GPU Id : %d" % (config_values[x].gpuId) print "Ecc Mode : %s" % (convert_value_to_string(config_values[x].mEccMode)) print "Auto Boost : %s" % (convert_value_to_string(config_values[x].mPerfState.autoBoost)) print "Sync Boost : %s" % (convert_value_to_string(config_values[x].mPerfState.autoBoost)) print "Mem Clock : %s" % (convert_value_to_string(config_values[x].mPerfState.minVPState.memClk)) print "SM Clock : %s" % (convert_value_to_string(config_values[x].mPerfState.minVPState.procClk)) print "Power Limit : %s" % (convert_value_to_string(config_values[x].mPowerLimit.val)) print "Compute Mode: %s" % (convert_value_to_string(config_values[x].mComputeMode)) print "\n" count = count + 1 if count == NUM_ITERATIONS: break sleep(2)
def helper_verify_power_value_standalone(handle, groupId, expected_power): """ Helper Method to verify power value """ groupInfo = dcgm_agent.dcgmGroupGetInfo( handle, groupId, dcgm_structs.c_dcgmGroupInfo_version2) status_handle = dcgm_agent.dcgmStatusCreate() config_values = dcgm_agent.dcgmConfigGet( handle, groupId, dcgm_structs.DCGM_CONFIG_CURRENT_STATE, groupInfo.count, status_handle) assert len( config_values) > 0, "Failed to get configuration using dcgmConfigGet" for x in range(0, groupInfo.count): if (config_values[x].mPowerLimit.val != dcgmvalue.DCGM_INT32_NOT_SUPPORTED): assert config_values[x].mPowerLimit.type == dcgm_structs.DCGM_CONFIG_POWER_CAP_INDIVIDUAL, \ "The power limit type for gpuId %d is incorrect. Returned: %d Expected :%d" \ % (x, config_values[x].mPowerLimit.type, dcgm_structs.DCGM_CONFIG_POWER_CAP_INDIVIDUAL) assert config_values[x].mPowerLimit.val == expected_power, "The power limit value for gpuID %d is incorrect. Returned: %d Expected: %d" \ % (x, config_values[x].mPowerLimit.val, expected_power) pass ret = dcgm_agent.dcgmStatusDestroy(status_handle) assert (ret == dcgm_structs.DCGM_ST_OK ), "Failed to remove status handler, error: %s" % ret
def Get(self, configType): status = pydcgm.DcgmStatus() gpuIds = self._dcgmGroup.GetGpuIds() configList = dcgm_agent.dcgmConfigGet(self._dcgmHandle.handle, self._groupId, configType, len(gpuIds), status.handle) #Throw specific errors before return error status.ThrowExceptionOnErrors() return configList
def helper_verify_config_values_standalone(handle, groupId, expected_power, expected_ecc, \ expected_proc_clock, expected_mem_clock, expected_compute_mode, \ expected_sync_boost, expected_auto_boost): """ Helper Method to verify all the values for the current configuration are as expected """ groupInfo = dcgm_agent.dcgmGroupGetInfo( handle, groupId, dcgm_structs.c_dcgmGroupInfo_version2) status_handle = dcgm_agent.dcgmStatusCreate() config_values = dcgm_agent.dcgmConfigGet( handle, groupId, dcgm_structs.DCGM_CONFIG_CURRENT_STATE, groupInfo.count, status_handle) assert len( config_values) > 0, "Failed to get configuration using dcgmConfigGet" for x in xrange(0, groupInfo.count): assert config_values[x].mPowerLimit.type == dcgm_structs.DCGM_CONFIG_POWER_CAP_INDIVIDUAL, \ "The power limit type for gpuId %d is incorrect. Returned: %d Expected :%d" \ % (x, config_values[x].mPowerLimit.type, dcgm_structs.DCGM_CONFIG_POWER_CAP_INDIVIDUAL) assert config_values[x].mPowerLimit.val == expected_power, "The power limit value for gpuID %d is incorrect. Returned: %d Expected: %d" \ % (x, config_values[x].mPowerLimit.val, expected_power) assert config_values[x].mPerfState.syncBoost == expected_sync_boost, "The syncboost value for gpuID %d is incorrect."\ " Returned: %d Expected: %d" \ % (x, config_values[x].mPerfState.syncBoost, expected_sync_boost) assert config_values[x].mPerfState.autoBoost == expected_auto_boost, "The autoboost value for gpuID %d is incorrect."\ " Returned: %d Expected: %d" \ % (x, config_values[x].mPerfState.autoBoost, expected_auto_boost) assert config_values[x].mPerfState.minVPState.memClk == expected_mem_clock, "The min mem clock value for gpuID %d is incorrect."\ " Returned: %d Expected: %d" \ % (x, config_values.mPerfState.minVPState.memClk , expected_mem_clock) assert config_values[x].mPerfState.minVPState.procClk == expected_proc_clock, "The min proc clock value for gpuID %d is incorrect."\ " Returned: %d Expected: %d" \ % (x, config_values[x].mPerfState.minVPState.procClk , expected_proc_clock) assert config_values[x].mComputeMode == expected_compute_mode, "The compute mode value for gpuID %d is incorrect."\ " Returned: %d Expected: %d" \ % (x, config_values[x].mComputeMode, expected_compute_mode) assert config_values[x].mEccMode == expected_ecc, "The ecc mode value for gpuID %d is incorrect."\ " Returned: %d Expected: %d" \ % (x, config_values[x].mEccMode, expected_ecc) pass ret = dcgm_agent.dcgmStatusDestroy(status_handle) assert (ret == dcgm_structs.DCGM_ST_OK ), "Failed to remove status handler, error: %s" % ret
autoBoost_set = 1 print "configure autobost" assert attributesForDevices[ 0].vpStates.count > 0, "Can't find clocks for the device" total_clocks = attributesForDevices[0].vpStates.count proc_clk_set = attributesForDevices[0].vpStates.vpState[total_clocks / 2].procClk mem_clk_set = attributesForDevices[0].vpStates.vpState[total_clocks / 2].memClk ## Always Switch the ecc mode ecc_set = 1 groupInfo = dcgm_agent.dcgmGroupGetInfo(handle, groupId, dcgm_structs.c_dcgmGroupInfo_version2) config_values = dcgm_agent.dcgmConfigGet( handle, groupId, dcgm_structs.DCGM_CONFIG_CURRENT_STATE, groupInfo.count, 0) assert len(config_values) > 0, "Failed to work with NULL status handle" eccmodeOnGroupExisting = config_values[0].mEccMode if eccmodeOnGroupExisting == 0: ecc_set = 1 else: ecc_set = 0 syncboost_set = 1 compute_set = dcgm_structs.DCGM_CONFIG_COMPUTEMODE_DEFAULT config_values = dcgm_structs.c_dcgmDeviceConfig_v1() config_values.mEccMode = ecc_set config_values.mPerfState.syncBoost = syncboost_set
def test_dcgm_configure_ecc_mode(handle, gpuIds): test_utils.skip_test("Skipping this test until bug 200377294 is fixed") groupId = dcgm_agent.dcgmGroupCreate(handle, dcgm_structs.DCGM_GROUP_EMPTY, "test1") validDevice = -1 for x in gpuIds: fvSupported = dcgm_agent_internal.dcgmGetLatestValuesForFields( handle, x, [ dcgm_fields.DCGM_FI_DEV_ECC_CURRENT, ]) if (fvSupported[0].value.i64 != dcgmvalue.DCGM_INT64_NOT_SUPPORTED): validDevice = x break if (validDevice == -1): test_utils.skip_test( "Can only run if at least one GPU with ECC is present") ret = dcgm_agent.dcgmGroupAddDevice(handle, groupId, validDevice) assert (ret == dcgm_structs.DCGM_ST_OK ), "Failed to add a device to the group %d. Return %d" % ( groupId.value, ret) groupInfo = dcgm_agent.dcgmGroupGetInfo(handle, groupId) #Create a status handle status_handle = dcgm_agent.dcgmStatusCreate() ## Get original ECC mode on the device config_values = dcgm_agent.dcgmConfigGet( handle, groupId, dcgm_structs.DCGM_CONFIG_CURRENT_STATE, groupInfo.count, status_handle) assert len( config_values) > 0, "Failed to get configuration using dcgmConfigGet" eccmodeOnGroupExisting = config_values[0].mEccMode if eccmodeOnGroupExisting == 0: eccmodeOnGroupToSet = 1 else: eccmodeOnGroupToSet = 0 #print eccmodeOnGroupExisting #print eccmodeOnGroupToSet ## Toggle the ECC mode on the group config_values = dcgm_structs.c_dcgmDeviceConfig_v1() config_values.mEccMode = eccmodeOnGroupToSet config_values.mPerfState.syncBoost = dcgmvalue.DCGM_INT32_BLANK config_values.mPerfState.targetClocks.memClock = dcgmvalue.DCGM_INT32_BLANK config_values.mPerfState.targetClocks.smClock = dcgmvalue.DCGM_INT32_BLANK config_values.mComputeMode = dcgmvalue.DCGM_INT32_BLANK config_values.mPowerLimit.type = dcgmvalue.DCGM_INT32_BLANK config_values.mPowerLimit.val = dcgmvalue.DCGM_INT32_BLANK #Clear the status handle to log the errors while setting the config ret = dcgm_agent.dcgmStatusClear(status_handle) assert ret == dcgm_structs.DCGM_ST_OK, "Failed to clear the status handle. Return %d" % ret try: ret = dcgm_agent.dcgmConfigSet(handle, groupId, config_values, status_handle) except dcgm_structs.DCGMError as e: pass errors = helper_get_status_list(status_handle) if len(errors) > 0: for error in errors: if error.status == dcgm_structs.DCGM_ST_RESET_REQUIRED: test_utils.skip_test( "Skipping the test - Unable to reset the Gpu, FieldId - %d, Return - %d" % (error.fieldId, error.status)) else: test_utils.skip_test( "Skipping the test - Unable to set the ECC mode. FieldId - %d, Return %d" % (error.fieldId, error.status)) #Sleep after reset time.sleep(2) #Clear the status handle to log the errors while setting the config ret = dcgm_agent.dcgmStatusClear(status_handle) assert ret == dcgm_structs.DCGM_ST_OK, "Failed to clear the status handle. Return %d" % ret #Get the current configuration config_values = dcgm_agent.dcgmConfigGet( handle, groupId, dcgm_structs.DCGM_CONFIG_CURRENT_STATE, groupInfo.count, status_handle) assert len( config_values) > 0, "Failed to get configuration using dcgmConfigGet" fvs = dcgm_agent_internal.dcgmGetLatestValuesForFields( handle, validDevice, [ dcgm_fields.DCGM_FI_DEV_ECC_PENDING, dcgm_fields.DCGM_FI_DEV_ECC_CURRENT ]) if fvs[0].value.i64 != fvs[1].value.i64: logger.warning( "Pending ECC %d != Current ECC %d for gpuId %d. Box probably needs a reboot" % (fvs[0].value.i64, fvs[1].value.i64, validDevice)) else: assert config_values[0].mEccMode == (eccmodeOnGroupToSet), "ECC mode %d different from the set value %d" % \ (config_values[0].mEccMode, eccmodeOnGroupToSet)