def unInstallAgent(Framework): protocolName = Framework.getProperty(InventoryUtils.STATE_PROPERTY_CONNECTED_SHELL_PROTOCOL_NAME) logger.debug('Protocal name: ', protocolName) client = Framework.getConnectedClient() uduid = InventoryUtils.getUduid(client) logger.debug('UD_UNIQUE_ID: ', uduid) Framework.setProperty(InventoryUtils.ATTR_UD_UNIQUE_ID, uduid) if protocolName == ClientsConsts.DDM_AGENT_PROTOCOL_NAME: # Should release lock first if there will be no connected credential after agent uninstallation. logger.debug('The connected credential is UDA. Try to release lock first.') LockUtils.releaseScannerLock(Framework) if AgentUtils.isAgentInstalled(Framework): logger.debug('There is an agent in remote machine.') # Run uninstall command. shouldStop = AgentUtils.agentUnInstallRoutine(Framework) if shouldStop != 0: Framework.setStepExecutionStatus(WorkflowStepStatus.FATAL_FAILURE) logger.debug('Failed to uninstall agent.') else: logger.debug('There is no agent in remote machine. The job will be done.') reason = 'There is no agent in remote machine' Framework.setProperty(InventoryUtils.generateSkipStep('Check Agent UnInstalled'), reason) Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def checkUpgradeStatus(Framework): if Framework.getProperty( InventoryUtils.STATE_PROPERTY_AGENT_INSTALLED) is None: agentId = Framework.getDestinationAttribute('agentId') # in migration job if agentId is None or not len(str(agentId).strip()): Framework.reportError( inventoryerrorcodes. INVENTORY_DISCOVERY_FAILED_AGENT_INSTALL_AFTER_UNINTALL, None) Framework.setStepExecutionStatus(WorkflowStepStatus.FAILURE) return logger.debug( 'Old agent was uninstalled but failed to install new one. Upgrade process failed. Sending to delete UD agent object' ) agentOsh = modeling.createOshByCmdbIdString( ClientsConsts.DDM_AGENT_PROTOCOL_NAME, agentId) Framework.deleteObject(agentOsh) logger.debug("Restore Non UDA Shell since the UDA has been removed.") AgentUtils.reportNonUDAShell(Framework) Framework.flushObjects() Framework.reportError( inventoryerrorcodes. INVENTORY_DISCOVERY_FAILED_AGENT_INSTALL_AFTER_UNINTALL, None) Framework.setStepExecutionStatus(WorkflowStepStatus.FATAL_FAILURE) else: logger.debug('Upgrade UDA is successful.') Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def unInstallAgent(Framework): protocolName = Framework.getProperty( InventoryUtils.STATE_PROPERTY_CONNECTED_SHELL_PROTOCOL_NAME) logger.debug('Protocal name: ', protocolName) client = Framework.getConnectedClient() uduid = InventoryUtils.getUduid(client) logger.debug('UD_UNIQUE_ID: ', uduid) Framework.setProperty(InventoryUtils.ATTR_UD_UNIQUE_ID, uduid) if protocolName == ClientsConsts.DDM_AGENT_PROTOCOL_NAME: # Should release lock first if there will be no connected credential after agent uninstallation. logger.debug( 'The connected credential is UDA. Try to release lock first.') LockUtils.releaseScannerLock(Framework) if AgentUtils.isAgentInstalled(Framework): logger.debug('There is an agent in remote machine.') # Run uninstall command. shouldStop = AgentUtils.agentUnInstallRoutine(Framework) if shouldStop != 0: Framework.setStepExecutionStatus(WorkflowStepStatus.FATAL_FAILURE) logger.debug('Failed to uninstall agent.') else: logger.debug( 'There is no agent in remote machine. The job will be done.') reason = 'There is no agent in remote machine' Framework.setProperty( InventoryUtils.generateSkipStep('Check Agent UnInstalled'), reason) Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def CheckInstallErrorCode(Framework): if AgentUtils.isUpgradeProcess(Framework): if AgentUtils.isUpgradeByUDAgent(Framework): client = Framework.getConnectedClient() agentVersion = client.getVersion() logger.debug("The current agent version is:", agentVersion) connectedUDACredentialId = client.getCredentialId() logger.debug('Credential id on remote:', connectedUDACredentialId) installCredentialId = Framework.getParameter(AgentUtils.UDAGENT_CONNECT_CREDENTIAL_ID_PARAM) logger.debug('Credential id for upgrade:', installCredentialId) if not AgentUtils.versionsEqual(Framework, agentVersion) or (installCredentialId and installCredentialId != connectedUDACredentialId): logger.debug("Notice: The connected client is still old UDA.") errorCode = AgentUtils.getUpgradeErrorCode(Framework) else: errorCode = AgentUtils.getInstallErrorCode(Framework) if errorCode: if errorCode.isSuccess(): Framework.setProperty(CheckAgentInstalled.FIRST_TRY_INSTALL_AGENT, CheckAgentInstalled.FIRST_TRY_INSTALL_AGENT) Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS) elif errorCode.isInProgress(): logger.debug('UDA install command is in progress, will check after parking') Framework.setStepExecutionStatus(WorkflowStepStatus.FAILURE) else: logger.debug('Failed to install UDA according to install error code.') Framework.reportError("Install/Upgrade UDA failed. Reason is:%s" % errorCode.getMessage()) Framework.setStepExecutionStatus(WorkflowStepStatus.FATAL_FAILURE) else: logger.debug('Can not get error code now, will check after parking') Framework.setStepExecutionStatus(WorkflowStepStatus.FAILURE)
def tearDown(self): try: if self.container: self.container.close() # wait a few seconds and then destroy time.sleep(5) if self.container: self.container.destroy() # cleanup the pool if self.pool: self.pool.disconnect() self.pool.destroy(1) if self.hostfile is not None: os.remove(self.hostfile) except DaosApiError as excep: print(excep) print(traceback.format_exc()) self.fail("Test failed during teardown.\n") finally: if self.agent_sessions: AgentUtils.stop_agent(self.hostlist, self.agent_sessions) server_utils.stop_server(hosts=self.hostlist)
def tearDown(self): try: os.remove(self.hostfile) finally: if self.agent_sessions: AgentUtils.stop_agent(self.hostlist, self.agent_sessions) server_utils.stop_server(hosts=self.hostlist)
def connectToRemoteNode(Framework): if AgentUtils.isMigrateNeeded(Framework): #setting connected client identifier #using host name since uduid is stored in agent options and on old and new ddmi agent their location is different logger.debug('Connected using uda.') client = Framework.getConnectedClient() sysInfo = client.getSysInfo() hostName = sysInfo.getProperty('computerName') Framework.setProperty(InventoryUtils.UD_HOSTNAME, hostName) AgentUtils.setUdAgentProtocolForMigration(Framework, client.getCredentialId()) logger.debug('Migrate is going to be performed') if client.hasShell(): logger.debug('The connected Agent already supports shell, assume it is a non-native agent.') reason = 'The connected Agent already supports shell,it may be a non-native agent.' Framework.setProperty(InventoryUtils.generateSkipStep('Install Non-Native UD Agent'), reason) #Framework.setProperty(InventoryUtils.generateSkipStep('Check Non-Native Agent Installed'), reason) platform = Framework.getProperty(InventoryUtils.STATE_PROPERTY_PLATFORM) if platform == 'windows': # In windows, it is native already if it has shell. logger.debug('This is windows, it must be native agent.') Framework.setProperty(AgentUtils.DOWNLOAD_MIGRATE_LOG_FILE, '') reason = 'Native installation is used for Windows platform.' Framework.setProperty(InventoryUtils.generateSkipStep('Init Update from Non-Native to Native'), reason) Framework.setProperty(InventoryUtils.generateSkipStep('Install Native UD Agent'), reason) else: logger.debug('The connected client does NOT support the shell capability. This is DDMi agent!') Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def test_simple_delete(self): """ Test destroying a pool created on a single server, nobody is using the pool, force is not needed. :avocado: tags=pool,pooldestroy,quick """ self.hostlist = self.params.get("test_machines1", '/run/hosts/') hostfile = write_host_file.write_host_file(self.hostlist, self.tmp) self.agent_sessions = AgentUtils.run_agent(self.basepath, self.hostlist) server_utils.run_server(hostfile, self.server_group, self.basepath) setid = self.params.get("setname", '/run/setnames/validsetname/') try: # use the uid/gid of the user running the test, these should # be perfectly valid uid = os.geteuid() gid = os.getegid() # TODO make these params in the yaml daosctl = self.basepath + '/install/bin/daosctl' create_cmd = ('{0} create-pool -m {1} -u {2} -g {3} -s {4}' .format(daosctl, 0x731, uid, gid, setid)) uuid_str = """{0}""".format(process.system_output(create_cmd)) print ("uuid is {0}\n".format(uuid_str)) host = self.hostlist[0] exists = check_for_pool.check_for_pool(host, uuid_str) if exists != 0: self.fail("Pool {0} not found on host {1}.\n" .format(uuid_str, host)) delete_cmd = ('{0} destroy-pool -i {1} -s {2}' .format(daosctl, uuid_str, setid)) process.system(delete_cmd) exists = check_for_pool.check_for_pool(host, uuid_str) if exists == 0: self.fail("Pool {0} found on host {1} when not expected.\n" .format(uuid_str, host)) except Exception as excep: print(excep) print(traceback.format_exc()) self.fail("Expecting to pass but test has failed.\n") # no matter what happens shutdown the server finally: try: os.remove(hostfile) finally: if self.agent_sessions: AgentUtils.stop_agent(self.hostlist, self.agent_sessions) server_utils.stop_server(hosts=self.hostlist)
def tearDown(self): try: if self.agent_sessions: AgentUtils.stop_agent(self.hostlist, self.agent_sessions) server_utils.stop_server(hosts=self.hostlist) finally: # really make sure everything is gone check_for_pool.cleanup_pools(self.hostlist)
def InstallAgentBasicResources(Framework): result = AgentUtils.installAgentBasicResources(Framework) if result: AgentUtils.executeAgentBasicResourcesProcessCommands(Framework) Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS) else: logger.debug("Failed to install basic agent resources.") Framework.setStepExecutionStatus(WorkflowStepStatus.FAILURE)
def tearDown(self): try: if self.pool is not None and self.pool.attached: self.pool.destroy(1) finally: if self.agent_sessions: AgentUtils.stop_agent(self.hostlist, self.agent_sessions) server_utils.stop_server(hosts=self.hostlist)
def tearDown(self): try: if self.container: self.container.close() finally: if self.agent_sessions: AgentUtils.stop_agent(self.hostlist, self.agent_sessions) server_utils.stop_server(hosts=self.hostlist)
def tearDown(self): try: if self.pool is not None: self.pool.destroy(1) finally: if self.agent_sessions: AgentUtils.stop_agent(self.hostlist_clients, self.agent_sessions) server_utils.stop_server(hosts=self.hostlist_servers)
def test_destroy_connect(self): """ Test destroying a pool that has a connected client with force == false. Should fail. :avocado: tags=pool,pooldestroy,x """ host = self.hostlist[0] try: # write out a hostfile and start the servers with it self.hostlist = self.params.get("test_machines1", '/run/hosts/') hostfile = write_host_file.write_host_file(self.hostlist, self.tmp) self.agent_sessions = AgentUtils.run_agent(self.basepath, self.hostlist) server_utils.run_server(hostfile, self.server_group, self.basepath) # parameters used in pool create createmode = self.params.get("mode", '/run/poolparams/createmode/') createuid = self.params.get("uid", '/run/poolparams/createuid/') creategid = self.params.get("gid", '/run/poolparams/creategid/') createsetid = self.params.get("setname", '/run/poolparams/createset/') createsize = self.params.get("size", '/run/poolparams/createsize/') # initialize a python pool object then create the underlying # daos storage pool = DaosPool(self.context) pool.create(createmode, createuid, creategid, createsize, createsetid, None) # need a connection to create container pool.connect(1 << 1) # destroy pool with connection open pool.destroy(0) # should throw an exception and not hit this self.fail("Shouldn't hit this line.\n") except DaosApiError as excep: print("got exception which is expected so long as it is BUSY") print(excep) print(traceback.format_exc()) # pool should still be there exists = check_for_pool.check_for_pool(host, pool.get_uuid_str) if exists != 0: self.fail("Pool gone, but destroy should have failed.\n") # no matter what happens cleanup finally: if self.agent_sessions: AgentUtils.stop_agent(self.hostlist, self.agent_sessions) server_utils.stop_server(hosts=self.hostlist) os.remove(hostfile)
def tearDown(self): try: if self.agent_sessions: AgentUtils.stop_agent(self.hostlist_servers, self.agent_sessions) finally: server_utils.stop_server(hosts=self.hostlist_servers) super(TestWithServers, self).tearDown()
def tearDown(self): try: if self.hostfile is not None: os.remove(self.hostfile) if self.pool: self.pool.destroy(1) finally: if self.agent_sessions: AgentUtils.stop_agent(self.hostlist, self.agent_sessions) server_utils.stop_server(hosts=self.hostlist)
def tearDown(self): try: self.container.close() self.container.destroy() self.pool.disconnect() self.pool.destroy(1) finally: if self.agent_sessions: AgentUtils.stop_agent(self.hostlist, self.agent_sessions) server_utils.stop_server(hosts=self.hostlist)
def tearDown(self): try: if self.pool_connect: self.pool.disconnect() if self.pool: self.pool.destroy(1) finally: if self.agent_sessions: AgentUtils.stop_agent(self.hostlist_clients, self.agent_sessions) server_utils.stop_server(hosts=self.hostlist)
def tearDown(self): """ cleanup after the test """ try: os.remove(self.hostfile) if self.pool: self.pool.destroy(1) finally: if self.agent_sessions: AgentUtils.stop_agent(self.hostlist, self.agent_sessions) server_utils.stop_server(hosts=self.hostlist)
def tearDown(self): # shut 'er down """ wrap pool destroy in a try; in case pool create didn't succeed, we still need the server to be shut down in any case """ try: self.pool.destroy(1) finally: if self.agent_sessions: AgentUtils.stop_agent(self.hostlist, self.agent_sessions) server_utils.stop_server(hosts=self.hostlist)
def tearDown(self): try: if self.hostfile_clients is not None: os.remove(self.hostfile_clients) if self.hostfile_servers is not None: os.remove(self.hostfile_servers) if self.pool is not None and self.pool.attached: self.pool.destroy(1) finally: if self.agent_sessions: AgentUtils.stop_agent(self.hostlist_clients, self.agent_sessions) server_utils.stop_server(hosts=self.hostlist_servers)
def tearDown(self): try: if self.container1 is not None: self.container1.destroy() if self.container2 is not None: self.container2.destroy() if self.pool1 is not None and self.pool1.attached: self.pool1.destroy(1) if self.pool2 is not None and self.pool2.attached: self.pool2.destroy(1) finally: if self.agent_sessions: AgentUtils.stop_agent(self.hostlist, self.agent_sessions) server_utils.stop_server(hosts=self.hostlist)
def test_many_servers(self): """ Test destroy on a large (relative) number of servers. :avocado: tags=pool,pooldestroy,destroybig """ try: # write out a hostfile and start the servers with it self.hostlist = self.params.get("test_machines6", '/run/hosts/') hostfile = write_host_file.write_host_file(self.hostlist, self.tmp) self.agent_sessions = AgentUtils.run_agent(self.basepath, self.hostlist) server_utils.run_server(hostfile, self.server_group, self.basepath) # parameters used in pool create createmode = self.params.get("mode", '/run/poolparams/createmode/') createuid = self.params.get("uid", '/run/poolparams/createuid/') creategid = self.params.get("gid", '/run/poolparams/creategid/') createsetid = self.params.get("setname", '/run/poolparams/createset/') createsize = self.params.get("size", '/run/poolparams/createsize/') # initialize a python pool object then create the underlying # daos storage pool = DaosPool(self.context) pool.create(createmode, createuid, creategid, createsize, createsetid, None) time.sleep(1) # okay, get rid of it pool.destroy(1) except DaosApiError as excep: print(excep) print(traceback.format_exc()) self.fail("6 server test failed.\n") except Exception as excep: self.fail("Daos code segfaulted most likely. Error: %s" % excep) # no matter what happens cleanup finally: if self.agent_sessions: AgentUtils.stop_agent(self.hostlist, self.agent_sessions) server_utils.stop_server(hosts=self.hostlist) os.remove(hostfile)
def setUp(self): self.agent_sessions = None self.hostlist = None # get paths from the build_vars generated by build with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../../../.build_vars.json")) as build_file: build_paths = json.load(build_file) self.basepath = os.path.normpath(build_paths['PREFIX'] + "/../") self.hostlist = self.params.get("test_machines", '/run/hosts/') # NULL is causing connect to blow up so skip that test for now uuidlist = self.params.get("uuid", '/run/connecttests/UUID/*/') connectuuid = uuidlist[0] if connectuuid == 'NULLPTR': self.cancel("skipping null pointer test until DAOS-1781 is fixed") # launch the server self.hostfile = write_host_file.write_host_file(self.hostlist, self.workdir) server_group = self.params.get("name", '/server_config/', 'daos_server') self.agent_sessions = AgentUtils.run_agent(self.basepath, self.hostlist) server_utils.run_server(self.hostfile, server_group, self.basepath)
def checkUpgradeRequired(Framework): Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS) try: agentVersion = Framework.getDestinationAttribute('version') if AgentUtils.versionsEqual(Framework, agentVersion): logger.debug( 'Installed agent version equals to local installer version, skipping upgrade' ) Framework.setProperty( InventoryUtils.STEP_SKIP_ALL_STEPS_PROPERTY, 'Upgrade not required, installed agent version equals to the local installer version' ) else: logger.debug( 'Installed agent version does not equal to local installer version, Going to execute agent upgrade' ) except: errorMessage = str(sys.exc_info()[1]) logger.debugException( 'Failed to compare agent version with current installer:' + errorMessage) Framework.reportError( inventoryerrorcodes. INVENTORY_DISCOVERY_AGENT_VERSION_COMPARISON_FAILED, [errorMessage]) logger.debug('Going to execute agent upgrade')
def setUp(self): self.agent_sessions = None # get paths from the build_vars generated by build self.hostlist = None with open('../../../.build_vars.json') as build_file: build_paths = json.load(build_file) self.basepath = os.path.normpath(build_paths['PREFIX'] + "/../") self.server_group = self.params.get("name", '/server_config/', 'daos_server') # parameters used in pool create self.createmode = self.params.get("mode", '/run/createtests/createmode/') self.createuid = os.geteuid() self.creategid = os.getegid() self.createsetid = self.params.get("setname", '/run/createtests/createset/') self.createsize = self.params.get("size", '/run/createtests/createsize/') # setup the DAOS python API self.context = DaosContext(build_paths['PREFIX'] + '/lib/') self.pool = None self.container = None # hostlist and logging self.d_log = DaosLog(self.context) self.hostlist = self.params.get("test_machines", '/run/hosts/*') self.hostfile = write_host_file.write_host_file(self.hostlist, self.workdir) self.agent_sessions = AgentUtils.run_agent(self.basepath, self.hostlist) server_utils.run_server(self.hostfile, self.server_group, self.basepath)
def installAgent(Framework): if not AgentUtils.AgentMigrateRoutine(Framework): Framework.reportError( inventoryerrorcodes.INVENTORY_DISCOVERY_FAILED_AGENT_INSTALL, None) Framework.setStepExecutionStatus(WorkflowStepStatus.FATAL_FAILURE) else: Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def checkAgentUnInstalledRoutine(Framework): Framework.setStepExecutionStatus(WorkflowStepStatus.FAILURE) protocolName = Framework.getProperty( InventoryUtils.STATE_PROPERTY_CONNECTED_SHELL_PROTOCOL_NAME) logger.debug('Protocal name: ', protocolName) if protocolName != ClientsConsts.DDM_AGENT_PROTOCOL_NAME: # Wait for 5 secs to let agent to be uninstalled. time.sleep(5) errorCode = AgentUtils.getUninstallErrorCode(Framework) if errorCode: if errorCode.isSuccess(): logger.debug('Agent is uninstalled!!!!') Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS) elif errorCode.isInProgress(): logger.debug( 'Can not get error code from agent. Maybe the UDA is uninstalling. It will check again after parking.' ) else: logger.debug( 'Failed to uninstall agent according to error code.') Framework.setStepExecutionStatus( WorkflowStepStatus.FATAL_FAILURE) Framework.reportError(errorCode.getMessage()) else: logger.debug( 'Can not get error code now. It will check again after parking.' ) else: logger.debug( 'The connected credential is UDA. Maybe the UDA is uninstalling. It will check again after parking.' )
def setUp(self): self.agent_sessions = None self.hostlist = None self.subtest_name = self.params.get("test_name", '/run/daos_tests/Tests/*') self.subtest_name = self.subtest_name.replace(" ", "_") # get paths from the build_vars generated by build with open('../../../.build_vars.json') as build_vars: build_paths = json.load(build_vars) self.basepath = os.path.normpath(build_paths['PREFIX'] + "/../") self.server_group = self.params.get("name", '/server_config/', 'daos_server') self.daos_test = self.basepath + '/install/bin/daos_test' self.orterun = self.basepath + '/install/bin/orterun' self.hostlist = self.params.get("test_machines", '/run/hosts/*') self.hostfile = write_host_file.write_host_file(self.hostlist, self.workdir) logfile_env = os.environ['D_LOG_FILE'] self.log_dir, self.server_log = os.path.split(logfile_env) self.client_log = os.path.join(self.log_dir, self.subtest_name + "_" + CLIENT_LOG) #To generate the seperate client log file self.orterun_env = '-x D_LOG_FILE={}'.format(self.client_log) self.agent_sessions = AgentUtils.run_agent(self.basepath, self.hostlist) server_utils.run_server(self.hostfile, self.server_group, self.basepath)
def setUp(self): self.agent_sessions = None self.hostlist = None self.pool = None # get paths from the build_vars generated by build with open('../../../.build_vars.json') as build_file: build_paths = json.load(build_file) self.basepath = os.path.normpath(build_paths['PREFIX'] + "/../") self.server_group = self.params.get("name", '/server_config/', 'daos_server') # setup the DAOS python API self.context = DaosContext(build_paths['PREFIX'] + '/lib/') self.pool = None self.hostlist = self.params.get("test_machines", '/run/hosts/*') self.hostfile = write_host_file.write_host_file(self.hostlist, self.workdir) print("Host file is: {}".format(self.hostfile)) self.agent_sessions = AgentUtils.run_agent(self.basepath, self.hostlist) server_utils.run_server(self.hostfile, self.server_group, self.basepath) time.sleep(10)
def setUp(self): self.agent_sessions = None with open('../../../.build_vars.json') as json_f: build_paths = json.load(json_f) basepath = os.path.normpath(build_paths['PREFIX'] + "/../") server_group = self.params.get("name", '/server_config/', 'daos_server') self.context = DaosContext(build_paths['PREFIX'] + '/lib/') self.container = None self.hostlist = self.params.get("test_machines", '/run/hosts/*') self.hostfile = write_host_file.write_host_file(self.hostlist, self.workdir) self.agent_sessions = AgentUtils.run_agent(basepath, self.hostlist) server_utils.run_server(self.hostfile, server_group, basepath) self.pool = DaosPool(self.context) self.pool.create(self.params.get("mode", '/run/pool/createmode/*'), os.geteuid(), os.getegid(), self.params.get("size", '/run/pool/createsize/*'), self.params.get("setname", '/run/pool/createset/*'), None) self.pool.connect(1 << 1)
def setUp(self): """ setup for the test """ self.agent_sessions = None # get paths from the build_vars generated by build with open('../../../.build_vars.json') as build_file: build_paths = json.load(build_file) self.context = DaosContext(build_paths['PREFIX'] + '/lib/') self.basepath = os.path.normpath(build_paths['PREFIX'] + "/../") # generate a hostfile self.hostlist = self.params.get("test_machines", '/run/hosts/') self.hostfile = write_host_file.write_host_file(self.hostlist, self.workdir) # fire up the DAOS servers self.server_group = self.params.get("name", '/run/server_config/', 'daos_server') self.agent_sessions = AgentUtils.run_agent(self.basepath, self.hostlist) server_utils.run_server(self.hostfile, self.server_group, build_paths['PREFIX'] + '/../') # create a pool to test with createmode = self.params.get("mode", '/run/pool/createmode/') createuid = self.params.get("uid", '/run/pool/createuid/') creategid = self.params.get("gid", '/run/pool/creategid/') createsetid = self.params.get("setname", '/run/pool/createset/') createsize = self.params.get("size", '/run/pool/createsize/') self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid) self.pool.get_uuid_str() time.sleep(2)
def setUp(self): self.agent_sessions = None # get paths from the build_vars generated by build with open('../../../.build_vars.json') as build_file: build_paths = json.load(build_file) self.basepath = os.path.normpath(build_paths['PREFIX'] + "/../") self.server_group = self.params.get("name", '/server_config/', 'daos_server') # setup the DAOS python API self.context = DaosContext(build_paths['PREFIX'] + '/lib/') self.hostlist_servers = self.params.get("test_servers", '/run/hosts/') self.hostfile_servers = ( write_host_file.write_host_file(self.hostlist_servers, self.workdir)) print("Host file servers is: {}".format(self.hostfile_servers)) self.hostlist_clients = self.params.get("test_clients", '/run/hosts/') self.hostfile_clients = ( write_host_file.write_host_file(self.hostlist_clients, self.workdir)) print("Host file clients is: {}".format(self.hostfile_clients)) # start servers self.agent_sessions = AgentUtils.run_agent(self.basepath, self.hostlist_servers, self.hostlist_clients) server_utils.run_server(self.hostfile_servers, self.server_group, self.basepath) self.mpio = None
def connectToRemoteNode(Framework): Framework.setProperty(InventoryUtils.STATE_PROPERTY_RESOLVED_BASEDIR, Framework.getProperty(InventoryUtils.STATE_PROPERTY_RESOLVED_CONFIGURED_BASEDIR)) Framework.getConnectedClient().setOptionsDirectory(Framework.getProperty(InventoryUtils.STATE_PROPERTY_RESOLVED_BASEDIR)) if AgentUtils.isUpgradeProcess(Framework): # we here means that destination data shows version different from OOTB installer version. # need to recheck with real agent logger.debug('Checking if real version of agent differs from OOTB installer version') agentVersion = None connectedUDACredentialId = None installCredentialId = Framework.getParameter(AgentUtils.UDAGENT_CONNECT_CREDENTIAL_ID_PARAM) logger.debug('Credential id will be used:', installCredentialId) client = Framework.getConnectedClient() if AgentUtils.isUpgradeByUDAgent(Framework): agentVersion = client.getVersion() connectedUDACredentialId = client.getCredentialId() logger.debug('Credential id on remote:', connectedUDACredentialId) AgentUtils.updateCallHomeParams(Framework) AgentUtils.updateSWUtilization(Framework) InventoryUtils.setConnectedClientIdentifier(Framework, client) #Same version and same credential, skip upgrade if AgentUtils.versionsEqual(Framework, agentVersion) and (not installCredentialId or installCredentialId == connectedUDACredentialId): logger.debug('Installed agent version equals to local installer version, skipping upgrade') Framework.setProperty(InventoryUtils.STEP_SKIP_ALL_STEPS_PROPERTY, 'Upgrade not required, real installed agent version equals to the local installer version') client.close() elif Framework.getParameter("UpgradeAgent") == 'false': logger.debug("Upgrade is not required because the job parameter 'UpgradeAgent' is false") Framework.setProperty(InventoryUtils.STEP_SKIP_ALL_STEPS_PROPERTY, "Upgrade is not required because the job parameter 'UpgradeAgent' is false") client.close() else: logger.debug('Installed agent version does not equal to local installer version, Going to execute agent upgrade') Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def setUp(self): # initialising variables self.basepath = None self.server_group = None self.context = None self.pool = None self.mpio = None self.agent_sessions = None self.hostlist_servers = None self.hostfile_servers = None self.hostlist_clients = None self.hostfile_clients = None self.createsvc = None # get paths from the build_vars generated by build with open('../../../.build_vars.json') as var_file: build_paths = json.load(var_file) self.basepath = os.path.normpath(build_paths['PREFIX'] + "/../") self.server_group = self.params.get("name", '/server_config/', 'daos_server') # setup the DAOS python API self.context = DaosContext(build_paths['PREFIX'] + '/lib/') self.hostlist_servers = self.params.get("test_servers", '/run/hosts/') self.hostfile_servers = write_host_file.write_host_file( self.hostlist_servers, self.workdir) print("Host file servers is: {}".format(self.hostfile_servers)) self.hostlist_clients = self.params.get("test_clients", '/run/hosts/') self.hostfile_clients = write_host_file.write_host_file( self.hostlist_clients, self.workdir, None) print("Host file clients is: {}".format(self.hostfile_clients)) self.agent_sessions = AgentUtils.run_agent(self.basepath, self.hostlist_servers, self.hostlist_clients) # start servers server_utils.run_server(self.hostfile_servers, self.server_group, self.basepath) try: # parameters used in pool create createmode = self.params.get("mode", '/run/pool/createmode/*/') createuid = os.geteuid() creategid = os.getegid() createsetid = self.params.get("setname", '/run/pool/createset/') createsize = self.params.get("size", '/run/pool/createsize/') self.createsvc = self.params.get("svcn", '/run/pool/createsvc/') # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None, None, self.createsvc) except (DaosApiError) as excep: self.fail("<Test Failed at pool create> \n{1}".format(excep))
def checkUpgradeStatus(Framework): if Framework.getProperty(InventoryUtils.STATE_PROPERTY_AGENT_INSTALLED) is None: agentId = Framework.getDestinationAttribute('agentId') # in migration job if agentId is None or not len(str(agentId).strip()) : Framework.reportError(inventoryerrorcodes.INVENTORY_DISCOVERY_FAILED_AGENT_INSTALL_AFTER_UNINTALL, None) Framework.setStepExecutionStatus(WorkflowStepStatus.FAILURE) return logger.debug('Old agent was uninstalled but failed to install new one. Upgrade process failed. Sending to delete UD agent object') agentOsh = modeling.createOshByCmdbIdString(ClientsConsts.DDM_AGENT_PROTOCOL_NAME, agentId) Framework.deleteObject(agentOsh) logger.debug("Restore Non UDA Shell since the UDA has been removed.") AgentUtils.reportNonUDAShell(Framework) Framework.flushObjects() Framework.reportError(inventoryerrorcodes.INVENTORY_DISCOVERY_FAILED_AGENT_INSTALL_AFTER_UNINTALL, None) Framework.setStepExecutionStatus(WorkflowStepStatus.FATAL_FAILURE) else: logger.debug('Upgrade UDA is successful.') Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def test_delete_doesnt_exist(self): """ Test destroying a pool uuid that doesn't exist. :avocado: tags=pool,pooldestroy """ self.hostlist = self.params.get("test_machines1", '/run/hosts/') hostfile = write_host_file.write_host_file(self.hostlist, self.tmp) self.agent_sessions = AgentUtils.run_agent(self.basepath, self.hostlist) server_utils.run_server(hostfile, self.server_group, self.basepath) setid = self.params.get("setname", '/run/setnames/validsetname/') host = self.hostlist[0] try: # randomly selected uuid, that is exceptionally unlikely to exist bogus_uuid = '81ef94d7-a59d-4a5e-935b-abfbd12f2105' # TODO make these params in the yaml daosctl = self.basepath + '/install/bin/daosctl' delete_cmd = ('{0} destroy-pool -i {1} -s {2}'.format(daosctl, bogus_uuid, setid)) process.system(delete_cmd) # the above command should fail resulting in an exception so if # we get here the test has failed self.fail("Pool {0} found on host {1} when not expected.\n" .format(bogus_uuid, host)) except Exception as _excep: # expecting an exception so catch and pass the test pass # no matter what happens shutdown the server finally: if self.agent_sessions: AgentUtils.stop_agent(self.hostlist, self.agent_sessions) server_utils.stop_server(hosts=self.hostlist) os.remove(hostfile)
def checkNonNativeAgentInstalled(Framework): # Set the nonnative flags Framework.setProperty(InventoryUtils.STATE_PROPERTY_IS_MIGRATE, 'false') InventoryUtils.resetBaseDir(Framework) AgentUtils.setUpgradingNativeAgent(Framework, 'true') # Ensure we're disconnected InventoryUtils.releaseConnection(Framework) Framework.setProperty(AgentUtils.DOWNLOAD_MIGRATE_LOG_FILE, AgentUtils.DOWNLOAD_MIGRATE_LOG_FILE) # For now - the usual check logger.debug( 'Going to check whether non-native agent already installed or not') warningsList = [] errorsList = [] agent = AgentUtils.agentConnect( Framework, AgentUtils.getUdAgentProtocolForMigration(Framework).getIdAsString(), warningsList, errorsList) if not agent: for errobj in warningsList: logger.reportWarningObject(errobj) for errobj in errorsList: logger.reportErrorObject(errobj) Framework.reportError( inventoryerrorcodes.INVENTORY_DISCOVERY_ENSURE_CONNECTED_FAILED, ['Could not connect to the remote agent']) Framework.setStepExecutionStatus(WorkflowStepStatus.FAILURE) else: logger.debug('Connected to agent!!!!') Framework.setProperty(AgentUtils.DOWNLOAD_MIGRATE_LOG_FILE, '') InventoryUtils.setConnectedClientIdentifier(Framework, agent) agent.close() Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def CheckInstallErrorCode(Framework): if AgentUtils.isUpgradeProcess(Framework): if AgentUtils.isUpgradeByUDAgent(Framework): client = Framework.getConnectedClient() agentVersion = client.getVersion() logger.debug("The current agent version is:", agentVersion) connectedUDACredentialId = client.getCredentialId() logger.debug('Credential id on remote:', connectedUDACredentialId) installCredentialId = Framework.getParameter( AgentUtils.UDAGENT_CONNECT_CREDENTIAL_ID_PARAM) logger.debug('Credential id for upgrade:', installCredentialId) if not AgentUtils.versionsEqual(Framework, agentVersion) or ( installCredentialId and installCredentialId != connectedUDACredentialId): logger.debug("Notice: The connected client is still old UDA.") errorCode = AgentUtils.getUpgradeErrorCode(Framework) else: errorCode = AgentUtils.getInstallErrorCode(Framework) if errorCode: if errorCode.isSuccess(): Framework.setProperty(CheckAgentInstalled.FIRST_TRY_INSTALL_AGENT, CheckAgentInstalled.FIRST_TRY_INSTALL_AGENT) Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS) elif errorCode.isInProgress(): logger.debug( 'UDA install command is in progress, will check after parking') Framework.setStepExecutionStatus(WorkflowStepStatus.FAILURE) else: logger.debug( 'Failed to install UDA according to install error code.') Framework.reportError("Install/Upgrade UDA failed. Reason is:%s" % errorCode.getMessage()) Framework.setStepExecutionStatus(WorkflowStepStatus.FATAL_FAILURE) else: logger.debug('Can not get error code now, will check after parking') Framework.setStepExecutionStatus(WorkflowStepStatus.FAILURE)
def connectToRemoteNode(Framework): Framework.setProperty( InventoryUtils.STATE_PROPERTY_RESOLVED_BASEDIR, Framework.getProperty( InventoryUtils.STATE_PROPERTY_RESOLVED_CONFIGURED_BASEDIR)) Framework.getConnectedClient().setOptionsDirectory( Framework.getProperty(InventoryUtils.STATE_PROPERTY_RESOLVED_BASEDIR)) if AgentUtils.isUpgradeProcess(Framework): # we here means that destination data shows version different from OOTB installer version. # need to recheck with real agent logger.debug( 'Checking if real version of agent differs from OOTB installer version' ) agentVersion = None connectedUDACredentialId = None installCredentialId = Framework.getParameter( AgentUtils.UDAGENT_CONNECT_CREDENTIAL_ID_PARAM) logger.debug('Credential id will be used:', installCredentialId) client = Framework.getConnectedClient() if AgentUtils.isUpgradeByUDAgent(Framework): agentVersion = client.getVersion() connectedUDACredentialId = client.getCredentialId() logger.debug('Credential id on remote:', connectedUDACredentialId) AgentUtils.updateCallHomeParams(Framework) AgentUtils.updateSWUtilization(Framework) InventoryUtils.setConnectedClientIdentifier(Framework, client) #Same version and same credential, skip upgrade if AgentUtils.versionsEqual(Framework, agentVersion) and ( not installCredentialId or installCredentialId == connectedUDACredentialId): logger.debug( 'Installed agent version equals to local installer version, skipping upgrade' ) Framework.setProperty( InventoryUtils.STEP_SKIP_ALL_STEPS_PROPERTY, 'Upgrade not required, real installed agent version equals to the local installer version' ) client.close() elif Framework.getParameter("UpgradeAgent") == 'false': logger.debug( "Upgrade is not required because the job parameter 'UpgradeAgent' is false" ) Framework.setProperty( InventoryUtils.STEP_SKIP_ALL_STEPS_PROPERTY, "Upgrade is not required because the job parameter 'UpgradeAgent' is false" ) client.close() else: logger.debug( 'Installed agent version does not equal to local installer version, Going to execute agent upgrade' ) Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def checkOSVersion(Framework): if AgentUtils.isOSVersionSupported(Framework): Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS) else: logger.debug( 'Could not determine os version to install using the identification output' ) Framework.reportError( inventoryerrorcodes. INVENTORY_DISCOVERY_REMOTE_OS_VERSION_NOT_SUPPORTED, None) Framework.setStepExecutionStatus(WorkflowStepStatus.FAILURE)
def ensureUninstall(Framework): # this is placeholder step - to ensure that uninstall process finished # As uninstall can tale time and just failure in connect does not ensure that # uninstall process already finished we give 2 minutes for uninstall during upgrade process process if AgentUtils.isUpgradeByUDAgent(Framework): Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS) return unintallEnsured = Framework.getProperty(ENSURE_UNINSTALL_PROPERTY) if not unintallEnsured: logger.debug('Going to give chance for agent uninstaller') Framework.setProperty(ENSURE_UNINSTALL_PROPERTY, ENSURE_UNINSTALL_PROPERTY) Framework.setStepExecutionStatus(WorkflowStepStatus.FAILURE) else: Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def CheckAgentInstalledBeforeInstallation(Framework): try: logger.debug('Check if the UDA has already been installed...') agentInstalled = AgentUtils.isAgentInstalled(Framework) logger.debug('Agent installed result:%s' % agentInstalled) # Framework.setProperty('isAgentInstalled', agentInstalled) if agentInstalled: logger.debug('Check if the UDA can be connected...') agentConnected = isAgentConnected(Framework) logger.debug('Agent connecting result:%s' % agentConnected) # Framework.setProperty('isAgentConnected', agentConnected) if agentConnected: reason = 'The UDA has been already installed and can be connected successfully.' Framework.setProperty(InventoryUtils.generateSkipStep('Install Agent'), reason) Framework.setProperty(InventoryUtils.generateSkipStep('Check Agent Install Error Code'), reason) Framework.setProperty(InventoryUtils.generateSkipStep('Check Agent Installed'), reason) else: logger.debug('Agent installed but can not connect, reinstall it.') Framework.setProperty(InventoryUtils.STATE_PROPERTY_IS_UPGRADE, 'true') except: Framework.setStepExecutionStatus(WorkflowStepStatus.FATAL_FAILURE) Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def checkAgentInstalled(Framework, ignoreError=False): # Ensure we're disconnected InventoryUtils.releaseConnection(Framework) Framework.setProperty(AgentUtils.DOWNLOAD_INSTALL_LOG_FILE, AgentUtils.DOWNLOAD_INSTALL_LOG_FILE) if Framework.getProperty(FIRST_TRY_INSTALL_AGENT) is None: # we don't want immediately check whether agent installed or not, since for sure it is not. go to parking # to let others to install logger.debug('UD agent install command just run, will check after parking') Framework.setProperty(FIRST_TRY_INSTALL_AGENT, FIRST_TRY_INSTALL_AGENT) Framework.setStepExecutionStatus(WorkflowStepStatus.FAILURE) return else: logger.debug('Going to check whether agent already installed or not') # errorCode = AgentUtils.getInstallErrorCode(Framework) # if not errorCode.isSuccess(): # logger.debug('Failed to install agent.') # Framework.reportError(inventoryerrorcodes.INVENTORY_DISCOVERY_FAILED_AGENT_INSTALL, None) # Framework.setStepExecutionStatus(WorkflowStepStatus.FATAL_FAILURE) # return warningsList = [] errorsList = [] # When we migrate ddmi to uda, we already know what cred_id to use ddmiMigrationCredId = AgentUtils.getUdAgentProtocolForMigration(Framework) if ddmiMigrationCredId: conToUse = ddmiMigrationCredId.getIdAsString() else: conToUse = None agent = AgentUtils.agentConnect(Framework, conToUse, warningsList, errorsList) if not agent: if not ignoreError: for errobj in warningsList: logger.reportWarningObject(errobj) for errobj in errorsList: logger.reportErrorObject(errobj) Framework.reportError(inventoryerrorcodes.INVENTORY_DISCOVERY_ENSURE_CONNECTED_FAILED, ['Could not connect to the remote agent']) Framework.setStepExecutionStatus(WorkflowStepStatus.FAILURE) else: try: logger.debug('Connected to agent!!!!') # Check whether the agent is native agentsConfigFile = Framework.getConfigFile(CollectorsConstants.AGENTSSBYPLATFORM_FILE_NAME) platform = Framework.getProperty(InventoryUtils.STATE_PROPERTY_PLATFORM) architecture = Framework.getProperty(InventoryUtils.STATE_PROPERTY_ARCHITECTURE) agentPlatformConfig = agentsConfigFile.getPlatformConfiguration(platform, architecture) isNativeCmd = agentPlatformConfig.getIsNativeCmd() logger.debug('Native command is [' + str(isNativeCmd) + ']') if isNativeCmd and len(isNativeCmd) > 0: isNativeCmd = InventoryUtils.handleBaseDirPath(Framework, isNativeCmd) isNative = agent.executeCmd(isNativeCmd) if isNative != 'true': logger.debug('Could not verify whether the remote agent is native') Framework.reportError(inventoryerrorcodes.INVENTORY_DISCOVERY_ENSURE_CONNECTED_FAILED, ['Remote agent doesnt appear to be natively installed']) Framework.setStepExecutionStatus(WorkflowStepStatus.FAILURE) return # Reporting agent osh to framework Framework.setProperty(AgentUtils.DOWNLOAD_INSTALL_LOG_FILE, '') Framework.setProperty(InventoryUtils.STATE_PROPERTY_AGENT_INSTALLED, InventoryUtils.STATE_PROPERTY_AGENT_INSTALLED) AgentUtils.saveGlobalState(agent, Framework) OSHVResult = ObjectStateHolderVector() ip = Framework.getProperty(InventoryUtils.STATE_PROPERTY_CONNECTED_SHELL_IP) hostOsh = modeling.createHostOSH(ip) uduid = InventoryUtils.getUduid(agent) hostOsh.setStringAttribute(InventoryUtils.ATTR_UD_UNIQUE_ID, uduid) agentOsh = AgentUtils.createAgentOsh(agent, Framework) agentOsh.setContainer(hostOsh) OSHVResult.add(hostOsh) OSHVResult.add(agentOsh) Framework.sendObjects(OSHVResult) Framework.flushObjects() Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS) finally: if agent: agent.close()
def RemoveAgentData(Framework): logger.debug('It will remove files under the agent data folder.') AgentUtils.removeAgentFolder(Framework) Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def downloadLogsIfNeeded(Framework): platform = Framework.getProperty(InventoryUtils.STATE_PROPERTY_PLATFORM) logger.debug('Checking if to print install/uninstall logs') downloadMigrateLog = Framework.getProperty( AgentUtils.DOWNLOAD_MIGRATE_LOG_FILE) downloadInstallLog = Framework.getProperty( AgentUtils.DOWNLOAD_INSTALL_LOG_FILE) downloadUnInstallLog = Framework.getProperty( AgentUtils.DOWNLOAD_UNINSTALL_LOG_FILE) if not downloadMigrateLog and not downloadInstallLog and not downloadUnInstallLog: logger.debug('Migrate/Install/UnInstall log should not be downloaded') return try: logger.debug('Releasing old connection') InventoryUtils.releaseConnection(Framework) logger.debug('Preparing framework for new connection') AgentUtils.prepareFrameworkForShellOrAgentConnect(Framework) except: errorMessage = str(sys.exc_info()[1]) logger.debugException( 'Failed to initialize connection for downloading agent log files' + errorMessage) return if downloadMigrateLog: # If need to download migrate log, we need to connect to DDMi agent as well Framework.setProperty(InventoryUtils.STATE_PROPERTY_IS_MIGRATE, str('true')) if not InventoryUtils.ensureConnection(Framework): logger.debug( 'Failed to connect to the remote machine, no logs available') else: ip_address = Framework.getTriggerCIData('ip_address') localInstallFile = None localUnInstallFile = None try: try: agentsConfigFile = Framework.getConfigFile( CollectorsConstants.AGENTSSBYPLATFORM_FILE_NAME) BASEDIR = Framework.getProperty( InventoryUtils.STATE_PROPERTY_RESOLVED_BASEDIR) architecture = Framework.getProperty( InventoryUtils.STATE_PROPERTY_ARCHITECTURE) agentPlatformConfig = agentsConfigFile.getPlatformConfiguration( platform, architecture) ip_address_str = str(ip_address) if (ip_address_str.find(':') <> -1): ip_address_str = ip_address_str.replace(':', '-') if downloadMigrateLog: logger.debug('Download the migrate log') installLogFile = agentPlatformConfig.getUpgradeLogFile() localInstallFile = File( AGENTS_LOGS_TEMP_DIR_FILE, ip_address_str + '-' + installLogFile) getLogFileContent(Framework, localInstallFile, str(BASEDIR) + installLogFile) if downloadInstallLog: logger.debug('Download the install/update log') if AgentUtils.isUpgradeByUDAgent(Framework): installLogFile = agentPlatformConfig.getUpgradeLogFile( ) else: installLogFile = agentPlatformConfig.getInstallLogFile( ) localInstallFile = File( AGENTS_LOGS_TEMP_DIR_FILE, ip_address_str + '-' + installLogFile) getLogFileContent(Framework, localInstallFile, str(BASEDIR) + installLogFile) if downloadUnInstallLog: logger.debug('Download the uninstall log') unInstallLogFile = agentPlatformConfig.getUnInstallLogFile( ) localUnInstallFile = File( AGENTS_LOGS_TEMP_DIR_FILE, ip_address_str + '-' + unInstallLogFile) getLogFileContent(Framework, localUnInstallFile, str(BASEDIR) + unInstallLogFile) except: errorMessage = str(sys.exc_info()[1]) logger.debugException(errorMessage) Framework.reportError( inventoryerrorcodes. INVENTORY_DISCOVERY_FAILED_EXECUTE_STEP, ['FinalizeAndReleaseResources', errorMessage]) finally: try: if localInstallFile and not localInstallFile.delete(): logger.debug('File was not deleted:' + localInstallFile.getCanonicalPath()) except: logger.debugException('Failed to delete ' + localInstallFile.getCanonicalPath()) try: logger.debug('Going to delete file ' + localInstallFile.getCanonicalPath()) if localUnInstallFile and not localUnInstallFile.delete(): logger.debug('File was not deleted:' + localUnInstallFile.getCanonicalPath()) except: logger.debugException('Failed to delete ' + localUnInstallFile.getCanonicalPath())
def StepMain(Framework): AgentUtils.prepareFrameworkForShellOrAgentConnect(Framework) InventoryUtils.executeStep(Framework, connectToRemoteNode, InventoryUtils.STEP_REQUIRES_CONNECTION, InventoryUtils.STEP_DOESNOT_REQUIRES_LOCK)