def __init__(self): """c'tor just setting validation order """ self.validator = ( self._hasRequestName, self._hasOwner, self._hasOperations, self._hasType, self._hasFiles, self._hasRequiredAttrs, self._hasChecksumAndChecksumType, ) configPath = PathFinder.getAgentSection( "RequestManagement/RequestExecutingAgent") # # operation handlers over here opHandlersPath = "%s/%s" % (configPath, "OperationHandlers") opHandlers = gConfig.getSections(opHandlersPath) if not opHandlers["OK"]: gLogger.error(opHandlers["Message"]) else: self.opHandlers = set(opHandlers["Value"])
def __init__(self, *args, **kwargs): """ c'tor :param self: self reference :param str agentName: name of agent :param str loadName: name of module :param bool baseAgentName: whatever :param dict properties: whatever else """ AgentModule.__init__(self, *args, **kwargs) agentName = args[0] ## save config path self.__configPath = PathFinder.getAgentSection(agentName) self.log.info("Will use %s config path" % self.__configPath) ## ProcessPool related stuff self.__requestsPerCycle = self.am_getOption("RequestsPerCycle", 10) self.log.info("requests/cycle = %d" % self.__requestsPerCycle) self.__minProcess = self.am_getOption("MinProcess", 1) self.log.info("ProcessPool min process = %d" % self.__minProcess) self.__maxProcess = self.am_getOption("MaxProcess", 4) self.log.info("ProcessPool max process = %d" % self.__maxProcess) self.__queueSize = self.am_getOption("ProcessPoolQueueSize", 10) self.log.info("ProcessPool queue size = %d" % self.__queueSize) self.__poolTimeout = int(self.am_getOption("ProcessPoolTimeout", 300)) self.log.info("ProcessPool timeout = %d seconds" % self.__poolTimeout) self.__taskTimeout = int(self.am_getOption("ProcessTaskTimeout", 300)) self.log.info("ProcessTask timeout = %d seconds" % self.__taskTimeout) ## request type self.__requestType = self.am_getOption("RequestType", self.__requestType) self.log.info("Will process '%s' request type." % str(self.__requestType)) ## shifter proxy self.am_setOption("shifterProxy", "DataManager") self.log.info("Will use DataManager proxy by default.") ## common monitor activity self.monitor.registerActivity("Iteration", "Agent Loops", self.__class__.__name__, "Loops/min", gMonitor.OP_SUM) self.monitor.registerActivity("Execute", "Request Processed", self.__class__.__name__, "Requests/min", gMonitor.OP_SUM) self.monitor.registerActivity("Done", "Request Completed", self.__class__.__name__, "Requests/min", gMonitor.OP_SUM) ## create request dict self.__requestHolder = dict()
def initialize(self): self.section = PathFinder.getAgentSection(AGENT_NAME) self.RequestDB = RequestDBMySQL() self.TransferDB = TransferDB() self.DataLog = DataLoggingClient() self.factory = StorageFactory() self.rm = ReplicaManager() # This sets the Default Proxy to used as that defined under # /Operations/Shifter/DataManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption('shifterProxy', 'DataManager') return S_OK()
def initialize( self ): self.section = PathFinder.getAgentSection( AGENT_NAME ) self.RequestDB = RequestDBMySQL() self.TransferDB = TransferDB() self.DataLog = DataLoggingClient() self.factory = StorageFactory() self.rm = ReplicaManager() # This sets the Default Proxy to used as that defined under # /Operations/Shifter/DataManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption( 'shifterProxy', 'DataManager' ) return S_OK()
def __init__( self, *args, **kwargs ): """ c'tor :param self: self reference :param str agentName: name of agent :param str loadName: name of module :param bool baseAgentName: whatever :param dict properties: whatever else """ AgentModule.__init__( self, *args, **kwargs ) agentName = args[0] ## save config path self.__configPath = PathFinder.getAgentSection( agentName ) self.log.info( "Will use %s config path" % self.__configPath ) ## ProcessPool related stuff self.__requestsPerCycle = self.am_getOption( "RequestsPerCycle", 10 ) self.log.info("requests/cycle = %d" % self.__requestsPerCycle ) self.__minProcess = self.am_getOption( "MinProcess", 1 ) self.log.info("ProcessPool min process = %d" % self.__minProcess ) self.__maxProcess = self.am_getOption( "MaxProcess", 4 ) self.log.info("ProcessPool max process = %d" % self.__maxProcess ) self.__queueSize = self.am_getOption( "ProcessPoolQueueSize", 10 ) self.log.info("ProcessPool queue size = %d" % self.__queueSize ) self.__poolTimeout = int( self.am_getOption( "ProcessPoolTimeout", 300 ) ) self.log.info("ProcessPool timeout = %d seconds" % self.__poolTimeout ) self.__taskTimeout = int( self.am_getOption( "ProcessTaskTimeout", 300 ) ) self.log.info("ProcessTask timeout = %d seconds" % self.__taskTimeout ) ## request type self.__requestType = self.am_getOption( "RequestType", self.__requestType ) self.log.info( "Will process '%s' request type." % str( self.__requestType ) ) ## shifter proxy self.am_setOption( "shifterProxy", "DataManager" ) self.log.info( "Will use DataManager proxy by default." ) ## common monitor activity self.monitor.registerActivity( "Iteration", "Agent Loops", self.__class__.__name__, "Loops/min", gMonitor.OP_SUM ) self.monitor.registerActivity( "Execute", "Request Processed", self.__class__.__name__, "Requests/min", gMonitor.OP_SUM ) self.monitor.registerActivity( "Done", "Request Completed", self.__class__.__name__, "Requests/min", gMonitor.OP_SUM ) ## create request dict self.__requestHolder = dict()
def initialize( self ): self.logger = gLogger.getSubLogger( "Monitoring" ) self.logger.debug( "Initializing Monitoring Client" ) self.sourceDict[ 'setup' ] = gConfig.getValue( "/DIRAC/Setup" ) self.sourceDict[ 'site' ] = DIRAC.siteName() if self.sourceDict[ 'componentType' ] == self.COMPONENT_SERVICE: self.cfgSection = PathFinder.getSystemSection( self.sourceDict[ 'componentName' ] ) elif self.sourceDict[ 'componentType' ] == self.COMPONENT_AGENT: self.cfgSection = PathFinder.getAgentSection( self.sourceDict[ 'componentName' ] ) self.setComponentLocation( Network.getFQDN() ) elif self.sourceDict[ 'componentType' ] == self.COMPONENT_WEB: self.cfgSection = "/Website" self.setComponentLocation( 'http://%s' % Network.getFQDN() ) self.setComponentName( 'Web' ) elif self.sourceDict[ 'componentType' ] == self.COMPONENT_SCRIPT: self.cfgSection = "/Script" else: raise Exception( "Component type has not been defined" ) gMonitoringFlusher.registerMonitoringClient( self ) #ExitCallback.registerExitCallback( self.forceFlush ) self.__initialized = True
def initialize( self ): self.logger = gLogger.getSubLogger( "Monitoring" ) self.logger.debug( "Initializing Monitoring Client" ) self.sourceDict[ 'setup' ] = gConfig.getValue( "/DIRAC/Setup" ) self.sourceDict[ 'site' ] = DIRAC.siteName() if self.sourceDict[ 'componentType' ] == self.COMPONENT_SERVICE: self.cfgSection = PathFinder.getSystemSection( self.sourceDict[ 'componentName' ] ) elif self.sourceDict[ 'componentType' ] == self.COMPONENT_AGENT: self.cfgSection = PathFinder.getAgentSection( self.sourceDict[ 'componentName' ] ) self.setComponentLocation( Network.getFQDN() ) elif self.sourceDict[ 'componentType' ] == self.COMPONENT_WEB: self.cfgSection = "/WebApp" self.setComponentLocation( 'http://%s' % Network.getFQDN() ) self.setComponentName( 'WebApp' ) elif self.sourceDict[ 'componentType' ] == self.COMPONENT_SCRIPT: self.cfgSection = "/Script" else: raise Exception( "Component type has not been defined" ) gMonitoringFlusher.registerMonitoringClient( self ) # ExitCallback.registerExitCallback( self.forceFlush ) self.__initialized = True
def __init__(self): """ c'tor just setting validation order """ self.validator = (self._hasRequestName, self._hasOwner, self._hasOperations, self._hasType, self._hasFiles, self._hasRequiredAttrs, self._hasChecksumAndChecksumType) configPath = PathFinder.getAgentSection("RequestManagement/RequestExecutingAgent") # # operation handlers over here opHandlersPath = "%s/%s" % (configPath, "OperationHandlers") opHandlers = gConfig.getSections(opHandlersPath) if not opHandlers["OK"]: gLogger.error(opHandlers["Message"]) else: self.opHandlers = set(opHandlers["Value"])
def initialize(self): self.logger = gLogger.getSubLogger("Monitoring") self.logger.debug("Initializing Monitoring Client") self.sourceDict["setup"] = gConfig.getValue("/DIRAC/Setup") self.sourceDict["site"] = DIRAC.siteName() if self.sourceDict["componentType"] == self.COMPONENT_SERVICE: self.cfgSection = PathFinder.getSystemSection( self.sourceDict["componentName"]) elif self.sourceDict["componentType"] == self.COMPONENT_AGENT: self.cfgSection = PathFinder.getAgentSection( self.sourceDict["componentName"]) self.setComponentLocation(Network.getFQDN()) elif self.sourceDict["componentType"] == self.COMPONENT_WEB: self.cfgSection = "/WebApp" self.setComponentLocation("http://%s" % Network.getFQDN()) self.setComponentName("WebApp") elif self.sourceDict["componentType"] == self.COMPONENT_SCRIPT: self.cfgSection = "/Script" elif self.sourceDict["componentType"] == self.COMPONENT_TORNADO: self.cfgSection = "/Tornado" else: raise Exception("Component type has not been defined") gMonitoringFlusher.registerMonitoringClient(self) self.__initialized = True
def __init__( self, *args, **kwargs ): """ c'tor """ # # call base class ctor AgentModule.__init__( self, *args, **kwargs ) # # ProcessPool related stuff self.__requestsPerCycle = self.am_getOption( "RequestsPerCycle", self.__requestsPerCycle ) self.log.info( "Requests/cycle = %d" % self.__requestsPerCycle ) self.__minProcess = self.am_getOption( "MinProcess", self.__minProcess ) self.log.info( "ProcessPool min process = %d" % self.__minProcess ) self.__maxProcess = self.am_getOption( "MaxProcess", 4 ) self.log.info( "ProcessPool max process = %d" % self.__maxProcess ) self.__queueSize = self.am_getOption( "ProcessPoolQueueSize", self.__queueSize ) self.log.info( "ProcessPool queue size = %d" % self.__queueSize ) self.__poolTimeout = int( self.am_getOption( "ProcessPoolTimeout", self.__poolTimeout ) ) self.log.info( "ProcessPool timeout = %d seconds" % self.__poolTimeout ) self.__poolSleep = int( self.am_getOption( "ProcessPoolSleep", self.__poolSleep ) ) self.log.info( "ProcessPool sleep time = %d seconds" % self.__poolSleep ) self.__taskTimeout = int( self.am_getOption( "ProcessTaskTimeout", self.__taskTimeout ) ) self.log.info( "ProcessTask timeout = %d seconds" % self.__taskTimeout ) self.__bulkRequest = self.am_getOption( "BulkRequest", 0 ) self.log.info( "Bulk request size = %d" % self.__bulkRequest ) # # keep config path and agent name self.agentName = self.am_getModuleParam( "fullName" ) self.__configPath = PathFinder.getAgentSection( self.agentName ) # # operation handlers over here opHandlersPath = "%s/%s" % ( self.__configPath, "OperationHandlers" ) opHandlers = gConfig.getSections( opHandlersPath ) if not opHandlers["OK"]: self.log.error( opHandlers["Message" ] ) raise AgentConfigError( "OperationHandlers section not found in CS under %s" % self.__configPath ) opHandlers = opHandlers["Value"] self.timeOuts = dict() # # handlers dict self.handlersDict = dict() for opHandler in opHandlers: opHandlerPath = "%s/%s/Location" % ( opHandlersPath, opHandler ) opLocation = gConfig.getValue( opHandlerPath, "" ) if not opLocation: self.log.error( "%s not set for %s operation handler" % ( opHandlerPath, opHandler ) ) continue self.timeOuts[opHandler] = { "PerFile": self.__fileTimeout, "PerOperation": self.__operationTimeout } opTimeout = gConfig.getValue( "%s/%s/TimeOut" % ( opHandlersPath, opHandler ), 0 ) if opTimeout: self.timeOuts[opHandler]["PerOperation"] = opTimeout fileTimeout = gConfig.getValue( "%s/%s/TimeOutPerFile" % ( opHandlersPath, opHandler ), 0 ) if fileTimeout: self.timeOuts[opHandler]["PerFile"] = fileTimeout self.handlersDict[opHandler] = opLocation self.log.info( "Operation handlers:" ) for item in enumerate ( self.handlersDict.items() ): opHandler = item[1][0] self.log.info( "[%s] %s: %s (timeout: %d s + %d s per file)" % ( item[0], item[1][0], item[1][1], self.timeOuts[opHandler]['PerOperation'], self.timeOuts[opHandler]['PerFile'] ) ) # # common monitor activity gMonitor.registerActivity( "Iteration", "Agent Loops", "RequestExecutingAgent", "Loops/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "Processed", "Request Processed", "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "Done", "Request Completed", "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM ) # # create request dict self.__requestCache = dict() self.FTSMode = self.am_getOption( "FTSMode", False )
def __init__( self, agentName, loadName, baseAgentName = False, properties = {} ): """ Common __init__ method for all Agents. All Agent modules must define: __doc__ __RCSID__ They are used to populate __codeProperties The following Options are used from the Configuration: - /LocalSite/InstancePath - /DIRAC/Setup - Status - Enabled - PollingTime default = 120 - MaxCycles default = 500 - ControlDirectory control/SystemName/AgentName - WorkDirectory work/SystemName/AgentName - shifterProxy '' - shifterProxyLocation WorkDirectory/SystemName/AgentName/.shifterCred It defines the following default Options that can be set via Configuration (above): - MonitoringEnabled True - Enabled True if Status == Active - PollingTime 120 - MaxCycles 500 - ControlDirectory control/SystemName/AgentName - WorkDirectory work/SystemName/AgentName - shifterProxy False - shifterProxyLocation work/SystemName/AgentName/.shifterCred different defaults can be set in the initialize() method of the Agent using am_setOption() In order to get a shifter proxy in the environment during the execute() the configuration Option 'shifterProxy' must be set, a default may be given in the initialize() method. """ if baseAgentName and agentName == baseAgentName: self.log = gLogger standaloneModule = True else: self.log = gLogger.getSubLogger( agentName, child = False ) standaloneModule = False self.__basePath = gConfig.getValue( '/LocalSite/InstancePath', rootPath ) self.__agentModule = None self.__codeProperties = {} self.__getCodeInfo() self.__moduleProperties = { 'fullName' : agentName, 'loadName' : loadName, 'section' : PathFinder.getAgentSection( agentName ), 'loadSection' : PathFinder.getAgentSection( loadName ), 'standalone' : standaloneModule, 'cyclesDone' : 0, 'totalElapsedTime' : 0, 'setup' : gConfig.getValue( "/DIRAC/Setup", "Unknown" ), 'alive' : True } self.__moduleProperties[ 'system' ], self.__moduleProperties[ 'agentName' ] = agentName.split( "/" ) self.__configDefaults = {} self.__configDefaults[ 'MonitoringEnabled'] = True self.__configDefaults[ 'Enabled'] = self.am_getOption( "Status", "Active" ).lower() in ( 'active' ) self.__configDefaults[ 'PollingTime'] = self.am_getOption( "PollingTime", 120 ) self.__configDefaults[ 'MaxCycles'] = self.am_getOption( "MaxCycles", 500 ) self.__configDefaults[ 'ControlDirectory' ] = os.path.join( self.__basePath, 'control', *agentName.split( "/" ) ) self.__configDefaults[ 'WorkDirectory' ] = os.path.join( self.__basePath, 'work', *agentName.split( "/" ) ) self.__configDefaults[ 'shifterProxy' ] = '' self.__configDefaults[ 'shifterProxyLocation' ] = os.path.join( self.__configDefaults[ 'WorkDirectory' ], '.shifterCred' ) if isinstance( properties, dict): for key in properties: self.__moduleProperties[ key ] = properties[ key ] self.__moduleProperties[ 'executors' ] = [ ( self.execute, () ) ] self.__moduleProperties[ 'shifterProxy' ] = False self.__monitorLastStatsUpdate = -1 self.monitor = None self.__initializeMonitor() self.__initialized = False
def setUp( self ): """ test setup """ self.configPath = PathFinder.getAgentSection( "DataManagement/TransferAgent" ) self.channels = { 1L: {'Status': 'Active', 'Files': 0, 'Destination': 'CERN', 'Source': 'CERN', 'ChannelName': 'CERN-CERN', 'Size': 0}, 2L: {'Status': 'Active', 'Files': 0, 'Destination': 'CNAF', 'Source': 'CERN', 'ChannelName': 'CERN-CNAF', 'Size': 0}, 3L: {'Status': 'Active', 'Files': 0, 'Destination': 'GRIDKA', 'Source': 'CERN', 'ChannelName': 'CERN-GRIDKA', 'Size': 0}, 4L: {'Status': 'Active', 'Files': 0, 'Destination': 'IN2P3', 'Source': 'CERN', 'ChannelName': 'CERN-IN2P3', 'Size': 0}, 5L: {'Status': 'Active', 'Files': 0, 'Destination': 'NIKHEF', 'Source': 'CERN', 'ChannelName': 'CERN-NIKHEF', 'Size': 0}, 6L: {'Status': 'Active', 'Files': 2, 'Destination': 'PIC', 'Source': 'CERN', 'ChannelName': 'CERN-PIC', 'Size': 0}, 7L: {'Status': 'Active', 'Files': 0, 'Destination': 'RAL', 'Source': 'CERN', 'ChannelName': 'CERN-RAL', 'Size': 0}, 8L: {'Status': 'Active', 'Files': 0, 'Destination': 'CERN', 'Source': 'CNAF', 'ChannelName': 'CNAF-CERN', 'Size': 0}, 9L: {'Status': 'Active', 'Files': 0, 'Destination': 'CNAF', 'Source': 'CNAF', 'ChannelName': 'CNAF-CNAF', 'Size': 0}, 10L: {'Status': 'Active', 'Files': 0, 'Destination': 'GRIDKA', 'Source': 'CNAF', 'ChannelName': 'CNAF-GRIDKA', 'Size': 0}, 11L: {'Status': 'Active', 'Files': 0, 'Destination': 'IN2P3', 'Source': 'CNAF', 'ChannelName': 'CNAF-IN2P3', 'Size': 0}, 12L: {'Status': 'Active', 'Files': 0, 'Destination': 'NIKHEF', 'Source': 'CNAF', 'ChannelName': 'CNAF-NIKHEF', 'Size': 0}, 13L: {'Status': 'Active', 'Files': 0, 'Destination': 'PIC', 'Source': 'CNAF', 'ChannelName': 'CNAF-PIC', 'Size': 0}, 14L: {'Status': 'Active', 'Files': 0, 'Destination': 'RAL', 'Source': 'CNAF', 'ChannelName': 'CNAF-RAL', 'Size': 0}, 15L: {'Status': 'Active', 'Files': 0, 'Destination': 'CERN', 'Source': 'GRIDKA', 'ChannelName': 'GRIDKA-CERN', 'Size': 0}, 16L: {'Status': 'Active', 'Files': 0, 'Destination': 'CNAF', 'Source': 'GRIDKA', 'ChannelName': 'GRIDKA-CNAF', 'Size': 0}, 17L: {'Status': 'Active', 'Files': 0, 'Destination': 'GRIDKA', 'Source': 'GRIDKA', 'ChannelName': 'GRIDKA-GRIDKA', 'Size': 0}, 18L: {'Status': 'Active', 'Files': 0, 'Destination': 'IN2P3', 'Source': 'GRIDKA', 'ChannelName': 'GRIDKA-IN2P3', 'Size': 0}, 19L: {'Status': 'Active', 'Files': 0, 'Destination': 'NIKHEF', 'Source': 'GRIDKA', 'ChannelName': 'GRIDKA-NIKHEF', 'Size': 0}, 20L: {'Status': 'Active', 'Files': 0, 'Destination': 'PIC', 'Source': 'GRIDKA', 'ChannelName': 'GRIDKA-PIC', 'Size': 0}, 21L: {'Status': 'Active', 'Files': 0, 'Destination': 'RAL', 'Source': 'GRIDKA', 'ChannelName': 'GRIDKA-RAL', 'Size': 0}, 22L: {'Status': 'Active', 'Files': 0, 'Destination': 'CERN', 'Source': 'IN2P3', 'ChannelName': 'IN2P3-CERN', 'Size': 0}, 23L: {'Status': 'Active', 'Files': 0, 'Destination': 'CNAF', 'Source': 'IN2P3', 'ChannelName': 'IN2P3-CNAF', 'Size': 0}, 24L: {'Status': 'Active', 'Files': 0, 'Destination': 'GRIDKA', 'Source': 'IN2P3', 'ChannelName': 'IN2P3-GRIDKA', 'Size': 0}, 25L: {'Status': 'Active', 'Files': 0, 'Destination': 'IN2P3', 'Source': 'IN2P3', 'ChannelName': 'IN2P3-IN2P3', 'Size': 0}, 26L: {'Status': 'Active', 'Files': 0, 'Destination': 'NIKHEF', 'Source': 'IN2P3', 'ChannelName': 'IN2P3-NIKHEF', 'Size': 0}, 27L: {'Status': 'Active', 'Files': 0, 'Destination': 'PIC', 'Source': 'IN2P3', 'ChannelName': 'IN2P3-PIC', 'Size': 0}, 28L: {'Status': 'Active', 'Files': 0, 'Destination': 'RAL', 'Source': 'IN2P3', 'ChannelName': 'IN2P3-RAL', 'Size': 0}, 29L: {'Status': 'Active', 'Files': 0, 'Destination': 'CERN', 'Source': 'NIKHEF', 'ChannelName': 'NIKHEF-CERN', 'Size': 0}, 30L: {'Status': 'Active', 'Files': 0, 'Destination': 'CNAF', 'Source': 'NIKHEF', 'ChannelName': 'NIKHEF-CNAF', 'Size': 0}, 31L: {'Status': 'Active', 'Files': 0, 'Destination': 'GRIDKA', 'Source': 'NIKHEF', 'ChannelName': 'NIKHEF-GRIDKA', 'Size': 0}, 32L: {'Status': 'Active', 'Files': 0, 'Destination': 'IN2P3', 'Source': 'NIKHEF', 'ChannelName': 'NIKHEF-IN2P3', 'Size': 0}, 33L: {'Status': 'Active', 'Files': 0, 'Destination': 'NIKHEF', 'Source': 'NIKHEF', 'ChannelName': 'NIKHEF-NIKHEF', 'Size': 0}, 34L: {'Status': 'Active', 'Files': 0, 'Destination': 'PIC', 'Source': 'NIKHEF', 'ChannelName': 'NIKHEF-PIC', 'Size': 0}, 35L: {'Status': 'Active', 'Files': 0, 'Destination': 'RAL', 'Source': 'NIKHEF', 'ChannelName': 'NIKHEF-RAL', 'Size': 0}, 36L: {'Status': 'Active', 'Files': 0, 'Destination': 'CERN', 'Source': 'PIC', 'ChannelName': 'PIC-CERN', 'Size': 0}, 37L: {'Status': 'Active', 'Files': 0, 'Destination': 'CNAF', 'Source': 'PIC', 'ChannelName': 'PIC-CNAF', 'Size': 0}, 38L: {'Status': 'Active', 'Files': 0, 'Destination': 'GRIDKA', 'Source': 'PIC', 'ChannelName': 'PIC-GRIDKA', 'Size': 0}, 39L: {'Status': 'Active', 'Files': 0, 'Destination': 'IN2P3', 'Source': 'PIC', 'ChannelName': 'PIC-IN2P3', 'Size': 0}, 40L: {'Status': 'Active', 'Files': 0, 'Destination': 'NIKHEF', 'Source': 'PIC', 'ChannelName': 'PIC-NIKHEF', 'Size': 0}, 41L: {'Status': 'Active', 'Files': 0, 'Destination': 'PIC', 'Source': 'PIC', 'ChannelName': 'PIC-PIC', 'Size': 0}, 42L: {'Status': 'Active', 'Files': 0, 'Destination': 'RAL', 'Source': 'PIC', 'ChannelName': 'PIC-RAL', 'Size': 0}, 43L: {'Status': 'Active', 'Files': 0, 'Destination': 'CERN', 'Source': 'RAL', 'ChannelName': 'RAL-CERN', 'Size': 0}, 44L: {'Status': 'Active', 'Files': 0, 'Destination': 'CNAF', 'Source': 'RAL', 'ChannelName': 'RAL-CNAF', 'Size': 0}, 45L: {'Status': 'Active', 'Files': 0, 'Destination': 'GRIDKA', 'Source': 'RAL', 'ChannelName': 'RAL-GRIDKA', 'Size': 0}, 46L: {'Status': 'Active', 'Files': 0, 'Destination': 'IN2P3', 'Source': 'RAL', 'ChannelName': 'RAL-IN2P3', 'Size': 0}, 47L: {'Status': 'Active', 'Files': 0, 'Destination': 'NIKHEF', 'Source': 'RAL', 'ChannelName': 'RAL-NIKHEF', 'Size': 0}, 48L: {'Status': 'Active', 'Files': 0, 'Destination': 'PIC', 'Source': 'RAL', 'ChannelName': 'RAL-PIC', 'Size': 0}, 49L: {'Status': 'Active', 'Files': 0, 'Destination': 'RAL', 'Source': 'RAL', 'ChannelName': 'RAL-RAL', 'Size': 0}, 50L: {'Status': 'Active', 'Files': 0, 'Destination': 'SARA', 'Source': 'CERN', 'ChannelName': 'CERN-SARA', 'Size': 0}, 51L: {'Status': 'Active', 'Files': 0, 'Destination': 'CERN', 'Source': 'SARA', 'ChannelName': 'SARA-CERN', 'Size': 0}, 52L: {'Status': 'Active', 'Files': 0, 'Destination': 'CNAF', 'Source': 'SARA', 'ChannelName': 'SARA-CNAF', 'Size': 0}, 53L: {'Status': 'Active', 'Files': 0, 'Destination': 'GRIDKA', 'Source': 'SARA', 'ChannelName': 'SARA-GRIDKA', 'Size': 0}, 54L: {'Status': 'Active', 'Files': 0, 'Destination': 'IN2P3', 'Source': 'SARA', 'ChannelName': 'SARA-IN2P3', 'Size': 0}, 55L: {'Status': 'Active', 'Files': 0, 'Destination': 'NIKHEF', 'Source': 'SARA', 'ChannelName': 'SARA-NIKHEF', 'Size': 0}, 56L: {'Status': 'Active', 'Files': 0, 'Destination': 'PIC', 'Source': 'SARA', 'ChannelName': 'SARA-PIC', 'Size': 0}, 57L: {'Status': 'Active', 'Files': 0, 'Destination': 'RAL', 'Source': 'SARA', 'ChannelName': 'SARA-RAL', 'Size': 0}, 58L: {'Status': 'Active', 'Files': 0, 'Destination': 'SARA', 'Source': 'SARA', 'ChannelName': 'SARA-SARA', 'Size': 0}, 59L: {'Status': 'Active', 'Files': 0, 'Destination': 'SARA', 'Source': 'CNAF', 'ChannelName': 'CNAF-SARA', 'Size': 0}, 60L: {'Status': 'Active', 'Files': 0, 'Destination': 'SARA', 'Source': 'GRIDKA', 'ChannelName': 'GRIDKA-SARA', 'Size': 0}, 61L: {'Status': 'Active', 'Files': 0, 'Destination': 'SARA', 'Source': 'IN2P3', 'ChannelName': 'IN2P3-SARA', 'Size': 0}, 62L: {'Status': 'Active', 'Files': 0, 'Destination': 'SARA', 'Source': 'NIKHEF', 'ChannelName': 'NIKHEF-SARA', 'Size': 0}, 63L: {'Status': 'Active', 'Files': 0, 'Destination': 'SARA', 'Source': 'PIC', 'ChannelName': 'PIC-SARA', 'Size': 0}, 64L: {'Status': 'Active', 'Files': 0, 'Destination': 'SARA', 'Source': 'RAL', 'ChannelName': 'RAL-SARA', 'Size': 0}, 65L: {'Status': 'Active', 'Files': 0, 'Destination': 'CERN', 'Source': 'RAL-HEP', 'ChannelName': 'CERN-RAL-HEP', 'Size':0 } } self.bands = dict.fromkeys( self.channels.keys() ) for k in self.bands: self.bands[k] = { 'SuccessfulFiles': 0, 'FailedFiles': 0, 'Throughput': 0, 'Fileput': 0 } #self.bands[1]["Fileput"] = 10.0 self.bands[14]["Fileput"] = 10.0 self.failedFiles = dict.fromkeys( self.channels.keys(), 0 )
def __init__( self, *args, **kwargs ): """ c'tor """ # # call base class ctor AgentModule.__init__( self, *args, **kwargs ) # # ProcessPool related stuff self.__requestsPerCycle = self.am_getOption( "RequestsPerCycle", self.__requestsPerCycle ) self.log.info( "Requests/cycle = %d" % self.__requestsPerCycle ) self.__minProcess = self.am_getOption( "MinProcess", self.__minProcess ) self.log.info( "ProcessPool min process = %d" % self.__minProcess ) self.__maxProcess = self.am_getOption( "MaxProcess", 4 ) self.log.info( "ProcessPool max process = %d" % self.__maxProcess ) self.__queueSize = self.am_getOption( "ProcessPoolQueueSize", self.__queueSize ) self.log.info( "ProcessPool queue size = %d" % self.__queueSize ) self.__poolTimeout = int( self.am_getOption( "ProcessPoolTimeout", self.__poolTimeout ) ) self.log.info( "ProcessPool timeout = %d seconds" % self.__poolTimeout ) self.__poolSleep = int( self.am_getOption( "ProcessPoolSleep", self.__poolSleep ) ) self.log.info( "ProcessPool sleep time = %d seconds" % self.__poolSleep ) self.__bulkRequest = self.am_getOption( "BulkRequest", 0 ) self.log.info( "Bulk request size = %d" % self.__bulkRequest ) # # keep config path and agent name self.agentName = self.am_getModuleParam( "fullName" ) self.__configPath = PathFinder.getAgentSection( self.agentName ) # # operation handlers over here opHandlersPath = "%s/%s" % ( self.__configPath, "OperationHandlers" ) opHandlers = gConfig.getSections( opHandlersPath ) if not opHandlers["OK"]: self.log.error( opHandlers["Message" ] ) raise AgentConfigError( "OperationHandlers section not found in CS under %s" % self.__configPath ) opHandlers = opHandlers["Value"] self.timeOuts = dict() # # handlers dict self.handlersDict = dict() for opHandler in opHandlers: opHandlerPath = "%s/%s/Location" % ( opHandlersPath, opHandler ) opLocation = gConfig.getValue( opHandlerPath, "" ) if not opLocation: self.log.error( "%s not set for %s operation handler" % ( opHandlerPath, opHandler ) ) continue self.timeOuts[opHandler] = { "PerFile": self.__fileTimeout, "PerOperation": self.__operationTimeout } opTimeout = gConfig.getValue( "%s/%s/TimeOut" % ( opHandlersPath, opHandler ), 0 ) if opTimeout: self.timeOuts[opHandler]["PerOperation"] = opTimeout fileTimeout = gConfig.getValue( "%s/%s/TimeOutPerFile" % ( opHandlersPath, opHandler ), 0 ) if fileTimeout: self.timeOuts[opHandler]["PerFile"] = fileTimeout self.handlersDict[opHandler] = opLocation self.log.info( "Operation handlers:" ) for item in enumerate ( self.handlersDict.items() ): opHandler = item[1][0] self.log.info("[%s] %s: %s (timeout: %d s + %d s per file)" % (item[0], item[1][0], item[1][1], self.timeOuts[opHandler]['PerOperation'], self.timeOuts[opHandler]['PerFile'])) # # common monitor activity gMonitor.registerActivity( "Iteration", "Agent Loops", "RequestExecutingAgent", "Loops/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "Processed", "Request Processed", "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "Done", "Request Completed", "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM ) # # create request dict self.__requestCache = dict() # ?? Probably should be removed self.FTSMode = self.am_getOption( "FTSMode", False )
def __init__(self, agentName, loadName, baseAgentName=False, properties={}): """ Common __init__ method for all Agents. All Agent modules must define: __doc__ __RCSID__ They are used to populate __codeProperties The following Options are used from the Configuration: - /LocalSite/InstancePath - /DIRAC/Setup - Status - Enabled - PollingTime default = 120 - MaxCycles default = 500 - WatchdogTime default = 0 (disabled) - ControlDirectory control/SystemName/AgentName - WorkDirectory work/SystemName/AgentName - shifterProxy '' - shifterProxyLocation WorkDirectory/SystemName/AgentName/.shifterCred It defines the following default Options that can be set via Configuration (above): - MonitoringEnabled True - Enabled True if Status == Active - PollingTime 120 - MaxCycles 500 - ControlDirectory control/SystemName/AgentName - WorkDirectory work/SystemName/AgentName - shifterProxy False - shifterProxyLocation work/SystemName/AgentName/.shifterCred different defaults can be set in the initialize() method of the Agent using am_setOption() In order to get a shifter proxy in the environment during the execute() the configuration Option 'shifterProxy' must be set, a default may be given in the initialize() method. """ if baseAgentName and agentName == baseAgentName: self.log = gLogger standaloneModule = True else: self.log = gLogger.getSubLogger(agentName, child=False) standaloneModule = False self.__basePath = gConfig.getValue("/LocalSite/InstancePath", rootPath) self.__agentModule = None self.__codeProperties = {} self.__getCodeInfo() self.__moduleProperties = { "fullName": agentName, "loadName": loadName, "section": PathFinder.getAgentSection(agentName), "loadSection": PathFinder.getAgentSection(loadName), "standalone": standaloneModule, "cyclesDone": 0, "totalElapsedTime": 0, "setup": gConfig.getValue("/DIRAC/Setup", "Unknown"), "alive": True, } self.__moduleProperties["system"], self.__moduleProperties[ "agentName"] = agentName.split("/") self.__configDefaults = {} self.__configDefaults["MonitoringEnabled"] = True self.__configDefaults["Enabled"] = self.am_getOption( "Status", "Active").lower() in ("active") self.__configDefaults["PollingTime"] = self.am_getOption( "PollingTime", 120) self.__configDefaults["MaxCycles"] = self.am_getOption( "MaxCycles", 500) self.__configDefaults["WatchdogTime"] = self.am_getOption( "WatchdogTime", 0) self.__configDefaults["ControlDirectory"] = os.path.join( self.__basePath, "control", *agentName.split("/")) self.__configDefaults["WorkDirectory"] = os.path.join( self.__basePath, "work", *agentName.split("/")) self.__configDefaults["shifterProxy"] = "" self.__configDefaults["shifterProxyLocation"] = os.path.join( self.__configDefaults["WorkDirectory"], ".shifterCred") if isinstance(properties, dict): for key in properties: self.__moduleProperties[key] = properties[key] self.__moduleProperties["executors"] = [(self.execute, ())] self.__moduleProperties["shifterProxy"] = False self.__monitorLastStatsUpdate = -1 self.monitor = None self.__initializeMonitor() self.__initialized = False
def showChannels(): """ print info about the last hour performance of FTS system """ global gProblematic taSection = PathFinder.getAgentSection("DataManagement/TransferAgent") timeScale = gConfig.getOption(taSection + "/ThroughputTimescale", 3600) if not timeScale["OK"]: gLogger.error(timeScale["Message"]) DIRAC.exit(1) timeScale = int(timeScale["Value"]) accFailureRate = gConfig.getOption( taSection + "/StrategyHandler/AcceptableFailureRate", 75) if not accFailureRate["OK"]: gLogger.error(accFailureRate["Message"]) accFailureRate = int(accFailureRate["Value"]) accFailedFiles = gConfig.getOption( taSection + "/StrategyHandler/AcceptableFailedFiles", 5) if not accFailedFiles["OK"]: gLogger.error(accFailedFiles["Message"]) accFailedFiles = int(accFailedFiles["Value"]) scInfo = "timescale = %s s\nacc failure rate = %s %%\nacc distinct failed files = %s" % ( timeScale, accFailureRate, accFailedFiles) ## db monitor transferDB = RPCClient("DataManagement/TransferDBMonitoring") ## get channels channels = transferDB.getChannelQueues() if not channels["OK"]: gLogger.error(channels["Message"]) DIRAC.exit(1) channels = channels["Value"] ## gend bandwidths bands = transferDB.getChannelObservedThroughput(timeScale) if not bands["OK"]: gLogger.error(bands["Message"]) DIRAC.exit(1) bands = bands["Value"] ## get failed files badFiles = transferDB.getCountFileToFTS(timeScale, "Failed") if not badFiles["OK"]: gLogger.error(badFiles["Message"]) DIRAC.exit(1) badFiles = badFiles["Value"] if badFiles["Value"] else {} colorize = sys.stdout.isatty() header = " %2s | %-15s | %8s | %8s | %8s | %8s | %8s | %12s | %8s | %8s" % ( "Id", "Name", "Status", "Waiting", "Success", "Failed", "FilePut", "ThroughPut", "FailRate", "FailedFiles") dashLine = "-" * len(header) lineTemplate = " %2s | %-15s | %8s | %8d | %8d | %8d | %8.2f | %12.2f | %8.2f | %8d" printOut = [] for chId, channel in channels.items(): name = channel["ChannelName"] color = None status = channel["Status"] if status == "Active": status = "OK" waitingFiles = channel["Files"] waitingSize = channel["Size"] failedFiles = successFiles = filePut = througPut = fRate = 0 fFiles = 0 if chId in badFiles: fFiles = int(badFiles[chId]) if chId in bands: band = bands[chId] failedFiles = int(band["FailedFiles"]) successFiles = int(band["SuccessfulFiles"]) filePut = band["Fileput"] throughPut = band["Throughput"] if failedFiles or successFiles: fRate = 100.0 * float(failedFiles) / (float(failedFiles) + float(successFiles)) if fRate > 0 and colorize: color = "yellow" status = "Poor" if fRate > accFailureRate and fFiles > accFailedFiles: status = "Closed" if colorize: color = "red" if gProblematic and not fRate: continue if colorize and color: line = colors[color] % lineTemplate else: line = lineTemplate printOut.append( line % (chId, name, status, waitingFiles if waitingFiles else 0, successFiles if successFiles else 0, failedFiles if failedFiles else 0, filePut if filePut else 0, throughPut if througPut else 0, fRate if fRate else 0, fFiles if fFiles else 0)) if printOut: printOut = [scInfo, header, dashLine] + printOut for line in printOut: gLogger.always(line) else: gLogger.always("Noting to display...")
def showChannels(): """ print info about the last hour performance of FTS system """ global gProblematic taSection = PathFinder.getAgentSection("DataManagement/TransferAgent") timeScale = gConfig.getOption( taSection + "/ThroughputTimescale", 3600 ) if not timeScale["OK"]: gLogger.error( timeScale["Message"] ) DIRAC.exit(1) timeScale = int( timeScale["Value"] ) accFailureRate = gConfig.getOption( taSection + "/StrategyHandler/AcceptableFailureRate", 75 ) if not accFailureRate["OK"]: gLogger.error( accFailureRate["Message"] ) accFailureRate = int( accFailureRate["Value"] ) accFailedFiles = gConfig.getOption( taSection + "/StrategyHandler/AcceptableFailedFiles", 5 ) if not accFailedFiles["OK"]: gLogger.error( accFailedFiles["Message"] ) accFailedFiles = int( accFailedFiles["Value"] ) scInfo = "timescale = %s s\nacc failure rate = %s %%\nacc distinct failed files = %s" % ( timeScale, accFailureRate, accFailedFiles ) ## db monitor transferDB = RPCClient( "DataManagement/TransferDBMonitoring" ) ## get channels channels = transferDB.getChannelQueues() if not channels["OK"]: gLogger.error( channels["Message"] ) DIRAC.exit(1) channels = channels["Value"] ## gend bandwidths bands = transferDB.getChannelObservedThroughput( timeScale ) if not bands["OK"]: gLogger.error( bands["Message"] ) DIRAC.exit(1) bands = bands["Value"] ## get failed files badFiles = transferDB.getCountFileToFTS( timeScale, "Failed" ) if not badFiles["OK"]: gLogger.error( badFiles["Message"] ) DIRAC.exit(1) badFiles = badFiles["Value"] if badFiles["Value"] else {} colorize = sys.stdout.isatty() header = " %2s | %-15s | %8s | %8s | %8s | %8s | %8s | %12s | %8s | %8s" % ( "Id", "Name", "Status", "Waiting", "Success", "Failed", "FilePut", "ThroughPut", "FailRate", "FailedFiles" ) dashLine = "-"*len(header) lineTemplate = " %2s | %-15s | %8s | %8d | %8d | %8d | %8.2f | %12.2f | %8.2f | %8d" printOut = [] for chId, channel in channels.items(): name = channel["ChannelName"] color = None status = channel["Status"] if status == "Active": status = "OK" waitingFiles = channel["Files"] waitingSize = channel["Size"] failedFiles = successFiles = filePut = througPut = fRate = 0 fFiles = 0 if chId in badFiles: fFiles = int(badFiles[chId]) if chId in bands: band = bands[chId] failedFiles = int(band["FailedFiles"]) successFiles = int(band["SuccessfulFiles"]) filePut = band["Fileput"] throughPut = band["Throughput"] if failedFiles or successFiles: fRate = 100.0 * float( failedFiles ) / ( float(failedFiles) + float( successFiles) ) if fRate > 0 and colorize: color = "yellow" status = "Poor" if fRate > accFailureRate and fFiles > accFailedFiles: status = "Closed" if colorize: color = "red" if gProblematic and not fRate: continue if colorize and color: line = colors[color] % lineTemplate else: line = lineTemplate printOut.append( line % ( chId, name, status, waitingFiles if waitingFiles else 0, successFiles if successFiles else 0, failedFiles if failedFiles else 0, filePut if filePut else 0, throughPut if througPut else 0, fRate if fRate else 0, fFiles if fFiles else 0 ) ) if printOut: printOut = [ scInfo, header, dashLine ] + printOut for line in printOut: gLogger.always( line ) else: gLogger.always("Noting to display...")
def __init__(self, agentName, baseAgentName=False, properties={}): """ Common __init__ method for all Agents. All Agent modules must define: __doc__ __RCSID__ They are used to populate __codeProperties The following Options are used from the Configuration: - /LocalSite/InstancePath - /DIRAC/Setup - Status - Enabled - PollingTime default = 120 - MaxCycles default = 500 - ControlDirectory control/SystemName/AgentName - WorkDirectory work/SystemName/AgentName - shifterProxy '' - shifterProxyLocation WorkDirectory/SystemName/AgentName/.shifterCred It defines the following default Options that can be set via Configuration (above): - MonitoringEnabled True - Enabled True if Status == Active - PollingTime 120 - MaxCycles 500 - ControlDirectory control/SystemName/AgentName - WorkDirectory work/SystemName/AgentName - shifterProxy False - shifterProxyLocation work/SystemName/AgentName/.shifterCred different defaults can be set in the initialize() method of the Agent using am_setOption() In order to get a shifter proxy in the environment during the execute() the configuration Option 'shifterProxy' must be set, a default may be given in the initialize() method. """ if baseAgentName and agentName == baseAgentName: self.log = gLogger standaloneModule = True else: self.log = gLogger.getSubLogger(agentName, child=False) standaloneModule = False self.__basePath = gConfig.getValue('/LocalSite/InstancePath', rootPath) self.__agentModule = None self.__codeProperties = {} self.__getCodeInfo() self.__moduleProperties = { 'fullName': agentName, 'section': PathFinder.getAgentSection(agentName), 'standalone': standaloneModule, 'cyclesDone': 0, 'totalElapsedTime': 0, 'setup': gConfig.getValue("/DIRAC/Setup", "Unknown") } self.__moduleProperties['system'], self.__moduleProperties[ 'agentName'] = agentName.split("/") self.__configDefaults = {} self.__configDefaults['MonitoringEnabled'] = True self.__configDefaults['Enabled'] = self.am_getOption( "Status", "Active").lower() in ('active') self.__configDefaults['PollingTime'] = self.am_getOption( "PollingTime", 120) self.__configDefaults['MaxCycles'] = self.am_getOption( "MaxCycles", 500) self.__configDefaults['ControlDirectory'] = os.path.join( self.__basePath, 'control', *agentName.split("/")) self.__configDefaults['WorkDirectory'] = os.path.join( self.__basePath, 'work', *agentName.split("/")) self.__configDefaults['shifterProxy'] = '' self.__configDefaults['shifterProxyLocation'] = os.path.join( self.__configDefaults['WorkDirectory'], '.shifterCred') if type(properties) == types.DictType: for key in properties: self.__moduleProperties[key] = properties[key] self.__moduleProperties['executors'] = [(self.execute, ())] self.__moduleProperties['alive'] = True self.__moduleProperties['shifterProxy'] = False self.__monitorLastStatsUpdate = -1 self.monitor = None self.__initializeMonitor() self.__initialized = False
def initialize(self): """initialize agent""" # # ProcessPool related stuff self.__requestsPerCycle = self.am_getOption("RequestsPerCycle", self.__requestsPerCycle) self.log.info("Requests/cycle = %d" % self.__requestsPerCycle) self.__minProcess = self.am_getOption("MinProcess", self.__minProcess) self.log.info("ProcessPool min process = %d" % self.__minProcess) self.__maxProcess = self.am_getOption("MaxProcess", self.__maxProcess) self.log.info("ProcessPool max process = %d" % self.__maxProcess) self.__queueSize = self.am_getOption("ProcessPoolQueueSize", self.__queueSize) self.log.info("ProcessPool queue size = %d" % self.__queueSize) self.__poolTimeout = int(self.am_getOption("ProcessPoolTimeout", self.__poolTimeout)) self.log.info("ProcessPool timeout = %d seconds" % self.__poolTimeout) self.__poolSleep = int(self.am_getOption("ProcessPoolSleep", self.__poolSleep)) self.log.info("ProcessPool sleep time = %d seconds" % self.__poolSleep) self.__bulkRequest = self.am_getOption("BulkRequest", self.__bulkRequest) self.log.info("Bulk request size = %d" % self.__bulkRequest) # Check if monitoring is enabled if "Monitoring" in Operations().getMonitoringBackends(monitoringType="RMSMonitoring"): # Enable RMS monitoring self.__rmsMonitoring = True self.log.info("Enable ES RMS Monitoring = %s" % self.__rmsMonitoring) # # keep config path and agent name self.agentName = self.am_getModuleParam("fullName") self.__configPath = PathFinder.getAgentSection(self.agentName) # # operation handlers over here opHandlersPath = "%s/%s" % (self.__configPath, "OperationHandlers") opHandlers = gConfig.getSections(opHandlersPath) if not opHandlers["OK"]: self.log.error(opHandlers["Message"]) raise AgentConfigError("OperationHandlers section not found in CS under %s" % self.__configPath) opHandlers = opHandlers["Value"] self.timeOuts = dict() # # handlers dict self.handlersDict = dict() for opHandler in opHandlers: opHandlerPath = "%s/%s/Location" % (opHandlersPath, opHandler) opLocation = gConfig.getValue(opHandlerPath, "") if not opLocation: self.log.error("%s not set for %s operation handler" % (opHandlerPath, opHandler)) continue self.timeOuts[opHandler] = {"PerFile": self.__fileTimeout, "PerOperation": self.__operationTimeout} opTimeout = gConfig.getValue("%s/%s/TimeOut" % (opHandlersPath, opHandler), 0) if opTimeout: self.timeOuts[opHandler]["PerOperation"] = opTimeout fileTimeout = gConfig.getValue("%s/%s/TimeOutPerFile" % (opHandlersPath, opHandler), 0) if fileTimeout: self.timeOuts[opHandler]["PerFile"] = fileTimeout self.handlersDict[opHandler] = opLocation self.log.info("Operation handlers:") for item in enumerate(self.handlersDict.items()): opHandler = item[1][0] self.log.info( "[%s] %s: %s (timeout: %d s + %d s per file)" % ( item[0], item[1][0], item[1][1], self.timeOuts[opHandler]["PerOperation"], self.timeOuts[opHandler]["PerFile"], ) ) if self.__rmsMonitoring: self.rmsMonitoringReporter = MonitoringReporter(monitoringType="RMSMonitoring") gThreadScheduler.addPeriodicTask(100, self.__rmsMonitoringReporting) # # create request dict self.__requestCache = dict() return S_OK()