def setup(self, decisionParams=None): """ method that sanitizes the decisionParams and ensures that at least it has the keys in `standardParamsDict`. This will be relevant while doing the matching with the RSS Policies configuration in the CS. There is one key-value pair, `active` which is added on this method. This allows policies to be de-activated from the CS, changing their active matchParam to something else than `Active`. examples: >>> pdp.setup( None ) >>> self.decisionParams { 'element' : None, 'name' : None, ... } >>> pdp.setup( { 'element' : 'AnElement' } ) >>> self.decisionParams { 'element' : 'AnElement', 'name' : None, ... } >>> pdp.setup( { 'NonStandardKey' : 'Something' } ) >>> self.decisionParams { 'NonStandardKey' : 'Something', 'element' : None,... } :Parameters: **decisionParams** - [ None, `dict` ] dictionary with the parameters to be matched with the RSS Policies configuration in the CS. """ standardParamsDict = { "element": None, "name": None, "elementType": None, "statusType": None, "status": None, "reason": None, "tokenOwner": None, # Last parameter allows policies to be de-activated "active": "Active", } if decisionParams is not None: standardParamsDict.update(decisionParams) if standardParamsDict["element"] is not None: self.log = gLogger.getSubLogger("PDP/%s" % standardParamsDict["element"]) if standardParamsDict["name"] is not None: self.log = gLogger.getSubLogger( "PDP/%s/%s" % (standardParamsDict["element"], standardParamsDict["name"]) ) self.log.verbose( "Setup - statusType: %s, status: %s" % (standardParamsDict["statusType"], standardParamsDict["status"]) ) self.decisionParams = standardParamsDict
def __init__( self, pilotAgentsDB = None, jobDB = None, tqDB = None, jlDB = None, opsHelper = None ): """ c'tor """ if pilotAgentsDB: self.pilotAgentsDB = pilotAgentsDB else: self.pilotAgentsDB = PilotAgentsDB() if jobDB: self.jobDB = jobDB else: self.jobDB = JobDB() if tqDB: self.tqDB = tqDB else: self.tqDB = TaskQueueDB() if jlDB: self.jlDB = jlDB else: self.jlDB = JobLoggingDB() if opsHelper: self.opsHelper = opsHelper else: self.opsHelper = Operations() self.log = gLogger.getSubLogger( "Matcher" ) self.limiter = Limiter( jobDB = self.jobDB, opsHelper = self.opsHelper )
def __init__( self ): self.log = gLogger.getSubLogger( "Script" ) #Set defaults for all workflow parameters here self.name = '' self.executable = '' self.logFile = '' self.arguments = ''
def __init__(self): super(PythiaAnalysis, self).__init__() self.NumberOfEvents = 1 self.enable = True self.STEP_NUMBER = '' self.debug = True self.log = gLogger.getSubLogger( "PythiaAnalysis" )
def __init__(self, dbname, fullname, debug=False): self.fullname = fullname database_name = dbname self.log = gLogger.getSubLogger(database_name) result = getDBParameters(fullname) if not result['OK']: raise RuntimeError('Cannot get database parameters: %s' % result['Message']) dbParameters = result['Value'] self.dbHost = dbParameters['Host'] self.dbPort = dbParameters['Port'] self.dbUser = dbParameters['User'] self.dbPass = dbParameters['Password'] self.dbName = dbParameters['DBName'] super(DB, self).__init__(hostName=self.dbHost, userName=self.dbUser, passwd=self.dbPass, dbName=self.dbName, port=self.dbPort, debug=debug) if not self._connected: raise RuntimeError("Can not connect to DB '%s', exiting..." % self.dbName) self.log.info("==================================================") self.log.info("User: "******"Host: " + self.dbHost) self.log.info("Port: " + str(self.dbPort)) #self.log.info("Password: "******"DBName: " + self.dbName) self.log.info("==================================================")
def getNewLDLibs(platform, application, applicationVersion): """ Prepare the LD_LIBRARY_PATH environment variable: make sure all lib folder are included :param string platform: System config used for the job :param string application: name of the application considered :param string applicationVersion: version of the application considered :return: new LD_LIBRARY_PATH """ log = gLogger.getSubLogger("GetLDLibs") log.verbose("Getting all lib folders") new_ld_lib_path = "" deps = resolveDeps(platform, application, applicationVersion) for dep in deps: res = getSoftwareFolder(platform, dep["app"], dep['version']) if not res['OK']: continue basedepfolder = res['Value'] if os.path.exists(os.path.join(basedepfolder, "lib")): log.verbose("Found lib folder in %s" % (basedepfolder)) newlibdir = os.path.join(basedepfolder, "lib") new_ld_lib_path = newlibdir ####Remove the libc removeLibc(new_ld_lib_path) if os.path.exists(os.path.join(basedepfolder, "LDLibs")): log.verbose("Found lib folder in %s" % (basedepfolder)) newlibdir = os.path.join(basedepfolder, "LDLibs") new_ld_lib_path = newlibdir ####Remove the libc removeLibc(new_ld_lib_path) if "LD_LIBRARY_PATH" in os.environ: if new_ld_lib_path: new_ld_lib_path = new_ld_lib_path + ":%s" % os.environ["LD_LIBRARY_PATH"] else: new_ld_lib_path = os.environ["LD_LIBRARY_PATH"] return new_ld_lib_path
def getNewPATH(platform, application, applicationVersion): """ Same as :func:`getNewLDLibs`,but for the PATH :param string platform: System config used for the job :param string application: name of the application considered :param string applicationVersion: version of the application considered :return: new PATH """ log = gLogger.getSubLogger("GetPaths") log.verbose("Getting all PATH folders") new_path = "" deps = resolveDeps(platform, application, applicationVersion) for dep in deps: res = getSoftwareFolder(platform, dep['app'], dep['version']) if not res['OK']: continue depfolder = res['Value'] if os.path.exists(os.path.join(depfolder, "bin")): log.verbose("Found bin folder in %s" % (depfolder)) newpathdir = os.path.join(depfolder, "bin") new_path = newpathdir if "PATH" in os.environ: if new_path: new_path = new_path + ":%s" % os.environ["PATH"] else: new_path = os.environ["PATH"] return new_path
def __init__( self, userName, password = '', tnsEntry = '', maxQueueSize = 100 ): """ set Oracle connection parameters and try to connect """ global gInstancesCount gInstancesCount += 1 self.__initialized = False self._connected = False if 'logger' not in dir( self ): self.logger = gLogger.getSubLogger( 'Oracle' ) # let the derived class decide what to do with if is not 1 self._threadsafe = cx_Oracle.threadsafety self.logger.debug( 'thread_safe = %s' % self._threadsafe ) self.__checkQueueSize( maxQueueSize ) self.__userName = str( userName ) self.__passwd = str( password ) self.__tnsName = str( tnsEntry ) # Create the connection Queue to reuse connections self.__connectionQueue = Queue.Queue( maxQueueSize ) # Create the connection Semaphore to limit total number of open connection self.__connectionSemaphore = threading.Semaphore( maxQueueSize ) self.__initialized = True self._connect()
def __init__(self, argumentsDict): """ Standard constructor """ self.arguments = argumentsDict self.name = COMPONENT_NAME self.log = gLogger.getSubLogger(self.name) self.ops = Operations()
def __init__( self, userCredPath, user, secret, endpointConfig, imageConfig): """ Constructor: uses user / secret authentication for the time being. copy the endpointConfig and ImageConfig dictionaries to the OcciClient :Parameters: **userCredPath** - `string` path to a valid x509 proxy **endpointConfig** - `dict` dictionary with the endpoint configuration ( WMS.Utilities.Configuration.OcciConfiguration ) **imageConfig** - `dict` dictionary with the image configuration ( WMS.Utilities.Configuration.ImageConfiguration ) """ # logger self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.endpointConfig = endpointConfig self.imageConfig = imageConfig self.__userCredPath = userCredPath self.__user = user self.__password = secret if userCredPath is not None: self.__authArg = ' --auth x509 --user-cred ' + self.__userCredPath + ' --voms ' else: self.__authArg = ' --auth basic --username %s --password %s ' % (self.__user, self.__password)
def __init__( self, mqManager, mqURI, producerId ): self._connectionManager = mqManager self._mqURI = mqURI self._destination = getDestinationAddress( self._mqURI ) self._mqService = getMQService( self._mqURI ) self._id = producerId self.log = gLogger.getSubLogger( self.__class__.__name__ )
def __init__( self, transClient = None, logger = None, requestClient = None, requestClass = None, requestValidator = None ): """ c'tor the requestClass is by default Request. If extensions want to use an extended type, they can pass it as a parameter. This is the same behavior as WorfkloTasks and jobClass """ if not logger: logger = gLogger.getSubLogger( 'RequestTasks' ) super( RequestTasks, self ).__init__( transClient, logger ) if not requestClient: self.requestClient = ReqClient() else: self.requestClient = requestClient if not requestClass: self.requestClass = Request else: self.requestClass = requestClass if not requestValidator: self.requestValidator = RequestValidator() else: self.requestValidator = requestValidator
def __init__( self, monitor = None ): self.__idMap = {} self.__execTypes = {} self.__executorsLock = threading.Lock() self.__tasksLock = threading.Lock() self.__freezerLock = threading.Lock() self.__tasks = {} self.__log = gLogger.getSubLogger( "ExecMind" ) self.__taskFreezer = [] self.__queues = ExecutorQueues( self.__log ) self.__states = ExecutorState( self.__log ) self.__cbHolder = ExecutorDispatcherCallbacks() self.__monitor = monitor gThreadScheduler.addPeriodicTask( 60, self.__doPeriodicStuff ) #If a task is frozen too many times, send error or forget task? self.__failedOnTooFrozen = True #If a task fails to properly dispatch, freeze or forget task? self.__freezeOnFailedDispatch = True #If a task needs to go to an executor that has not connected. Freeze or forget the task? self.__freezeOnUnknownExecutor = True if self.__monitor: self.__monitor.registerActivity( "executors", "Executor reactors connected", "Executors", "executors", self.__monitor.OP_MEAN, 300 ) self.__monitor.registerActivity( "tasks", "Tasks processed", "Executors", "tasks", self.__monitor.OP_RATE, 300 ) self.__monitor.registerActivity( "taskTime", "Task processing time", "Executors", "seconds", self.__monitor.OP_MEAN, 300 )
def __init__( self, loggerIn = None ): """ Initialization of module base. loggerIn is a logger object that can be passed so that the logging will be more clear. """ if not loggerIn: self.log = gLogger.getSubLogger( 'ModuleBase' ) else: self.log = loggerIn # These 2 are used in many places, so it's good to have them available here. self.opsH = Operations() self.dm = DataManager() # Some job parameters self.production_id = 0 self.prod_job_id = 0 self.jobID = 0 self.step_number = 0 self.step_id = 0 self.jobType = '' self.executable = '' self.command = None self.workflowStatus = None self.stepStatus = None self.workflow_commons = None self.step_commons = None # These are useful objects (see the getFileReporter(), getJobReporter() and getRequestContainer() functions) self.fileReport = None self.jobReport = None self.request = None
def resolveDeps(sysconfig, appli, appversion): """ Resolve the dependencies :param str sysconfig: system configuration :param str appli: application name :param str appversion: application version :return: list of dictionaries """ log = gLogger.getSubLogger("resolveDeps") ops = Operations() deps = ops.getSections('/AvailableTarBalls/%s/%s/%s/Dependencies' % (sysconfig, appli, appversion), '') depsarray = [] if deps['OK']: for dep in deps['Value']: vers = ops.getValue('/AvailableTarBalls/%s/%s/%s/Dependencies/%s/version' % (sysconfig, appli, appversion, dep), '') depvers = '' if vers: depvers = vers else: log.error("Retrieving dependency version for %s failed, skipping to next !" % (dep)) continue log.verbose("Found dependency %s %s" % (dep, depvers)) depdict = {} depdict["app"] = dep depdict["version"] = depvers depsarray.append(depdict) ##resolve recursive dependencies depsofdeps = resolveDeps(sysconfig, dep, depvers) depsarray.extend(depsofdeps) else: log.verbose("Could not find any dependency for %s %s, ignoring" % (appli, appversion)) return depsarray
def __init__( self, dbname, fullname, debug = False ): self.fullname = fullname database_name = dbname self.log = gLogger.getSubLogger( database_name ) result = getDBParameters( fullname ) if not result['OK'] : raise RuntimeError( 'Cannot get database parameters: %s' % result['Message'] ) dbParameters = result[ 'Value' ] self.dbHost = dbParameters[ 'Host' ] self.dbPort = dbParameters[ 'Port' ] self.dbUser = dbParameters[ 'User' ] self.dbPass = dbParameters[ 'Password' ] self.dbName = dbParameters[ 'DBName' ] MySQL.__init__( self, self.dbHost, self.dbUser, self.dbPass, self.dbName, self.dbPort, debug = debug ) if not self._connected: raise RuntimeError( 'Can not connect to DB %s, exiting...' % self.dbName ) self.log.info( "==================================================" ) #self.log.info("SystemInstance: "+self.system) self.log.info( "User: "******"Host: " + self.dbHost ) self.log.info( "Port: " + str( self.dbPort ) ) #self.log.info("Password: "******"DBName: " + self.dbName ) self.log.info( "==================================================" )
def __init__( self, imageName, endPoint ): """ Constructor: uses NovaConfiguration to parse the endPoint CS configuration and ImageConfiguration to parse the imageName CS configuration. :Parameters: **imageName** - `string` imageName as defined on CS:/Resources/VirtualMachines/Images **endPoint** - `string` endPoint as defined on CS:/Resources/VirtualMachines/CloudEndpoint """ # logger self.log = gLogger.getSubLogger( 'NovaImage %s: ' % imageName ) self.imageName = imageName self.endPoint = endPoint # their config() method returns a dictionary with the parsed configuration # they also provide a validate() method to make sure it is correct self.__imageConfig = ImageConfiguration( imageName ) self.__novaConfig = NovaConfiguration( endPoint ) # this object will connect to the server. Better keep it private. self.__clinova = None
def __init__(self, storageName, parameters): """ """ super(GFAL2_SRM2Storage, self).__init__(storageName, parameters) self.log = gLogger.getSubLogger("GFAL2_SRM2Storage", True) self.log.debug("GFAL2_SRM2Storage.__init__: Initializing object") self.pluginName = 'GFAL2_SRM2' # This attribute is used to know the file status (OFFLINE,NEARLINE,ONLINE) self._defaultExtendedAttributes = ['user.status'] # ## # Setting the default SRM parameters here. For methods where this # is not the default there is a method defined in this class, setting # the proper values and then calling the base class method. # ## self.gfal2requestLifetime = gConfig.getValue('/Resources/StorageElements/RequestLifeTime', 100) self.__setSRMOptionsToDefault() # This lists contains the list of protocols to ask to SRM to get a URL # It can be either defined in the plugin of the SE, or as a global option if 'ProtocolsList' in parameters: self.protocolsList = parameters['ProtocolsList'].split(',') else: self.log.debug("GFAL2_SRM2Storage: No protocols provided, using the default protocols.") self.protocolsList = self.defaultLocalProtocols self.log.debug('GFAL2_SRM2Storage: protocolsList = %s' % self.protocolsList)
def __init__( self, storageName, parameters ): """ c'tor :param self: self reference :param str storageName: SE name :param str protocol: protocol to use :param str rootdir: base path for vo files :param str host: SE host :param int port: port to use to communicate with :host: :param str spaceToken: space token :param str wspath: location of SRM on :host: """ # # init base class super( GFAL2_XROOTStorage, self ).__init__( storageName, parameters ) self.srmSpecificParse = False self.log = gLogger.getSubLogger( "GFAL2_XROOTStorage", True ) self.pluginName = 'GFAL2_XROOT' # why is this here ?! self.protocolParameters['WSUrl'] = 0 self.protocolParameters['SpaceToken'] = 0 # We don't need extended attributes for metadata self._defaultExtendedAttributes = None
def __init__( self, baseURL = "" ): self.__baseURL = baseURL.strip( "/" ) self.__routes = [] self.__handlers = [] self.__setupGroupRE = r"(?:/s:([\w-]*)/g:([\w.-]*))?" self.__shySetupGroupRE = r"(?:/s:(?:[\w-]*)/g:(?:[\w.-]*))?" self.log = gLogger.getSubLogger( "Routing" )
def __init__( self, user, secret, endpointConfig, imageConfig ): """ Multiple constructor depending on the passed parameters :Parameters: **user** - `string` username that will be used on the authentication **secret** - `string` password used on the authentication If secret is None then user actually is: **proxyPath** - `string` path to the valid X509 proxy **endpointConfig** - `dict` dictionary with the endpoint configuration ( WMS.Utilities.Configuration.NovaConfiguration ) **imageConfig** - `dict` dictionary with the image configuration ( WMS.Utilities.Configuration.ImageConfiguration ) """ # logger self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.endpointConfig = endpointConfig self.imageConfig = imageConfig # Variables needed to contact the service ex_force_auth_url = endpointConfig.get( 'ex_force_auth_url', None ) ex_force_service_region = endpointConfig.get( 'ex_force_service_region', None ) ex_force_auth_version = endpointConfig.get( 'ex_force_auth_version', None ) ex_tenant_name = endpointConfig.get( 'ex_tenant_name', None ) # we force SSL cacert, if defined ex_force_ca_cert = endpointConfig.get( 'ex_force_ca_cert', None ) if ex_force_ca_cert is not None: security.CA_CERTS_PATH = [ ex_force_ca_cert ] # get openstack driver openstack_driver = get_driver( Provider.OPENSTACK ) if secret == None: # with VOMS (from Alvaro Lopez trunk https://github.com/alvarolopez/libcloud/blob/trunk): proxyPath=user username = password = None self.__driver = openstack_driver( username, password, ex_force_auth_url = ex_force_auth_url, ex_force_service_region = ex_force_service_region, ex_force_auth_version = ex_force_auth_version, ex_tenant_name = ex_tenant_name, ex_voms_proxy = proxyPath ) else: # with user password username = user password = secret self.__driver = openstack_driver( username, password, ex_force_auth_url = ex_force_auth_url, ex_force_service_region = ex_force_service_region, ex_force_auth_version = ex_force_auth_version, ex_tenant_name = ex_tenant_name )
def __init__( self, connectionStorage = None ): self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.__lock = None if connectionStorage: self.__connectionStorage = connectionStorage else: self.__connectionStorage = {}
def __init__(self, name, decisionParams, enforcementResult, singlePolicyResults, clients): # enforcementResult supposed to look like: # { # 'Status' : <str>, # 'Reason' : <str>, # 'PolicyActions' : <list>, # [ 'EndDate' : <str> ] # } # decisionParams supposed to look like: # { # 'element' : None, # 'name' : None, # 'elementType' : None, # 'statusType' : None, # 'status' : None, # 'reason' : None, # 'tokenOwner' : None # } self.actionName = name # 'BaseAction' self.decisionParams = decisionParams self.enforcementResult = enforcementResult self.singlePolicyResults = singlePolicyResults self.clients = clients self.log = gLogger.getSubLogger(self.__class__.__name__) self.log.verbose("Running %s action" % self.__class__.__name__)
def __init__(self): super(WhizardAnalysis, self).__init__() self.enable = True self.STEP_NUMBER = '' self.debug = True self.log = gLogger.getSubLogger( "WhizardAnalysis" ) self.SteeringFile = '' self.OutputFile = '' self.NumberOfEvents = 1 self.Lumi = 0 self.applicationName = 'whizard' self.evttype = "" self.RandomSeed = 0 self.getProcessInFile = False self.datMan = DataManager() self.processlist = None self.parameters = {} self.susymodel = 0 self.Model = '' self.genmodel = GeneratorModels() self.eventstring = ['! ', 'Fatal error:', 'PYSTOP', 'No matrix element available', 'Floating point exception', 'Event generation finished.', " n_events","luminosity", " sum "] self.excludeAllButEventString = False self.steeringparameters = '' self.options = None self.optionsdict = {} self.OptionsDictStr = '' self.GenLevelCutDictStr = '' self.genlevelcuts = {} self.willCut = False self.useGridFiles = False
def __init__(self): ''' Constructor ''' self.log = gLogger.getSubLogger("WrapperCall") self.ops = Operations("glast.org") self.stdError = ''
def __init__(self, transClient=None, logger=None, requestClient=None, requestClass=None, requestValidator=None, ownerDN=None, ownerGroup=None): """ c'tor the requestClass is by default Request. If extensions want to use an extended type, they can pass it as a parameter. This is the same behavior as WorfkloTasks and jobClass """ if not logger: logger = gLogger.getSubLogger('RequestTasks') super(RequestTasks, self).__init__(transClient, logger) useCertificates = True if (bool(ownerDN) and bool(ownerGroup)) else False if not requestClient: self.requestClient = ReqClient(useCertificates=useCertificates, delegatedDN=ownerDN, delegatedGroup=ownerGroup) else: self.requestClient = requestClient if not requestClass: self.requestClass = Request else: self.requestClass = requestClass if not requestValidator: self.requestValidator = RequestValidator() else: self.requestValidator = requestValidator
def __init__(self): super(OverlayInput, self).__init__() self.enable = True self.STEP_NUMBER = '' self.log = gLogger.getSubLogger( "OverlayInput" ) self.applicationName = 'OverlayInput' self.curdir = os.getcwd() self.applicationLog = '' self.printoutflag = '' self.prodid = 0 self.detector = '' ##needed for backward compatibility self.detectormodel = "" self.energytouse = '' self.energy = 0 self.nbofeventsperfile = 100 self.lfns = [] self.nbfilestoget = 0 self.BkgEvtType = 'gghad' self.BXOverlay = 0 self.ggtohadint = 3.2 self.nbsigeventsperfile = 0 self.nbinputsigfile = 1 self.NbSigEvtsPerJob = 0 self.rm = ReplicaManager() self.fc = FileCatalogClient() self.site = DIRAC.siteName() self.machine = 'clic_cdr'
def __init__( self, instanceId, URL, accessKey, secretKey ): self.log = gLogger.getSubLogger( "CloudStackInstance id:%s" % instanceId ) self.__errorStatus = "" self.__instanceId = instanceId self.__csInstance = False self.__imageId = "unknown" self.__cloudStackURI = URL self.__csAccessKey = accessKey self.__csSecretKey = secretKey self.__cliCloudStack = CloudStackClient( self.__cloudStackURI, self.__csAccessKey, self.__csSecretKey ) #Try connection request = check_connection( self.__cloudStackURI, self.__csAccessKey, self.__csSecretKey ) if request.returncode != 0: self.__errorStatus = "Can't connect to CloudStack server %s\n%s" % ( self.__cloudStackURI, request.stdout ) self.log.error( self.__errorStatus ) return request = self.__cliCloudStack.get_image_ids_of_instance( self.__instanceId ) if request.returncode != 0: self.__errorStatus = "Cannot find instance %s" % self.__instanceId self.log.error( self.__errorStatus ) return self.__csInstance = request.stdout self.log.info( "CloudStack VM instance id %s initialized from image(%s)" % ( self.__instanceId, self.__csInstance ) ) return
def __init__(self): """Module initialization. """ super(ComputeOutputDataList, self).__init__() self.version = __RCSID__ self.log = gLogger.getSubLogger( "ComputeOutputData" ) self.listoutput = []
def getDIRACGOCDictionary(): """ Create a dictionary containing DIRAC site names and GOCDB site names using a configuration provided by CS. :return: A dictionary of DIRAC site names (key) and GOCDB site names (value). """ log = gLogger.getSubLogger( 'getDIRACGOCDictionary' ) log.debug( 'Begin function ...' ) result = gConfig.getConfigurationTree( '/Resources/Sites', 'Name' ) if not result['OK']: log.error( "getConfigurationTree() failed with message: %s" % result['Message'] ) return S_ERROR( 'Configuration is corrupted' ) siteNamesTree = result['Value'] dictionary = dict() PATHELEMENTS = 6 # site names have 6 elements in the path, i.e.: # /Resource/Sites/<GRID NAME>/<DIRAC SITE NAME>/Name # [0]/[1] /[2] /[3] /[4] /[5] for path, gocdbSiteName in siteNamesTree.iteritems(): elements = path.split( '/' ) if len( elements ) <> PATHELEMENTS: continue diracSiteName = elements[PATHELEMENTS - 2] dictionary[diracSiteName] = gocdbSiteName log.debug( 'End function.' ) return S_OK( dictionary )
class CachedJobState(object): log = gLogger.getSubLogger("CachedJobState") def __init__(self, jid, skipInitState=False): self.dOnlyCache = False self.__jid = jid self.__jobState = JobState(jid) self.cleanState(skipInitState=skipInitState) def cleanState(self, skipInitState=False): self.__cache = {} self.__jobLog = [] self.__insertIntoTQ = False self.__dirtyKeys = set() self.__manifest = False self.__initState = None self.__lastValidState = time.time() if not skipInitState: result = self.getAttributes( ["Status", "MinorStatus", "LastUpdateTime"]) if result['OK']: self.__initState = result['Value'] else: self.__initState = None def recheckValidity(self, graceTime=600): now = time.time() if graceTime <= 0 or now - self.__lastValidState > graceTime: self.__lastValidState = now result = self.__jobState.getAttributes( ["Status", "MinorStatus", "LastUpdateTime"]) if not result['OK']: return result currentState = result['Value'] if not currentState == self.__initState: return S_OK(False) return S_OK(True) return S_OK(self.valid) @property def valid(self): return self.__initState != None @property def jid(self): return self.__jid def getDirtyKeys(self): return set(self.__dirtyKeys) def commitChanges(self): if self.__initState == None: return S_ERROR("CachedJobState( %d ) is not valid" % self.__jid) changes = {} for k in self.__dirtyKeys: changes[k] = self.__cache[k] result = self.__jobState.commitCache(self.__initState, changes, self.__jobLog) try: result.pop('rpcStub') except KeyError: pass if not result['OK']: self.cleanState() return result if not result['Value']: self.cleanState() return S_ERROR("Initial state was different") newState = result['Value'] self.__jobLog = [] self.__dirtyKeys.clear() #Save manifest if self.__manifest and self.__manifest.isDirty(): result = self.__jobState.setManifest(self.__manifest) if not result['OK']: self.cleanState() for i in range(5): if self.__jobState.rescheduleJob()['OK']: break return result self.__manifest.clearDirty() #Insert into TQ if self.__insertIntoTQ: result = self.__jobState.insertIntoTQ() if not result['OK']: self.cleanState() for i in range(5): if self.__jobState.rescheduleJob()['OK']: break return result self.__insertIntoTQ = False self.__initState = newState self.__lastValidState = time.time() return S_OK() def serialize(self): if self.__manifest: manifest = (self.__manifest.dumpAsCFG(), self.__manifest.isDirty()) else: manifest = None return DEncode.encode( (self.__jid, self.__cache, self.__jobLog, manifest, self.__initState, self.__insertIntoTQ, tuple(self.__dirtyKeys))) @staticmethod def deserialize(stub): dataTuple, slen = DEncode.decode(stub) if len(dataTuple) != 7: return S_ERROR("Invalid stub") #jid if type(dataTuple[0]) not in (types.IntType, types.LongType): return S_ERROR("Invalid stub 0") #cache if type(dataTuple[1]) != types.DictType: return S_ERROR("Invalid stub 1") #trace if type(dataTuple[2]) != types.ListType: return S_ERROR("Invalid stub 2") #manifest tdt3 = type(dataTuple[3]) if tdt3 != types.NoneType and (tdt3 != types.TupleType and len(dataTuple[3]) != 2): return S_ERROR("Invalid stub 3") #initstate if type(dataTuple[4]) != types.DictType: return S_ERROR("Invalid stub 4") #Insert into TQ if type(dataTuple[5]) != types.BooleanType: return S_ERROR("Invalid stub 5") #Dirty Keys if type(dataTuple[6]) != types.TupleType: return S_ERROR("Invalid stub 6") cjs = CachedJobState(dataTuple[0], skipInitState=True) cjs.__cache = dataTuple[1] cjs.__jobLog = dataTuple[2] dt3 = dataTuple[3] if dataTuple[3]: manifest = JobManifest() result = manifest.loadCFG(dt3[0]) if not result['OK']: return result if dt3[1]: manifest.setDirty() else: manifest.clearDirty() cjs.__manifest = manifest cjs.__initState = dataTuple[4] cjs.__insertIntoTQ = dataTuple[5] cjs.__dirtyKeys = set(dataTuple[6]) return S_OK(cjs) def __cacheAdd(self, key, value): self.__cache[key] = value self.__dirtyKeys.add(key) def __cacheExists(self, keyList): if type(keyList) in types.StringTypes: keyList = [keyList] for key in keyList: if key not in self.__cache: return False return True def __cacheResult(self, cKey, functor, fArgs=None): keyType = type(cKey) #If it's a string if keyType in types.StringTypes: if cKey not in self.__cache: if self.dOnlyCache: return S_ERROR("%s is not cached") if not fArgs: fArgs = tuple() result = functor(*fArgs) if not result['OK']: return result data = result['Value'] self.__cache[cKey] = data return S_OK(self.__cache[cKey]) #Tuple/List elif keyType in (types.ListType, types.TupleType): if not self.__cacheExists(cKey): if self.dOnlyCache: return S_ERROR("%s is not cached") if not fArgs: fArgs = tuple() result = functor(*fArgs) if not result['OK']: return result data = result['Value'] if len(cKey) != len(data): gLogger.warn( "CachedJobState.__memorize( %s, %s = %s ) doesn't receive the same amount of values as keys" % (cKey, functor, data)) return data for i in range(len(cKey)): self.__cache[cKey[i]] = data[i] #Prepare result return S_OK(tuple([self.__cache[cK] for cK in cKey])) else: raise RuntimeError("Cache key %s does not have a valid type" % cKey) def __cacheDict(self, prefix, functor, keyList=None): if not keyList or not self.__cacheExists( ["%s.%s" % (prefix, key) for key in keyList]): result = functor(keyList) if not result['OK']: return result data = result['Value'] for key in data: cKey = "%s.%s" % (prefix, key) #If the key is already in the cache. DO NOT TOUCH. User may have already modified it. #We update the coming data with the cached data if cKey in self.__cache: data[key] = self.__cache[cKey] else: self.__cache[cKey] = data[key] return S_OK(data) return S_OK( dict([(key, self.__cache["%s.%s" % (prefix, key)]) for key in keyList])) def _inspectCache(self): return copy.deepcopy(self.__cache) def _clearCache(self): self.__cache = {} @property def _internals(self): if self.__manifest: manifest = (self.__manifest.dumpAsCFG(), self.__manifest.isDirty()) else: manifest = None return (self.__jid, self.dOnlyCache, dict(self.__cache), list(self.__jobLog), manifest, dict(self.__initState), list(self.__dirtyKeys)) # # Manifest # def getManifest(self): if not self.__manifest: result = self.__jobState.getManifest() if not result['OK']: return result self.__manifest = result['Value'] return S_OK(self.__manifest) def setManifest(self, manifest): if not isinstance(manifest, JobManifest): jobManifest = JobManifest() result = jobManifest.load(str(manifest)) if not result['OK']: return result manifest = jobManifest manifest.setDirty() self.__manifest = manifest # self.__manifest.clearDirty() return S_OK() # Attributes # def __addLogRecord(self, majorStatus=None, minorStatus=None, appStatus=None, source=None): record = {} if majorStatus: record['status'] = majorStatus if minorStatus: record['minor'] = minorStatus if appStatus: record['application'] = appStatus if not record: return if not source: source = "Unknown" self.__jobLog.append((record, Time.dateTime(), source)) def setStatus(self, majorStatus, minorStatus=None, appStatus=None, source=None): self.__cacheAdd('att.Status', majorStatus) if minorStatus: self.__cacheAdd('att.MinorStatus', minorStatus) if appStatus: self.__cacheAdd('att.ApplicationStatus', appStatus) self.__addLogRecord(majorStatus, minorStatus, appStatus, source) return S_OK() def setMinorStatus(self, minorStatus, source=None): self.__cacheAdd('att.MinorStatus', minorStatus) self.__addLogRecord(minorStatus=minorStatus, source=source) return S_OK() def getStatus(self): return self.__cacheResult(('att.Status', 'att.MinorStatus'), self.__jobState.getStatus) def setAppStatus(self, appStatus, source=None): self.__cacheAdd('att.ApplicationStatus', appStatus) self.__addLogRecord(appStatus=appStatus, source=source) return S_OK() def getAppStatus(self): return self.__cacheResult('att.ApplicationStatus', self.__jobState.getAppStatus) # # Attribs # def setAttribute(self, name, value): if type(name) not in types.StringTypes: return S_ERROR("Attribute name has to be a string") self.__cacheAdd("att.%s" % name, value) return S_OK() def setAttributes(self, attDict): if type(attDict) != types.DictType: return S_ERROR("Attributes has to be a dictionary and it's %s" % str(type(attDict))) for key in attDict: self.__cacheAdd("att.%s" % key, attDict[key]) return S_OK() def getAttribute(self, name): return self.__cacheResult('att.%s' % name, self.__jobState.getAttribute, (name, )) def getAttributes(self, nameList=None): return self.__cacheDict('att', self.__jobState.getAttributes, nameList) #Job params def setParameter(self, name, value): if type(name) not in types.StringTypes: return S_ERROR("Job parameter name has to be a string") self.__cacheAdd('jobp.%s' % name, value) return S_OK() def setParameters(self, pDict): if type(pDict) != types.DictType: return S_ERROR("Job parameters has to be a dictionary") for key in pDict: self.__cacheAdd('jobp.%s' % key, pDict[key]) return S_OK() def getParameter(self, name): return self.__cacheResult("jobp.%s" % name, self.__jobState.getParameter, (name, )) def getParameters(self, nameList=None): return self.__cacheDict('jobp', self.__jobState.getParameters, nameList) #Optimizer params def setOptParameter(self, name, value): if type(name) not in types.StringTypes: return S_ERROR("Optimizer parameter name has to be a string") self.__cacheAdd('optp.%s' % name, value) return S_OK() def setOptParameters(self, pDict): if type(pDict) != types.DictType: return S_ERROR("Optimizer parameters has to be a dictionary") for key in pDict: self.__cacheAdd('optp.%s' % key, pDict[key]) return S_OK() def getOptParameter(self, name): return self.__cacheResult("optp.%s" % name, self.__jobState.getOptParameter, (name, )) def getOptParameters(self, nameList=None): return self.__cacheDict('optp', self.__jobState.getOptParameters, nameList) #Other def resetJob(self, source=""): """ Reset the job! """ result = self.__jobState.resetJob(source=source) if result['OK']: self.__resetState() return result def getInputData(self): return self.__cacheResult("inputData", self.__jobState.getInputData) def insertIntoTQ(self): if self.valid: self.__insertIntoTQ = True return S_OK() return S_ERROR("Cached state is invalid")
def run(): ''' Main method ''' subLogger.info('Checking DB...') installDB() subLogger.info('[done]') #............................................................................... if __name__ == "__main__": subLogger = gLogger.getSubLogger(__file__) # Script initialization registerSwitches() registerUsageMessage() parseSwitches() # Run the script run() # Bye DIRACExit(0) ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def __init__(self): """ Standard constructor """ self.log = gLogger.getSubLogger(__name__)
def __init__(self): """ Standard constructor """ self.version = platform.uname() self.log = gLogger.getSubLogger( 'WatchdogFactory' ) self.watchDogsLocation = 'DIRAC.WorkloadManagementSystem.JobWrapper'
def __init__(self, transferClient=False, **kwargs): super(BundleDeliveryClient, self).__init__(**kwargs) self.setServer('Framework/BundleDelivery') self.transferClient = transferClient self.log = gLogger.getSubLogger("BundleDelivery")
def __init__(self): # constructor code self.version = __RCSID__ self.log = gLogger.getSubLogger('DISETRequest') self.portalEnv = False
def setParameters(self, parameters): self.log = gLogger.getSubLogger('%s/%s' % (__name__, parameters['ProviderName'])) self.parameters = parameters self.oauth2 = OAuth2(parameters['ProviderName'])
""" import errno # from DIRAC from DIRAC import S_OK, S_ERROR, gLogger from DIRAC.Core.DISET.RequestHandler import RequestHandler from DIRAC.Core.DISET.ThreadConfig import ThreadConfig from DIRAC.Core.Utilities.ReturnValues import returnSingleResult from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.Resources.Storage.StorageElement import StorageElement ######################################################################## LOG = gLogger.getSubLogger(__name__) class S3GatewayHandler(RequestHandler): """ .. class:: S3GatewayHandler """ # FC instance to check whether access is permitted or not _fc = None # Mapping between the S3 methods and the DFC methods _s3ToFC_methods = { "head_object": "getFileMetadata", "get_object":
def __init__(self, paramdict=None): """ Can define the full application by passing a dictionary in the constructor. >>> app = Application({"appname":"marlin","version":"v0111Prod", ... "steeringFile":"My_file.xml","numberOfEvents":1000}) :param dict paramdict: Dictionary of parameters that can be set. Will throw an exception if one of them does not exist. """ super(Application, self).__init__() #application nane (executable) self.appname = "" #application version self.version = "" #Steering file (duh!) self.steeringFile = "" #Input sandbox: steering file automatically added to SB self.inputSB = [] #Input file self.inputFile = "" #Output file self.outputFile = "" self.outputPath = "" self.outputDstFile = '' self.outputDstPath = '' self.outputRecFile = '' self.outputRecPath = '' self.outputSE = '' self._listofoutput = [] #Log file self.logFile = "" #Detector type (ILD or SID) self.detectortype = "" #Data type : gen, SIM, REC, DST self.datatype = "" #Extra command line arguments self.extraCLIArguments = "" ##Needed for Generation+Stdhepcut self.willBeCut = False ##Debug mode self.debug = False #Prod Parameters: things that appear on the prod details self.prodparameters = {} self.accountInProduction = True #Module name and description: Not to be set by the users, internal call only, used to get the Module objects self._modulename = '' self._moduledescription = '' self._importLocation = "ILCDIRAC.Workflow.Modules" #System Configuration: comes from Job definition self._platform = '' #Internal member: hold the list of the job's application set before self: used when using getInputFromApp self._job = None self._jobapps = [] self._jobsteps = [] self._jobtype = '' #input application: will link the OutputFile of the guys in there with the InputFile of the self self._inputapp = [] self._linkedidx = None #Needed to link the parameters. self._inputappstep = None #flag set to true in Job.append self.addedtojob = False ####Following are needed for error report self._log = gLogger.getSubLogger(self.__class__.__name__) self._errorDict = {} #This is used to filter out the members that should not be set when using a dict as input self._paramsToExclude = [ '_paramsToExclude', "_log", "_errorDict", "addedtojob", '_ops', "_inputappstep", "_linkedidx", "_inputapp", "_jobtype", "_jobsteps", "_jobapps", "_job", "_platform", "_importLocation", "_moduledescription", "_modulename", "prodparameters", "datatype", "detectortype", "_listofoutput", "inputSB", "appname", 'accountInProduction', 'outputPath' ] ### Next is to use the setattr method. self._setparams(paramdict)
def __init__(self, configSection, channels=None, bandwidths=None, failedFiles=None): """c'tor :param self: self reference :param str configSection: path on CS to ReplicationScheduler agent :param bandwithds: observed throughput on active channels :param channels: active channels :param int failedFiles: max number of distinct failed files to allow scheduling """ ## save config section self.configSection = configSection + "/" + self.__class__.__name__ ## ## sublogger self.log = gLogger.getSubLogger("StrategyHandler", child=True) self.log.setLevel( gConfig.getValue(self.configSection + "/LogLevel", "DEBUG")) self.supportedStrategies = [ 'Simple', 'DynamicThroughput', 'Swarm', 'MinimiseTotalWait' ] self.log.info("Supported strategies = %s" % ", ".join(self.supportedStrategies)) self.sigma = gConfig.getValue(self.configSection + '/HopSigma', 0.0) self.log.info("HopSigma = %s" % self.sigma) self.schedulingType = gConfig.getValue( self.configSection + '/SchedulingType', 'File') self.log.info("SchedulingType = %s" % self.schedulingType) self.activeStrategies = gConfig.getValue( self.configSection + '/ActiveStrategies', ['MinimiseTotalWait']) self.log.info("ActiveStrategies = %s" % ", ".join(self.activeStrategies)) self.numberOfStrategies = len(self.activeStrategies) self.log.info("Number of active strategies = %s" % self.numberOfStrategies) self.acceptableFailureRate = gConfig.getValue( self.configSection + '/AcceptableFailureRate', 75) self.log.info("AcceptableFailureRate = %s" % self.acceptableFailureRate) self.acceptableFailedFiles = gConfig.getValue( self.configSection + "/AcceptableFailedFiles", 5) self.log.info("AcceptableFailedFiles = %s" % self.acceptableFailedFiles) self.rwUpdatePeriod = gConfig.getValue( self.configSection + "/RssRWUpdatePeriod", 600) self.log.info("RSSUpdatePeriod = %s s" % self.rwUpdatePeriod) self.rwUpdatePeriod = datetime.timedelta(seconds=self.rwUpdatePeriod) ## bandwithds self.bandwidths = bandwidths if bandwidths else {} ## channels self.channels = channels if channels else {} ## distinct failed files per channel self.failedFiles = failedFiles if failedFiles else {} ## chosen strategy self.chosenStrategy = 0 ## fts graph self.ftsGraph = None ## timestamp for last update self.lastRssUpdate = datetime.datetime.now() # dispatcher self.strategyDispatcher = { "MinimiseTotalWait": self.minimiseTotalWait, "DynamicThroughput": self.dynamicThroughput, "Simple": self.simple, "Swarm": self.swarm } ## own RSS client self.resourceStatus = ResourceStatus() ## create fts graph ftsGraph = self.setup(self.channels, self.bandwidths, self.failedFiles) if not ftsGraph["OK"]: raise SHGraphCreationError(ftsGraph["Message"]) self.log.info("%s has been constructed" % self.__class__.__name__)
def initialize(self): super(MonitoringHistoryCorrector, self).initialize() self.log = gLogger.getSubLogger("MonitoringHistoryCorrector") return S_OK()
def __init__(self,ceType=''): """ Standard constructor """ self.ceType = ceType self.log = gLogger.getSubLogger( self.ceType )
def setParameters(self, params): """ Set the transformation parameters and extract transID """ self.params = params self.transID = params['TransformationID'] self.log = gLogger.getSubLogger(self.plugin + self.transInThread.get( self.transID, ' [NoThread] [%d] ' % self.transID))
def __init__(self, catalogName): """ Standard constructor """ self.fileName = catalogName self.name = COMPONENT_NAME self.log = gLogger.getSubLogger(self.name)
def __init__(self): """ Standard constructor """ self.log = gLogger.getSubLogger('ProcessMonitor') self.osType = platform.uname()
def moveFilesToDerivedTransformation(self, transDict, resetUnused=True): """ move files input to a transformation, to the derived one """ prod = transDict['TransformationID'] parentProd = int(transDict.get('InheritedFrom', 0)) movedFiles = {} log = gLogger.getSubLogger( "[None] [%d] .moveFilesToDerivedTransformation:" % prod) if not parentProd: log.warn("Transformation was not derived...") return S_OK((parentProd, movedFiles)) # get the lfns in status Unused/MaxReset of the parent production res = self.getTransformationFiles(condDict={ 'TransformationID': parentProd, 'Status': ['Unused', 'MaxReset'] }) if not res['OK']: log.error(" Error getting Unused files from transformation", "%d: %s" % (parentProd, res['Message'])) return res parentFiles = res['Value'] lfns = [lfnDict['LFN'] for lfnDict in parentFiles] if not lfns: log.info(" No files found to be moved from transformation", "%d" % parentProd) return S_OK((parentProd, movedFiles)) # get the lfns of the derived production that were Unused/MaxReset in the parent one res = self.getTransformationFiles(condDict={ 'TransformationID': prod, 'LFN': lfns }) if not res['OK']: log.error(" Error getting files from derived transformation:", res['Message']) return res derivedFiles = res['Value'] derivedStatusDict = dict((derivedDict['LFN'], derivedDict['Status']) for derivedDict in derivedFiles) newStatusFiles = {} parentStatusFiles = {} badStatusFiles = {} for parentDict in parentFiles: lfn = parentDict['LFN'] derivedStatus = derivedStatusDict.get(lfn) if derivedStatus: parentStatus = parentDict['Status'] # By default move to the parent status (which is Unused or MaxReset) status = parentStatus moveStatus = parentStatus # For MaxReset, set Unused if requested if parentStatus == 'MaxReset': if resetUnused: status = 'Unused' moveStatus = 'Unused from MaxReset' else: status = 'MaxReset-inherited' if derivedStatus.endswith('-inherited'): # This is the general case newStatusFiles.setdefault((status, parentStatus), []).append(lfn) movedFiles[moveStatus] = movedFiles.setdefault( moveStatus, 0) + 1 else: badStatusFiles[derivedStatus] = badStatusFiles.setdefault( derivedStatus, 0) + 1 if parentStatus == 'Unused': # If the file was Unused, set it NotProcessed in parent parentStatusFiles.setdefault('NotProcessed', []).append(lfn) else: parentStatusFiles.setdefault('Moved', []).append(lfn) for status, count in badStatusFiles.items(): # can be an iterator log.warn( 'Files found in an unexpected status in derived transformation', ': %d files in status %s' % (count, status)) # Set the status in the parent transformation first for status, lfnList in parentStatusFiles.items(): # can be an iterator for lfnChunk in breakListIntoChunks(lfnList, 5000): res = self.setFileStatusForTransformation( parentProd, status, lfnChunk) if not res['OK']: log.error( " Error setting status in transformation", "%d: status %s for %d files - %s" % (parentProd, status, len(lfnList), res['Message'])) # Set the status in the new transformation for (status, oldStatus ), lfnList in newStatusFiles.items(): # can be an iterator for lfnChunk in breakListIntoChunks(lfnList, 5000): res = self.setFileStatusForTransformation( prod, status, lfnChunk) if not res['OK']: log.error( " Error setting status in transformation", "%d: status %s for %d files; resetting them %s" % (parentProd, status, len(lfnChunk), oldStatus), res['Message']) res = self.setFileStatusForTransformation( parentProd, oldStatus, lfnChunk) if not res['OK']: log.error( " Error setting status in transformation", " %d: status %s for %d files" % (parentProd, oldStatus, len(lfnChunk)), res['Message']) else: log.info( 'Successfully moved files', ": %d files from %s to %s" % (len(lfnChunk), oldStatus, status)) # If files were Assigned or Unused at the time of derivation, try and update them as jobs may have run since then res = self.getTransformationFiles( condDict={ 'TransformationID': prod, 'Status': ['Assigned-inherited', 'Unused-inherited'] }) if res['OK']: assignedFiles = res['Value'] if assignedFiles: lfns = [lfnDict['LFN'] for lfnDict in assignedFiles] res = self.getTransformationFiles(condDict={ 'TransformationID': parentProd, 'LFN': lfns }) if res['OK']: parentFiles = res['Value'] processedLfns = [ lfnDict['LFN'] for lfnDict in parentFiles if lfnDict['Status'] == 'Processed' ] if processedLfns: res = self.setFileStatusForTransformation( prod, 'Processed-inherited', processedLfns) if res['OK']: log.info( 'Successfully set files status', ": %d files to status %s" % (len(processedLfns), 'Processed-inherited')) if not res['OK']: log.error("Error setting status for Assigned derived files", res['Message']) return S_OK((parentProd, movedFiles))
def __init__(self, parameters=None): self.log = gLogger.getSubLogger(self.__class__.__name__) self.parameters = parameters
def __init__(self): """c'tor :param self: self reference """ self.log = gLogger.getSubLogger(self.__class__.__name__, True) # # final states tuple self.finalStates = ('Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty') # # failed states tuple self.failedStates = ('Canceled', 'Failed', 'Hold', 'FinishedDirty') # # successful states tuple self.successfulStates = ('Finished', 'Done') # # all file states tuple self.fileStates = ('Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed', 'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting') self.statusSummary = {} # # request status self.requestStatus = 'Unknown' # # dict for FTS job files self.fileDict = {} # # dict for replicas information self.catalogReplicas = {} # # dict for metadata information self.catalogMetadata = {} # # dict for files that failed to register self.failedRegistrations = {} # # placehoder for FileCatalog reference self.oCatalog = None # # submit timestamp self.submitTime = '' # # placeholder FTS job GUID self.ftsGUID = '' # # placeholder for FTS server URL self.ftsServer = '' # # flag marking FTS job completness self.isTerminal = False # # completness percentage self.percentageComplete = 0.0 # # source SE name self.sourceSE = '' # # flag marking source SE validity self.sourceValid = False # # source space token self.sourceToken = '' # # target SE name self.targetSE = '' # # flag marking target SE validity self.targetValid = False # # target space token self.targetToken = '' # # placeholder for target StorageElement self.oTargetSE = None # # placeholder for source StorageElement self.oSourceSE = None # # checksum type, set it to default self.__cksmType = self.__defaultCksmType # # disable checksum test by default self.__cksmTest = False # # statuses that prevent submitting to FTS self.noSubmitStatus = ('Failed', 'Done', 'Staging') # # were sources resolved? self.sourceResolved = False # # Number of file transfers actually submitted self.submittedFiles = 0 self.transferTime = 0 self.submitCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/SubmitCommand', 'glite-transfer-submit') self.monitorCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/MonitorCommand', 'glite-transfer-status') self.ftsVersion = Operations().getValue('DataManagement/FTSVersion', 'FTS2') self.ftsJob = None self.ftsFiles = []
def __init__(self, baseURL=""): self.__baseURL = baseURL.strip("/") self.__routes = [] self.__handlers = [] self.__setupGroupRE = r"(?:/s:([\w-]*)/g:([\w-]*))?" self.log = gLogger.getSubLogger("Routing")
def __init__(self, requestJSON, handlersDict, csPath, agentName, standalone=False, requestClient=None, rmsMonitoring=False): """c'tor :param self: self reference :param str requestJSON: request serialized to JSON :param dict opHandlers: operation handlers """ self.request = Request(requestJSON) # # csPath self.csPath = csPath # # agent name self.agentName = agentName # # standalone flag self.standalone = standalone # # handlers dict self.handlersDict = handlersDict # # handlers class def self.handlers = {} # # own sublogger self.log = gLogger.getSubLogger( "pid_%s/%s" % (os.getpid(), self.request.RequestName)) # # get shifters info self.__managersDict = {} shifterProxies = self.__setupManagerProxies() if not shifterProxies["OK"]: self.log.error("Cannot setup shifter proxies", shifterProxies["Message"]) # This flag which is set and sent from the RequestExecutingAgent and is False by default. self.rmsMonitoring = rmsMonitoring if self.rmsMonitoring: self.rmsMonitoringReporter = MonitoringReporter( monitoringType="RMSMonitoring") else: # # initialize gMonitor gMonitor.setComponentType(gMonitor.COMPONENT_AGENT) gMonitor.setComponentName(self.agentName) gMonitor.initialize() # # own gMonitor activities gMonitor.registerActivity("RequestAtt", "Requests processed", "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM) gMonitor.registerActivity("RequestFail", "Requests failed", "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM) gMonitor.registerActivity("RequestOK", "Requests done", "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM) if requestClient is None: self.requestClient = ReqClient() else: self.requestClient = requestClient
def __init__(self): """ Standard constructor """ self.log = gLogger.getSubLogger('EndpointFactory')
def enforce(self, decisionParams): """Given a dictionary with decisionParams, it is passed to the PDP, which will return ( in case there is a/are positive match/es ) a dictionary containing three key-pair values: the original decisionParams ( `decisionParams` ), all the policies evaluated ( `singlePolicyResults` ) and the computed final result ( `policyCombinedResult` ). To know more about decisionParams, please read PDP.setup where the decisionParams are sanitized. examples: >>> pep.enforce( { 'element' : 'Site', 'name' : 'MySite' } ) >>> pep.enforce( { 'element' : 'Resource', 'name' : 'myce.domain.ch' } ) :Parameters: **decisionParams** - `dict` dictionary with the parameters that will be used to match policies. """ if not decisionParams: self.log.warn("No decision params...?") return S_OK() standardParamsDict = { "element": None, "name": None, "elementType": None, "statusType": None, "status": None, "reason": None, "tokenOwner": None, # Last parameter allows policies to be de-activated "active": "Active", } standardParamsDict.update(decisionParams) if standardParamsDict["element"] is not None: self.log = gLogger.getSubLogger( f"{self.__class__.__name__}/{standardParamsDict['element']}") if standardParamsDict["name"] is not None: self.log = gLogger.getSubLogger( f"{self.__class__.__name__}/{standardParamsDict['element']}/{standardParamsDict['name']}" ) self.log.verbose("Enforce - statusType: %s, status: %s" % (standardParamsDict["statusType"], standardParamsDict["status"])) decisionParams = dict(standardParamsDict) # Setup PDP with new parameters dictionary self.pdp.setup(decisionParams) # Run policies, get decision, get actions to apply resDecisions = self.pdp.takeDecision() if not resDecisions["OK"]: self.log.error("Something went wrong, not enforcing policies", "%s" % decisionParams) return resDecisions resDecisions = resDecisions["Value"] # We take from PDP the decision parameters used to find the policies decisionParams = resDecisions["decisionParams"] policyCombinedResult = resDecisions["policyCombinedResult"] singlePolicyResults = resDecisions["singlePolicyResults"] # We have run the actions and at this point, we are about to execute the actions. # One more final check before proceeding isNotUpdated = self.__isNotUpdated(decisionParams) if not isNotUpdated["OK"]: return isNotUpdated for policyActionName, policyActionType in policyCombinedResult[ "PolicyAction"]: try: actionMod = Utils.voimport( "DIRAC.ResourceStatusSystem.PolicySystem.Actions.%s" % policyActionType) except ImportError: self.log.error("Error importing %s action" % policyActionType) continue try: action = getattr(actionMod, policyActionType) except AttributeError: self.log.error("Error importing %s action class" % policyActionType) continue actionObj = action(policyActionName, decisionParams, policyCombinedResult, singlePolicyResults, self.clients) self.log.debug((policyActionName, policyActionType)) actionResult = actionObj.run() if not actionResult["OK"]: self.log.error(actionResult["Message"]) return S_OK(resDecisions)
def __init__(self, name, plugins=None, vo=None, hideExceptions=False): """ c'tor :param str name: SE name :param list plugins: requested storage plugins :param: vo """ self.methodName = None if vo: self.vo = vo else: result = getVOfromProxyGroup() if not result['OK']: return self.vo = result['Value'] self.opHelper = Operations(vo=self.vo) proxiedProtocols = gConfig.getValue( '/LocalSite/StorageElements/ProxyProtocols', "").split(',') useProxy = (gConfig.getValue( "/Resources/StorageElements/%s/AccessProtocol.1/Protocol" % name, "UnknownProtocol") in proxiedProtocols) if not useProxy: useProxy = gConfig.getValue( '/LocalSite/StorageElements/%s/UseProxy' % name, False) if not useProxy: useProxy = self.opHelper.getValue( '/Services/StorageElements/%s/UseProxy' % name, False) self.valid = True if plugins == None: res = StorageFactory(useProxy=useProxy, vo=self.vo).getStorages( name, pluginList=[], hideExceptions=hideExceptions) else: res = StorageFactory(useProxy=useProxy, vo=self.vo).getStorages( name, pluginList=plugins, hideExceptions=hideExceptions) if not res['OK']: self.valid = False self.name = name self.errorReason = res['Message'] else: factoryDict = res['Value'] self.name = factoryDict['StorageName'] self.options = factoryDict['StorageOptions'] self.localPlugins = factoryDict['LocalPlugins'] self.remotePlugins = factoryDict['RemotePlugins'] self.storages = factoryDict['StorageObjects'] self.protocolOptions = factoryDict['ProtocolOptions'] self.turlProtocols = factoryDict['TurlProtocols'] for storage in self.storages: storage.setStorageElement(self) self.log = gLogger.getSubLogger("SE[%s]" % self.name) self.useCatalogURL = gConfig.getValue( '/Resources/StorageElements/%s/UseCatalogURL' % self.name, False) # 'getTransportURL', self.readMethods = [ 'getFile', 'prestageFile', 'prestageFileStatus', 'getDirectory' ] self.writeMethods = [ 'retransferOnlineFile', 'putFile', 'replicateFile', 'pinFile', 'releaseFile', 'createDirectory', 'putDirectory' ] self.removeMethods = ['removeFile', 'removeDirectory'] self.checkMethods = [ 'exists', 'getDirectoryMetadata', 'getDirectorySize', 'getFileSize', 'getFileMetadata', 'listDirectory', 'isDirectory', 'isFile', ] self.okMethods = [ 'getLocalProtocols', 'getProtocols', 'getRemoteProtocols', 'getStorageElementName', 'getStorageParameters', 'getTransportURL', 'isLocalSE' ] self.__fileCatalog = None
def __init__(self, catalogs=None, vo=None): """ Default constructor """ self.valid = True self.timeout = 180 self.ro_methods = set() self.write_methods = set() self.no_lfn_methods = set() self.readCatalogs = [] self.writeCatalogs = [] self.rootConfigPath = '/Resources/FileCatalogs' self.vo = vo if vo else getVOfromProxyGroup().get('Value', None) self.log = gLogger.getSubLogger("FileCatalog") self.opHelper = Operations(vo=self.vo) catalogList = [] if isinstance(catalogs, basestring): catalogList = [catalogs] elif isinstance(catalogs, (list, tuple)): catalogList = list(catalogs) if catalogList: result = self._getEligibleCatalogs() if not result['OK']: self.log.error("Failed to get eligible catalog") return eligibleFileCatalogs = result['Value'] catalogCheck = True for catalog in catalogList: if catalog not in eligibleFileCatalogs: self.log.error("Specified catalog is not eligible", catalog) catalogCheck = False if catalogCheck: result = self._getSelectedCatalogs(catalogList) else: result = S_ERROR("Specified catalog is not eligible") else: result = self._getCatalogs() if not result['OK']: self.log.error("Failed to create catalog objects") self.valid = False elif (len(self.readCatalogs) == 0) and (len(self.writeCatalogs) == 0): self.log.error("No catalog object created") self.valid = False result = self.getMasterCatalogNames() masterCatalogs = result['Value'] # There can not be more than one master catalog haveMaster = False if len(masterCatalogs) > 1: self.log.error("More than one master catalog created") self.valid = False elif len(masterCatalogs) == 1: haveMaster = True # Get the list of write methods if haveMaster: # All the write methods must be present in the master _catalogName, oCatalog, _master = self.writeCatalogs[0] _roList, writeList, nolfnList = oCatalog.getInterfaceMethods() self.write_methods.update(writeList) self.no_lfn_methods.update(nolfnList) else: for _catalogName, oCatalog, _master in self.writeCatalogs: _roList, writeList, nolfnList = oCatalog.getInterfaceMethods() self.write_methods.update(writeList) self.no_lfn_methods.update(nolfnList) # Get the list of read methods for _catalogName, oCatalog, _master in self.readCatalogs: roList, _writeList, nolfnList = oCatalog.getInterfaceMethods() self.ro_methods.update(roList) self.no_lfn_methods.update(nolfnList) self.condParser = FCConditionParser(vo=self.vo, ro_methods=self.ro_methods)
def setUp( self ): gLogger.showHeaders( True ) self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.processPool = ProcessPool( 4, 8, 8 ) self.processPool.daemonize()
def __init__(self, rpcClient=False, transferClient=False): self.rpcClient = rpcClient self.transferClient = transferClient self.log = gLogger.getSubLogger("BundleDelivery")
def __init__( self, dbname, fullname, maxQueueSize ): self.database_name = dbname self.fullname = fullname self.cs_path = getDatabaseSection( fullname ) self.log = gLogger.getSubLogger( self.database_name ) self.dbHost = '' result = gConfig.getOption( self.cs_path + '/Host' ) if not result['OK']: self.log.fatal( 'Failed to get the configuration parameters: Host' ) return self.dbHost = result['Value'] # Check if the host is the local one and then set it to 'localhost' to use # a socket connection if self.dbHost != 'localhost': localHostName = socket.getfqdn() if localHostName == self.dbHost: self.dbHost = 'localhost' self.dbUser = '' result = gConfig.getOption( self.cs_path + '/User' ) if not result['OK']: # No individual user name found, try at the common place result = gConfig.getOption( '/Systems/Databases/User' ) if not result['OK']: self.log.fatal( 'Failed to get the configuration parameters: User' ) return self.dbUser = result['Value'] self.dbPass = '' result = gConfig.getOption( self.cs_path + '/Password' ) if not result['OK']: # No individual password found, try at the common place result = gConfig.getOption( '/Systems/Databases/Password' ) if not result['OK']: self.log.fatal( 'Failed to get the configuration parameters: Password' ) return self.dbPass = result['Value'] self.dbName = '' result = gConfig.getOption( self.cs_path + '/DBName' ) if not result['OK']: self.log.fatal( 'Failed to get the configuration parameters: DBName' ) return self.dbName = result['Value'] self.maxQueueSize = maxQueueSize result = gConfig.getOption( self.cs_path + '/MaxQueueSize' ) if result['OK']: self.maxQueueSize = int( result['Value'] ) MySQL.__init__( self, self.dbHost, self.dbUser, self.dbPass, self.dbName, maxQueueSize = maxQueueSize ) if not self._connected: err = 'Can not connect to DB, exiting...' self.log.fatal( err ) sys.exit( err ) self.log.info( "==================================================" ) #self.log.info("SystemInstance: "+self.system) self.log.info( "User: "******"Host: " + self.dbHost ) #self.log.info("Password: "******"DBName: " + self.dbName ) self.log.info( "MaxQueue: " + str(self.maxQueueSize) ) self.log.info( "==================================================" )
def __init__( self, taskID, timeWait, raiseException=False ): self.log = gLogger.getSubLogger( self.__class__.__name__ + "/%s" % taskID ) self.taskID = taskID self.timeWait = timeWait self.raiseException = raiseException
def __init__(self, name, protocols=None, vo=None): """ c'tor :param str name: SE name :param list protocols: requested protocols """ if vo: self.vo = vo else: result = getVOfromProxyGroup() if not result['OK']: return self.vo = result['Value'] self.opHelper = Operations(vo=self.vo) useProxy = gConfig.getValue( '/LocalSite/StorageElements/%s/UseProxy' % name, False) if not useProxy: useProxy = self.opHelper.getValue( '/Services/StorageElements/%s/UseProxy' % name, False) self.valid = True if protocols == None: res = StorageFactory(useProxy).getStorages(name, protocolList=[]) else: res = StorageFactory(useProxy).getStorages(name, protocolList=protocols) if not res['OK']: self.valid = False self.name = name self.errorReason = res['Message'] else: factoryDict = res['Value'] self.name = factoryDict['StorageName'] self.options = factoryDict['StorageOptions'] self.localProtocols = factoryDict['LocalProtocols'] self.remoteProtocols = factoryDict['RemoteProtocols'] self.storages = factoryDict['StorageObjects'] self.protocolOptions = factoryDict['ProtocolOptions'] self.turlProtocols = factoryDict['TurlProtocols'] self.log = gLogger.getSubLogger("SE[%s]" % self.name) self.readMethods = [ 'getFile', 'getAccessUrl', 'getTransportURL', 'prestageFile', 'prestageFileStatus', 'getDirectory' ] self.writeMethods = [ 'retransferOnlineFile', 'putFile', 'replicateFile', 'pinFile', 'releaseFile', 'createDirectory', 'putDirectory' ] self.removeMethods = ['removeFile', 'removeDirectory'] self.checkMethods = [ 'exists', 'getDirectoryMetadata', 'getDirectorySize', 'getFileSize', 'getFileMetadata', 'listDirectory', 'isDirectory', 'isFile', ] self.okMethods = [ 'getLocalProtocols', 'getPfnForProtocol', 'getPfnForLfn', 'getPfnPath', 'getProtocols', 'getRemoteProtocols', 'getStorageElementName', 'getStorageElementOption', 'getStorageParameters', 'isLocalSE' ]
def __init__(self, pid, exeThread, spObject, jobCPUTime, memoryLimit=0, processors=1, systemFlag='linux', jobArgs={}): """ Constructor, takes system flag as argument. """ self.stopSigStartSeconds = int(jobArgs.get('StopSigStartSeconds', 1800)) # 30 minutes self.stopSigFinishSeconds = int( jobArgs.get('StopSigFinishSeconds', 1800)) # 30 minutes self.stopSigNumber = int(jobArgs.get('StopSigNumber', 2)) # SIGINT self.stopSigRegex = jobArgs.get('StopSigRegex', None) self.stopSigSent = False self.log = gLogger.getSubLogger("Watchdog") self.systemFlag = systemFlag self.exeThread = exeThread self.wrapperPID = pid self.appPID = self.exeThread.getCurrentPID() self.spObject = spObject self.jobCPUTime = jobCPUTime self.memoryLimit = memoryLimit self.calibration = 0 self.initialValues = {} self.parameters = {} self.peekFailCount = 0 self.peekRetry = 5 self.profiler = Profiler(pid) self.checkError = '' self.currentStats = {} self.initialized = False self.count = 0 # defaults self.testWallClock = 1 self.testDiskSpace = 1 self.testLoadAvg = 1 self.maxWallClockTime = 3 * 24 * 60 * 60 self.testCPUConsumed = 1 self.testCPULimit = 0 self.testMemoryLimit = 0 self.testTimeLeft = 1 self.pollingTime = 10 # 10 seconds self.checkingTime = 30 * 60 # 30 minute period self.minCheckingTime = 20 * 60 # 20 mins self.wallClockCheckSeconds = 5 * 60 # 5 minutes self.maxWallClockTime = 3 * 24 * 60 * 60 # e.g. 4 days self.jobPeekFlag = 1 # on / off self.minDiskSpace = 10 # MB self.loadAvgLimit = 1000 # > 1000 and jobs killed self.sampleCPUTime = 30 * 60 # e.g. up to 20mins sample self.jobCPUMargin = 20 # %age buffer before killing job self.minCPUWallClockRatio = 5 # ratio %age self.nullCPULimit = 5 # After 5 sample times return null CPU consumption kill job self.checkCount = 0 self.wallClockCheckCount = 0 self.nullCPUCount = 0 self.grossTimeLeftLimit = 10 * self.checkingTime self.timeLeftUtil = TimeLeft() self.timeLeft = 0 self.littleTimeLeft = False self.scaleFactor = 1.0 self.processors = processors