def setup(self, decisionParams=None):
""" method that sanitizes the decisionParams and ensures that at least it has
the keys in `standardParamsDict`. This will be relevant while doing the matching
with the RSS Policies configuration in the CS.
There is one key-value pair, `active` which is added on this method. This allows
policies to be de-activated from the CS, changing their active matchParam to
something else than `Active`.
examples:
>>> pdp.setup( None )
>>> self.decisionParams
{ 'element' : None, 'name' : None, ... }
>>> pdp.setup( { 'element' : 'AnElement' } )
>>> self.decisionParams
{ 'element' : 'AnElement', 'name' : None, ... }
>>> pdp.setup( { 'NonStandardKey' : 'Something' } )
>>> self.decisionParams
{ 'NonStandardKey' : 'Something', 'element' : None,... }
:Parameters:
**decisionParams** - [ None, `dict` ]
dictionary with the parameters to be matched with the RSS Policies configuration
in the CS.
"""
standardParamsDict = {
"element": None,
"name": None,
"elementType": None,
"statusType": None,
"status": None,
"reason": None,
"tokenOwner": None,
# Last parameter allows policies to be de-activated
"active": "Active",
}
if decisionParams is not None:
standardParamsDict.update(decisionParams)
if standardParamsDict["element"] is not None:
self.log = gLogger.getSubLogger("PDP/%s" % standardParamsDict["element"])
if standardParamsDict["name"] is not None:
self.log = gLogger.getSubLogger(
"PDP/%s/%s" % (standardParamsDict["element"], standardParamsDict["name"])
)
self.log.verbose(
"Setup - statusType: %s, status: %s"
% (standardParamsDict["statusType"], standardParamsDict["status"])
)
self.decisionParams = standardParamsDict
def __init__( self, pilotAgentsDB = None, jobDB = None, tqDB = None, jlDB = None, opsHelper = None ):
""" c'tor
"""
if pilotAgentsDB:
self.pilotAgentsDB = pilotAgentsDB
else:
self.pilotAgentsDB = PilotAgentsDB()
if jobDB:
self.jobDB = jobDB
else:
self.jobDB = JobDB()
if tqDB:
self.tqDB = tqDB
else:
self.tqDB = TaskQueueDB()
if jlDB:
self.jlDB = jlDB
else:
self.jlDB = JobLoggingDB()
if opsHelper:
self.opsHelper = opsHelper
else:
self.opsHelper = Operations()
self.log = gLogger.getSubLogger( "Matcher" )
self.limiter = Limiter( jobDB = self.jobDB, opsHelper = self.opsHelper )
def __init__( self ): self.log = gLogger.getSubLogger( "Script" ) #Set defaults for all workflow parameters here self.name = '' self.executable = '' self.logFile = '' self.arguments = ''
def __init__(self): super(PythiaAnalysis, self).__init__() self.NumberOfEvents = 1 self.enable = True self.STEP_NUMBER = '' self.debug = True self.log = gLogger.getSubLogger( "PythiaAnalysis" )
def __init__(self, dbname, fullname, debug=False):
self.fullname = fullname
database_name = dbname
self.log = gLogger.getSubLogger(database_name)
result = getDBParameters(fullname)
if not result['OK']:
raise RuntimeError('Cannot get database parameters: %s' % result['Message'])
dbParameters = result['Value']
self.dbHost = dbParameters['Host']
self.dbPort = dbParameters['Port']
self.dbUser = dbParameters['User']
self.dbPass = dbParameters['Password']
self.dbName = dbParameters['DBName']
super(DB, self).__init__(hostName=self.dbHost,
userName=self.dbUser,
passwd=self.dbPass,
dbName=self.dbName,
port=self.dbPort,
debug=debug)
if not self._connected:
raise RuntimeError("Can not connect to DB '%s', exiting..." % self.dbName)
self.log.info("==================================================")
self.log.info("User: "******"Host: " + self.dbHost)
self.log.info("Port: " + str(self.dbPort))
#self.log.info("Password: "******"DBName: " + self.dbName)
self.log.info("==================================================")
def getNewLDLibs(platform, application, applicationVersion):
""" Prepare the LD_LIBRARY_PATH environment variable: make sure all lib folder are included
:param string platform: System config used for the job
:param string application: name of the application considered
:param string applicationVersion: version of the application considered
:return: new LD_LIBRARY_PATH
"""
log = gLogger.getSubLogger("GetLDLibs")
log.verbose("Getting all lib folders")
new_ld_lib_path = ""
deps = resolveDeps(platform, application, applicationVersion)
for dep in deps:
res = getSoftwareFolder(platform, dep["app"], dep['version'])
if not res['OK']:
continue
basedepfolder = res['Value']
if os.path.exists(os.path.join(basedepfolder, "lib")):
log.verbose("Found lib folder in %s" % (basedepfolder))
newlibdir = os.path.join(basedepfolder, "lib")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if os.path.exists(os.path.join(basedepfolder, "LDLibs")):
log.verbose("Found lib folder in %s" % (basedepfolder))
newlibdir = os.path.join(basedepfolder, "LDLibs")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if "LD_LIBRARY_PATH" in os.environ:
if new_ld_lib_path:
new_ld_lib_path = new_ld_lib_path + ":%s" % os.environ["LD_LIBRARY_PATH"]
else:
new_ld_lib_path = os.environ["LD_LIBRARY_PATH"]
return new_ld_lib_path
def getNewPATH(platform, application, applicationVersion):
""" Same as :func:`getNewLDLibs`,but for the PATH
:param string platform: System config used for the job
:param string application: name of the application considered
:param string applicationVersion: version of the application considered
:return: new PATH
"""
log = gLogger.getSubLogger("GetPaths")
log.verbose("Getting all PATH folders")
new_path = ""
deps = resolveDeps(platform, application, applicationVersion)
for dep in deps:
res = getSoftwareFolder(platform, dep['app'], dep['version'])
if not res['OK']:
continue
depfolder = res['Value']
if os.path.exists(os.path.join(depfolder, "bin")):
log.verbose("Found bin folder in %s" % (depfolder))
newpathdir = os.path.join(depfolder, "bin")
new_path = newpathdir
if "PATH" in os.environ:
if new_path:
new_path = new_path + ":%s" % os.environ["PATH"]
else:
new_path = os.environ["PATH"]
return new_path
def __init__( self, userName, password = '', tnsEntry = '', maxQueueSize = 100 ):
"""
set Oracle connection parameters and try to connect
"""
global gInstancesCount
gInstancesCount += 1
self.__initialized = False
self._connected = False
if 'logger' not in dir( self ):
self.logger = gLogger.getSubLogger( 'Oracle' )
# let the derived class decide what to do with if is not 1
self._threadsafe = cx_Oracle.threadsafety
self.logger.debug( 'thread_safe = %s' % self._threadsafe )
self.__checkQueueSize( maxQueueSize )
self.__userName = str( userName )
self.__passwd = str( password )
self.__tnsName = str( tnsEntry )
# Create the connection Queue to reuse connections
self.__connectionQueue = Queue.Queue( maxQueueSize )
# Create the connection Semaphore to limit total number of open connection
self.__connectionSemaphore = threading.Semaphore( maxQueueSize )
self.__initialized = True
self._connect()
def __init__(self, argumentsDict):
""" Standard constructor
"""
self.arguments = argumentsDict
self.name = COMPONENT_NAME
self.log = gLogger.getSubLogger(self.name)
self.ops = Operations()
def __init__( self, userCredPath, user, secret, endpointConfig, imageConfig):
"""
Constructor: uses user / secret authentication for the time being.
copy the endpointConfig and ImageConfig dictionaries to the OcciClient
:Parameters:
**userCredPath** - `string`
path to a valid x509 proxy
**endpointConfig** - `dict`
dictionary with the endpoint configuration ( WMS.Utilities.Configuration.OcciConfiguration )
**imageConfig** - `dict`
dictionary with the image configuration ( WMS.Utilities.Configuration.ImageConfiguration )
"""
# logger
self.log = gLogger.getSubLogger( self.__class__.__name__ )
self.endpointConfig = endpointConfig
self.imageConfig = imageConfig
self.__userCredPath = userCredPath
self.__user = user
self.__password = secret
if userCredPath is not None:
self.__authArg = ' --auth x509 --user-cred ' + self.__userCredPath + ' --voms '
else:
self.__authArg = ' --auth basic --username %s --password %s ' % (self.__user, self.__password)
def __init__( self, mqManager, mqURI, producerId ): self._connectionManager = mqManager self._mqURI = mqURI self._destination = getDestinationAddress( self._mqURI ) self._mqService = getMQService( self._mqURI ) self._id = producerId self.log = gLogger.getSubLogger( self.__class__.__name__ )
def __init__( self, transClient = None, logger = None, requestClient = None,
requestClass = None, requestValidator = None ):
""" c'tor
the requestClass is by default Request.
If extensions want to use an extended type, they can pass it as a parameter.
This is the same behavior as WorfkloTasks and jobClass
"""
if not logger:
logger = gLogger.getSubLogger( 'RequestTasks' )
super( RequestTasks, self ).__init__( transClient, logger )
if not requestClient:
self.requestClient = ReqClient()
else:
self.requestClient = requestClient
if not requestClass:
self.requestClass = Request
else:
self.requestClass = requestClass
if not requestValidator:
self.requestValidator = RequestValidator()
else:
self.requestValidator = requestValidator
def __init__( self, monitor = None ):
self.__idMap = {}
self.__execTypes = {}
self.__executorsLock = threading.Lock()
self.__tasksLock = threading.Lock()
self.__freezerLock = threading.Lock()
self.__tasks = {}
self.__log = gLogger.getSubLogger( "ExecMind" )
self.__taskFreezer = []
self.__queues = ExecutorQueues( self.__log )
self.__states = ExecutorState( self.__log )
self.__cbHolder = ExecutorDispatcherCallbacks()
self.__monitor = monitor
gThreadScheduler.addPeriodicTask( 60, self.__doPeriodicStuff )
#If a task is frozen too many times, send error or forget task?
self.__failedOnTooFrozen = True
#If a task fails to properly dispatch, freeze or forget task?
self.__freezeOnFailedDispatch = True
#If a task needs to go to an executor that has not connected. Freeze or forget the task?
self.__freezeOnUnknownExecutor = True
if self.__monitor:
self.__monitor.registerActivity( "executors", "Executor reactors connected",
"Executors", "executors", self.__monitor.OP_MEAN, 300 )
self.__monitor.registerActivity( "tasks", "Tasks processed",
"Executors", "tasks", self.__monitor.OP_RATE, 300 )
self.__monitor.registerActivity( "taskTime", "Task processing time",
"Executors", "seconds", self.__monitor.OP_MEAN, 300 )
def __init__( self, loggerIn = None ):
""" Initialization of module base.
loggerIn is a logger object that can be passed so that the logging will be more clear.
"""
if not loggerIn:
self.log = gLogger.getSubLogger( 'ModuleBase' )
else:
self.log = loggerIn
# These 2 are used in many places, so it's good to have them available here.
self.opsH = Operations()
self.dm = DataManager()
# Some job parameters
self.production_id = 0
self.prod_job_id = 0
self.jobID = 0
self.step_number = 0
self.step_id = 0
self.jobType = ''
self.executable = ''
self.command = None
self.workflowStatus = None
self.stepStatus = None
self.workflow_commons = None
self.step_commons = None
# These are useful objects (see the getFileReporter(), getJobReporter() and getRequestContainer() functions)
self.fileReport = None
self.jobReport = None
self.request = None
def resolveDeps(sysconfig, appli, appversion):
""" Resolve the dependencies
:param str sysconfig: system configuration
:param str appli: application name
:param str appversion: application version
:return: list of dictionaries
"""
log = gLogger.getSubLogger("resolveDeps")
ops = Operations()
deps = ops.getSections('/AvailableTarBalls/%s/%s/%s/Dependencies' % (sysconfig, appli,
appversion), '')
depsarray = []
if deps['OK']:
for dep in deps['Value']:
vers = ops.getValue('/AvailableTarBalls/%s/%s/%s/Dependencies/%s/version' % (sysconfig, appli,
appversion, dep), '')
depvers = ''
if vers:
depvers = vers
else:
log.error("Retrieving dependency version for %s failed, skipping to next !" % (dep))
continue
log.verbose("Found dependency %s %s" % (dep, depvers))
depdict = {}
depdict["app"] = dep
depdict["version"] = depvers
depsarray.append(depdict)
##resolve recursive dependencies
depsofdeps = resolveDeps(sysconfig, dep, depvers)
depsarray.extend(depsofdeps)
else:
log.verbose("Could not find any dependency for %s %s, ignoring" % (appli, appversion))
return depsarray
def __init__( self, dbname, fullname, debug = False ):
self.fullname = fullname
database_name = dbname
self.log = gLogger.getSubLogger( database_name )
result = getDBParameters( fullname )
if not result['OK'] :
raise RuntimeError( 'Cannot get database parameters: %s' % result['Message'] )
dbParameters = result[ 'Value' ]
self.dbHost = dbParameters[ 'Host' ]
self.dbPort = dbParameters[ 'Port' ]
self.dbUser = dbParameters[ 'User' ]
self.dbPass = dbParameters[ 'Password' ]
self.dbName = dbParameters[ 'DBName' ]
MySQL.__init__( self, self.dbHost, self.dbUser, self.dbPass,
self.dbName, self.dbPort, debug = debug )
if not self._connected:
raise RuntimeError( 'Can not connect to DB %s, exiting...' % self.dbName )
self.log.info( "==================================================" )
#self.log.info("SystemInstance: "+self.system)
self.log.info( "User: "******"Host: " + self.dbHost )
self.log.info( "Port: " + str( self.dbPort ) )
#self.log.info("Password: "******"DBName: " + self.dbName )
self.log.info( "==================================================" )
def __init__( self, imageName, endPoint ):
"""
Constructor: uses NovaConfiguration to parse the endPoint CS configuration
and ImageConfiguration to parse the imageName CS configuration.
:Parameters:
**imageName** - `string`
imageName as defined on CS:/Resources/VirtualMachines/Images
**endPoint** - `string`
endPoint as defined on CS:/Resources/VirtualMachines/CloudEndpoint
"""
# logger
self.log = gLogger.getSubLogger( 'NovaImage %s: ' % imageName )
self.imageName = imageName
self.endPoint = endPoint
# their config() method returns a dictionary with the parsed configuration
# they also provide a validate() method to make sure it is correct
self.__imageConfig = ImageConfiguration( imageName )
self.__novaConfig = NovaConfiguration( endPoint )
# this object will connect to the server. Better keep it private.
self.__clinova = None
def __init__(self, storageName, parameters):
""" """
super(GFAL2_SRM2Storage, self).__init__(storageName, parameters)
self.log = gLogger.getSubLogger("GFAL2_SRM2Storage", True)
self.log.debug("GFAL2_SRM2Storage.__init__: Initializing object")
self.pluginName = 'GFAL2_SRM2'
# This attribute is used to know the file status (OFFLINE,NEARLINE,ONLINE)
self._defaultExtendedAttributes = ['user.status']
# ##
# Setting the default SRM parameters here. For methods where this
# is not the default there is a method defined in this class, setting
# the proper values and then calling the base class method.
# ##
self.gfal2requestLifetime = gConfig.getValue('/Resources/StorageElements/RequestLifeTime', 100)
self.__setSRMOptionsToDefault()
# This lists contains the list of protocols to ask to SRM to get a URL
# It can be either defined in the plugin of the SE, or as a global option
if 'ProtocolsList' in parameters:
self.protocolsList = parameters['ProtocolsList'].split(',')
else:
self.log.debug("GFAL2_SRM2Storage: No protocols provided, using the default protocols.")
self.protocolsList = self.defaultLocalProtocols
self.log.debug('GFAL2_SRM2Storage: protocolsList = %s' % self.protocolsList)
def __init__( self, storageName, parameters ):
""" c'tor
:param self: self reference
:param str storageName: SE name
:param str protocol: protocol to use
:param str rootdir: base path for vo files
:param str host: SE host
:param int port: port to use to communicate with :host:
:param str spaceToken: space token
:param str wspath: location of SRM on :host:
"""
# # init base class
super( GFAL2_XROOTStorage, self ).__init__( storageName, parameters )
self.srmSpecificParse = False
self.log = gLogger.getSubLogger( "GFAL2_XROOTStorage", True )
self.pluginName = 'GFAL2_XROOT'
# why is this here ?!
self.protocolParameters['WSUrl'] = 0
self.protocolParameters['SpaceToken'] = 0
# We don't need extended attributes for metadata
self._defaultExtendedAttributes = None
def __init__( self, baseURL = "" ): self.__baseURL = baseURL.strip( "/" ) self.__routes = [] self.__handlers = [] self.__setupGroupRE = r"(?:/s:([\w-]*)/g:([\w.-]*))?" self.__shySetupGroupRE = r"(?:/s:(?:[\w-]*)/g:(?:[\w.-]*))?" self.log = gLogger.getSubLogger( "Routing" )
def __init__( self, user, secret, endpointConfig, imageConfig ):
"""
Multiple constructor depending on the passed parameters
:Parameters:
**user** - `string`
username that will be used on the authentication
**secret** - `string`
password used on the authentication
If secret is None then user actually is:
**proxyPath** - `string`
path to the valid X509 proxy
**endpointConfig** - `dict`
dictionary with the endpoint configuration ( WMS.Utilities.Configuration.NovaConfiguration )
**imageConfig** - `dict`
dictionary with the image configuration ( WMS.Utilities.Configuration.ImageConfiguration )
"""
# logger
self.log = gLogger.getSubLogger( self.__class__.__name__ )
self.endpointConfig = endpointConfig
self.imageConfig = imageConfig
# Variables needed to contact the service
ex_force_auth_url = endpointConfig.get( 'ex_force_auth_url', None )
ex_force_service_region = endpointConfig.get( 'ex_force_service_region', None )
ex_force_auth_version = endpointConfig.get( 'ex_force_auth_version', None )
ex_tenant_name = endpointConfig.get( 'ex_tenant_name', None )
# we force SSL cacert, if defined
ex_force_ca_cert = endpointConfig.get( 'ex_force_ca_cert', None )
if ex_force_ca_cert is not None:
security.CA_CERTS_PATH = [ ex_force_ca_cert ]
# get openstack driver
openstack_driver = get_driver( Provider.OPENSTACK )
if secret == None:
# with VOMS (from Alvaro Lopez trunk https://github.com/alvarolopez/libcloud/blob/trunk):
proxyPath=user
username = password = None
self.__driver = openstack_driver( username, password,
ex_force_auth_url = ex_force_auth_url,
ex_force_service_region = ex_force_service_region,
ex_force_auth_version = ex_force_auth_version,
ex_tenant_name = ex_tenant_name,
ex_voms_proxy = proxyPath
)
else:
# with user password
username = user
password = secret
self.__driver = openstack_driver( username, password,
ex_force_auth_url = ex_force_auth_url,
ex_force_service_region = ex_force_service_region,
ex_force_auth_version = ex_force_auth_version,
ex_tenant_name = ex_tenant_name
)
def __init__( self, connectionStorage = None ):
self.log = gLogger.getSubLogger( self.__class__.__name__ )
self.__lock = None
if connectionStorage:
self.__connectionStorage = connectionStorage
else:
self.__connectionStorage = {}
def __init__(self, name, decisionParams, enforcementResult, singlePolicyResults, clients):
# enforcementResult supposed to look like:
# {
# 'Status' : <str>,
# 'Reason' : <str>,
# 'PolicyActions' : <list>,
# [ 'EndDate' : <str> ]
# }
# decisionParams supposed to look like:
# {
# 'element' : None,
# 'name' : None,
# 'elementType' : None,
# 'statusType' : None,
# 'status' : None,
# 'reason' : None,
# 'tokenOwner' : None
# }
self.actionName = name # 'BaseAction'
self.decisionParams = decisionParams
self.enforcementResult = enforcementResult
self.singlePolicyResults = singlePolicyResults
self.clients = clients
self.log = gLogger.getSubLogger(self.__class__.__name__)
self.log.verbose("Running %s action" % self.__class__.__name__)
def __init__(self):
super(WhizardAnalysis, self).__init__()
self.enable = True
self.STEP_NUMBER = ''
self.debug = True
self.log = gLogger.getSubLogger( "WhizardAnalysis" )
self.SteeringFile = ''
self.OutputFile = ''
self.NumberOfEvents = 1
self.Lumi = 0
self.applicationName = 'whizard'
self.evttype = ""
self.RandomSeed = 0
self.getProcessInFile = False
self.datMan = DataManager()
self.processlist = None
self.parameters = {}
self.susymodel = 0
self.Model = ''
self.genmodel = GeneratorModels()
self.eventstring = ['! ', 'Fatal error:', 'PYSTOP', 'No matrix element available',
'Floating point exception', 'Event generation finished.', " n_events","luminosity",
" sum "]
self.excludeAllButEventString = False
self.steeringparameters = ''
self.options = None
self.optionsdict = {}
self.OptionsDictStr = ''
self.GenLevelCutDictStr = ''
self.genlevelcuts = {}
self.willCut = False
self.useGridFiles = False
def __init__(self):
'''
Constructor
'''
self.log = gLogger.getSubLogger("WrapperCall")
self.ops = Operations("glast.org")
self.stdError = ''
def __init__(self, transClient=None, logger=None, requestClient=None,
requestClass=None, requestValidator=None,
ownerDN=None, ownerGroup=None):
""" c'tor
the requestClass is by default Request.
If extensions want to use an extended type, they can pass it as a parameter.
This is the same behavior as WorfkloTasks and jobClass
"""
if not logger:
logger = gLogger.getSubLogger('RequestTasks')
super(RequestTasks, self).__init__(transClient, logger)
useCertificates = True if (bool(ownerDN) and bool(ownerGroup)) else False
if not requestClient:
self.requestClient = ReqClient(useCertificates=useCertificates,
delegatedDN=ownerDN,
delegatedGroup=ownerGroup)
else:
self.requestClient = requestClient
if not requestClass:
self.requestClass = Request
else:
self.requestClass = requestClass
if not requestValidator:
self.requestValidator = RequestValidator()
else:
self.requestValidator = requestValidator
def __init__(self):
super(OverlayInput, self).__init__()
self.enable = True
self.STEP_NUMBER = ''
self.log = gLogger.getSubLogger( "OverlayInput" )
self.applicationName = 'OverlayInput'
self.curdir = os.getcwd()
self.applicationLog = ''
self.printoutflag = ''
self.prodid = 0
self.detector = '' ##needed for backward compatibility
self.detectormodel = ""
self.energytouse = ''
self.energy = 0
self.nbofeventsperfile = 100
self.lfns = []
self.nbfilestoget = 0
self.BkgEvtType = 'gghad'
self.BXOverlay = 0
self.ggtohadint = 3.2
self.nbsigeventsperfile = 0
self.nbinputsigfile = 1
self.NbSigEvtsPerJob = 0
self.rm = ReplicaManager()
self.fc = FileCatalogClient()
self.site = DIRAC.siteName()
self.machine = 'clic_cdr'
def __init__( self, instanceId, URL, accessKey, secretKey ):
self.log = gLogger.getSubLogger( "CloudStackInstance id:%s" % instanceId )
self.__errorStatus = ""
self.__instanceId = instanceId
self.__csInstance = False
self.__imageId = "unknown"
self.__cloudStackURI = URL
self.__csAccessKey = accessKey
self.__csSecretKey = secretKey
self.__cliCloudStack = CloudStackClient( self.__cloudStackURI, self.__csAccessKey, self.__csSecretKey )
#Try connection
request = check_connection( self.__cloudStackURI, self.__csAccessKey, self.__csSecretKey )
if request.returncode != 0:
self.__errorStatus = "Can't connect to CloudStack server %s\n%s" % ( self.__cloudStackURI, request.stdout )
self.log.error( self.__errorStatus )
return
request = self.__cliCloudStack.get_image_ids_of_instance( self.__instanceId )
if request.returncode != 0:
self.__errorStatus = "Cannot find instance %s" % self.__instanceId
self.log.error( self.__errorStatus )
return
self.__csInstance = request.stdout
self.log.info( "CloudStack VM instance id %s initialized from image(%s)" % ( self.__instanceId, self.__csInstance ) )
return
def __init__(self): """Module initialization. """ super(ComputeOutputDataList, self).__init__() self.version = __RCSID__ self.log = gLogger.getSubLogger( "ComputeOutputData" ) self.listoutput = []
def getDIRACGOCDictionary():
"""
Create a dictionary containing DIRAC site names and GOCDB site names
using a configuration provided by CS.
:return: A dictionary of DIRAC site names (key) and GOCDB site names (value).
"""
log = gLogger.getSubLogger( 'getDIRACGOCDictionary' )
log.debug( 'Begin function ...' )
result = gConfig.getConfigurationTree( '/Resources/Sites', 'Name' )
if not result['OK']:
log.error( "getConfigurationTree() failed with message: %s" % result['Message'] )
return S_ERROR( 'Configuration is corrupted' )
siteNamesTree = result['Value']
dictionary = dict()
PATHELEMENTS = 6 # site names have 6 elements in the path, i.e.:
# /Resource/Sites/<GRID NAME>/<DIRAC SITE NAME>/Name
# [0]/[1] /[2] /[3] /[4] /[5]
for path, gocdbSiteName in siteNamesTree.iteritems():
elements = path.split( '/' )
if len( elements ) <> PATHELEMENTS:
continue
diracSiteName = elements[PATHELEMENTS - 2]
dictionary[diracSiteName] = gocdbSiteName
log.debug( 'End function.' )
return S_OK( dictionary )
class CachedJobState(object):
log = gLogger.getSubLogger("CachedJobState")
def __init__(self, jid, skipInitState=False):
self.dOnlyCache = False
self.__jid = jid
self.__jobState = JobState(jid)
self.cleanState(skipInitState=skipInitState)
def cleanState(self, skipInitState=False):
self.__cache = {}
self.__jobLog = []
self.__insertIntoTQ = False
self.__dirtyKeys = set()
self.__manifest = False
self.__initState = None
self.__lastValidState = time.time()
if not skipInitState:
result = self.getAttributes(
["Status", "MinorStatus", "LastUpdateTime"])
if result['OK']:
self.__initState = result['Value']
else:
self.__initState = None
def recheckValidity(self, graceTime=600):
now = time.time()
if graceTime <= 0 or now - self.__lastValidState > graceTime:
self.__lastValidState = now
result = self.__jobState.getAttributes(
["Status", "MinorStatus", "LastUpdateTime"])
if not result['OK']:
return result
currentState = result['Value']
if not currentState == self.__initState:
return S_OK(False)
return S_OK(True)
return S_OK(self.valid)
@property
def valid(self):
return self.__initState != None
@property
def jid(self):
return self.__jid
def getDirtyKeys(self):
return set(self.__dirtyKeys)
def commitChanges(self):
if self.__initState == None:
return S_ERROR("CachedJobState( %d ) is not valid" % self.__jid)
changes = {}
for k in self.__dirtyKeys:
changes[k] = self.__cache[k]
result = self.__jobState.commitCache(self.__initState, changes,
self.__jobLog)
try:
result.pop('rpcStub')
except KeyError:
pass
if not result['OK']:
self.cleanState()
return result
if not result['Value']:
self.cleanState()
return S_ERROR("Initial state was different")
newState = result['Value']
self.__jobLog = []
self.__dirtyKeys.clear()
#Save manifest
if self.__manifest and self.__manifest.isDirty():
result = self.__jobState.setManifest(self.__manifest)
if not result['OK']:
self.cleanState()
for i in range(5):
if self.__jobState.rescheduleJob()['OK']:
break
return result
self.__manifest.clearDirty()
#Insert into TQ
if self.__insertIntoTQ:
result = self.__jobState.insertIntoTQ()
if not result['OK']:
self.cleanState()
for i in range(5):
if self.__jobState.rescheduleJob()['OK']:
break
return result
self.__insertIntoTQ = False
self.__initState = newState
self.__lastValidState = time.time()
return S_OK()
def serialize(self):
if self.__manifest:
manifest = (self.__manifest.dumpAsCFG(), self.__manifest.isDirty())
else:
manifest = None
return DEncode.encode(
(self.__jid, self.__cache, self.__jobLog, manifest,
self.__initState, self.__insertIntoTQ, tuple(self.__dirtyKeys)))
@staticmethod
def deserialize(stub):
dataTuple, slen = DEncode.decode(stub)
if len(dataTuple) != 7:
return S_ERROR("Invalid stub")
#jid
if type(dataTuple[0]) not in (types.IntType, types.LongType):
return S_ERROR("Invalid stub 0")
#cache
if type(dataTuple[1]) != types.DictType:
return S_ERROR("Invalid stub 1")
#trace
if type(dataTuple[2]) != types.ListType:
return S_ERROR("Invalid stub 2")
#manifest
tdt3 = type(dataTuple[3])
if tdt3 != types.NoneType and (tdt3 != types.TupleType
and len(dataTuple[3]) != 2):
return S_ERROR("Invalid stub 3")
#initstate
if type(dataTuple[4]) != types.DictType:
return S_ERROR("Invalid stub 4")
#Insert into TQ
if type(dataTuple[5]) != types.BooleanType:
return S_ERROR("Invalid stub 5")
#Dirty Keys
if type(dataTuple[6]) != types.TupleType:
return S_ERROR("Invalid stub 6")
cjs = CachedJobState(dataTuple[0], skipInitState=True)
cjs.__cache = dataTuple[1]
cjs.__jobLog = dataTuple[2]
dt3 = dataTuple[3]
if dataTuple[3]:
manifest = JobManifest()
result = manifest.loadCFG(dt3[0])
if not result['OK']:
return result
if dt3[1]:
manifest.setDirty()
else:
manifest.clearDirty()
cjs.__manifest = manifest
cjs.__initState = dataTuple[4]
cjs.__insertIntoTQ = dataTuple[5]
cjs.__dirtyKeys = set(dataTuple[6])
return S_OK(cjs)
def __cacheAdd(self, key, value):
self.__cache[key] = value
self.__dirtyKeys.add(key)
def __cacheExists(self, keyList):
if type(keyList) in types.StringTypes:
keyList = [keyList]
for key in keyList:
if key not in self.__cache:
return False
return True
def __cacheResult(self, cKey, functor, fArgs=None):
keyType = type(cKey)
#If it's a string
if keyType in types.StringTypes:
if cKey not in self.__cache:
if self.dOnlyCache:
return S_ERROR("%s is not cached")
if not fArgs:
fArgs = tuple()
result = functor(*fArgs)
if not result['OK']:
return result
data = result['Value']
self.__cache[cKey] = data
return S_OK(self.__cache[cKey])
#Tuple/List
elif keyType in (types.ListType, types.TupleType):
if not self.__cacheExists(cKey):
if self.dOnlyCache:
return S_ERROR("%s is not cached")
if not fArgs:
fArgs = tuple()
result = functor(*fArgs)
if not result['OK']:
return result
data = result['Value']
if len(cKey) != len(data):
gLogger.warn(
"CachedJobState.__memorize( %s, %s = %s ) doesn't receive the same amount of values as keys"
% (cKey, functor, data))
return data
for i in range(len(cKey)):
self.__cache[cKey[i]] = data[i]
#Prepare result
return S_OK(tuple([self.__cache[cK] for cK in cKey]))
else:
raise RuntimeError("Cache key %s does not have a valid type" %
cKey)
def __cacheDict(self, prefix, functor, keyList=None):
if not keyList or not self.__cacheExists(
["%s.%s" % (prefix, key) for key in keyList]):
result = functor(keyList)
if not result['OK']:
return result
data = result['Value']
for key in data:
cKey = "%s.%s" % (prefix, key)
#If the key is already in the cache. DO NOT TOUCH. User may have already modified it.
#We update the coming data with the cached data
if cKey in self.__cache:
data[key] = self.__cache[cKey]
else:
self.__cache[cKey] = data[key]
return S_OK(data)
return S_OK(
dict([(key, self.__cache["%s.%s" % (prefix, key)])
for key in keyList]))
def _inspectCache(self):
return copy.deepcopy(self.__cache)
def _clearCache(self):
self.__cache = {}
@property
def _internals(self):
if self.__manifest:
manifest = (self.__manifest.dumpAsCFG(), self.__manifest.isDirty())
else:
manifest = None
return (self.__jid, self.dOnlyCache, dict(self.__cache),
list(self.__jobLog), manifest, dict(self.__initState),
list(self.__dirtyKeys))
#
# Manifest
#
def getManifest(self):
if not self.__manifest:
result = self.__jobState.getManifest()
if not result['OK']:
return result
self.__manifest = result['Value']
return S_OK(self.__manifest)
def setManifest(self, manifest):
if not isinstance(manifest, JobManifest):
jobManifest = JobManifest()
result = jobManifest.load(str(manifest))
if not result['OK']:
return result
manifest = jobManifest
manifest.setDirty()
self.__manifest = manifest
# self.__manifest.clearDirty()
return S_OK()
# Attributes
#
def __addLogRecord(self,
majorStatus=None,
minorStatus=None,
appStatus=None,
source=None):
record = {}
if majorStatus:
record['status'] = majorStatus
if minorStatus:
record['minor'] = minorStatus
if appStatus:
record['application'] = appStatus
if not record:
return
if not source:
source = "Unknown"
self.__jobLog.append((record, Time.dateTime(), source))
def setStatus(self,
majorStatus,
minorStatus=None,
appStatus=None,
source=None):
self.__cacheAdd('att.Status', majorStatus)
if minorStatus:
self.__cacheAdd('att.MinorStatus', minorStatus)
if appStatus:
self.__cacheAdd('att.ApplicationStatus', appStatus)
self.__addLogRecord(majorStatus, minorStatus, appStatus, source)
return S_OK()
def setMinorStatus(self, minorStatus, source=None):
self.__cacheAdd('att.MinorStatus', minorStatus)
self.__addLogRecord(minorStatus=minorStatus, source=source)
return S_OK()
def getStatus(self):
return self.__cacheResult(('att.Status', 'att.MinorStatus'),
self.__jobState.getStatus)
def setAppStatus(self, appStatus, source=None):
self.__cacheAdd('att.ApplicationStatus', appStatus)
self.__addLogRecord(appStatus=appStatus, source=source)
return S_OK()
def getAppStatus(self):
return self.__cacheResult('att.ApplicationStatus',
self.__jobState.getAppStatus)
#
# Attribs
#
def setAttribute(self, name, value):
if type(name) not in types.StringTypes:
return S_ERROR("Attribute name has to be a string")
self.__cacheAdd("att.%s" % name, value)
return S_OK()
def setAttributes(self, attDict):
if type(attDict) != types.DictType:
return S_ERROR("Attributes has to be a dictionary and it's %s" %
str(type(attDict)))
for key in attDict:
self.__cacheAdd("att.%s" % key, attDict[key])
return S_OK()
def getAttribute(self, name):
return self.__cacheResult('att.%s' % name,
self.__jobState.getAttribute, (name, ))
def getAttributes(self, nameList=None):
return self.__cacheDict('att', self.__jobState.getAttributes, nameList)
#Job params
def setParameter(self, name, value):
if type(name) not in types.StringTypes:
return S_ERROR("Job parameter name has to be a string")
self.__cacheAdd('jobp.%s' % name, value)
return S_OK()
def setParameters(self, pDict):
if type(pDict) != types.DictType:
return S_ERROR("Job parameters has to be a dictionary")
for key in pDict:
self.__cacheAdd('jobp.%s' % key, pDict[key])
return S_OK()
def getParameter(self, name):
return self.__cacheResult("jobp.%s" % name,
self.__jobState.getParameter, (name, ))
def getParameters(self, nameList=None):
return self.__cacheDict('jobp', self.__jobState.getParameters,
nameList)
#Optimizer params
def setOptParameter(self, name, value):
if type(name) not in types.StringTypes:
return S_ERROR("Optimizer parameter name has to be a string")
self.__cacheAdd('optp.%s' % name, value)
return S_OK()
def setOptParameters(self, pDict):
if type(pDict) != types.DictType:
return S_ERROR("Optimizer parameters has to be a dictionary")
for key in pDict:
self.__cacheAdd('optp.%s' % key, pDict[key])
return S_OK()
def getOptParameter(self, name):
return self.__cacheResult("optp.%s" % name,
self.__jobState.getOptParameter, (name, ))
def getOptParameters(self, nameList=None):
return self.__cacheDict('optp', self.__jobState.getOptParameters,
nameList)
#Other
def resetJob(self, source=""):
""" Reset the job!
"""
result = self.__jobState.resetJob(source=source)
if result['OK']:
self.__resetState()
return result
def getInputData(self):
return self.__cacheResult("inputData", self.__jobState.getInputData)
def insertIntoTQ(self):
if self.valid:
self.__insertIntoTQ = True
return S_OK()
return S_ERROR("Cached state is invalid")
def run():
'''
Main method
'''
subLogger.info('Checking DB...')
installDB()
subLogger.info('[done]')
#...............................................................................
if __name__ == "__main__":
subLogger = gLogger.getSubLogger(__file__)
# Script initialization
registerSwitches()
registerUsageMessage()
parseSwitches()
# Run the script
run()
# Bye
DIRACExit(0)
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def __init__(self):
""" Standard constructor
"""
self.log = gLogger.getSubLogger(__name__)
def __init__(self): """ Standard constructor """ self.version = platform.uname() self.log = gLogger.getSubLogger( 'WatchdogFactory' ) self.watchDogsLocation = 'DIRAC.WorkloadManagementSystem.JobWrapper'
def __init__(self, transferClient=False, **kwargs):
super(BundleDeliveryClient, self).__init__(**kwargs)
self.setServer('Framework/BundleDelivery')
self.transferClient = transferClient
self.log = gLogger.getSubLogger("BundleDelivery")
def __init__(self):
# constructor code
self.version = __RCSID__
self.log = gLogger.getSubLogger('DISETRequest')
self.portalEnv = False
def setParameters(self, parameters):
self.log = gLogger.getSubLogger('%s/%s' % (__name__, parameters['ProviderName']))
self.parameters = parameters
self.oauth2 = OAuth2(parameters['ProviderName'])
"""
import errno
# from DIRAC
from DIRAC import S_OK, S_ERROR, gLogger
from DIRAC.Core.DISET.RequestHandler import RequestHandler
from DIRAC.Core.DISET.ThreadConfig import ThreadConfig
from DIRAC.Core.Utilities.ReturnValues import returnSingleResult
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.Resources.Storage.StorageElement import StorageElement
########################################################################
LOG = gLogger.getSubLogger(__name__)
class S3GatewayHandler(RequestHandler):
"""
.. class:: S3GatewayHandler
"""
# FC instance to check whether access is permitted or not
_fc = None
# Mapping between the S3 methods and the DFC methods
_s3ToFC_methods = {
"head_object": "getFileMetadata",
"get_object":
def __init__(self, paramdict=None):
""" Can define the full application by passing a dictionary in the constructor.
>>> app = Application({"appname":"marlin","version":"v0111Prod",
... "steeringFile":"My_file.xml","numberOfEvents":1000})
:param dict paramdict: Dictionary of parameters that can be set. Will throw an exception if one of them does not exist.
"""
super(Application, self).__init__()
#application nane (executable)
self.appname = ""
#application version
self.version = ""
#Steering file (duh!)
self.steeringFile = ""
#Input sandbox: steering file automatically added to SB
self.inputSB = []
#Input file
self.inputFile = ""
#Output file
self.outputFile = ""
self.outputPath = ""
self.outputDstFile = ''
self.outputDstPath = ''
self.outputRecFile = ''
self.outputRecPath = ''
self.outputSE = ''
self._listofoutput = []
#Log file
self.logFile = ""
#Detector type (ILD or SID)
self.detectortype = ""
#Data type : gen, SIM, REC, DST
self.datatype = ""
#Extra command line arguments
self.extraCLIArguments = ""
##Needed for Generation+Stdhepcut
self.willBeCut = False
##Debug mode
self.debug = False
#Prod Parameters: things that appear on the prod details
self.prodparameters = {}
self.accountInProduction = True
#Module name and description: Not to be set by the users, internal call only, used to get the Module objects
self._modulename = ''
self._moduledescription = ''
self._importLocation = "ILCDIRAC.Workflow.Modules"
#System Configuration: comes from Job definition
self._platform = ''
#Internal member: hold the list of the job's application set before self: used when using getInputFromApp
self._job = None
self._jobapps = []
self._jobsteps = []
self._jobtype = ''
#input application: will link the OutputFile of the guys in there with the InputFile of the self
self._inputapp = []
self._linkedidx = None
#Needed to link the parameters.
self._inputappstep = None
#flag set to true in Job.append
self.addedtojob = False
####Following are needed for error report
self._log = gLogger.getSubLogger(self.__class__.__name__)
self._errorDict = {}
#This is used to filter out the members that should not be set when using a dict as input
self._paramsToExclude = [
'_paramsToExclude', "_log", "_errorDict", "addedtojob", '_ops',
"_inputappstep", "_linkedidx", "_inputapp", "_jobtype",
"_jobsteps", "_jobapps", "_job", "_platform", "_importLocation",
"_moduledescription", "_modulename", "prodparameters", "datatype",
"detectortype", "_listofoutput", "inputSB", "appname",
'accountInProduction', 'outputPath'
]
### Next is to use the setattr method.
self._setparams(paramdict)
def __init__(self,
configSection,
channels=None,
bandwidths=None,
failedFiles=None):
"""c'tor
:param self: self reference
:param str configSection: path on CS to ReplicationScheduler agent
:param bandwithds: observed throughput on active channels
:param channels: active channels
:param int failedFiles: max number of distinct failed files to allow scheduling
"""
## save config section
self.configSection = configSection + "/" + self.__class__.__name__
##
## sublogger
self.log = gLogger.getSubLogger("StrategyHandler", child=True)
self.log.setLevel(
gConfig.getValue(self.configSection + "/LogLevel", "DEBUG"))
self.supportedStrategies = [
'Simple', 'DynamicThroughput', 'Swarm', 'MinimiseTotalWait'
]
self.log.info("Supported strategies = %s" %
", ".join(self.supportedStrategies))
self.sigma = gConfig.getValue(self.configSection + '/HopSigma', 0.0)
self.log.info("HopSigma = %s" % self.sigma)
self.schedulingType = gConfig.getValue(
self.configSection + '/SchedulingType', 'File')
self.log.info("SchedulingType = %s" % self.schedulingType)
self.activeStrategies = gConfig.getValue(
self.configSection + '/ActiveStrategies', ['MinimiseTotalWait'])
self.log.info("ActiveStrategies = %s" %
", ".join(self.activeStrategies))
self.numberOfStrategies = len(self.activeStrategies)
self.log.info("Number of active strategies = %s" %
self.numberOfStrategies)
self.acceptableFailureRate = gConfig.getValue(
self.configSection + '/AcceptableFailureRate', 75)
self.log.info("AcceptableFailureRate = %s" %
self.acceptableFailureRate)
self.acceptableFailedFiles = gConfig.getValue(
self.configSection + "/AcceptableFailedFiles", 5)
self.log.info("AcceptableFailedFiles = %s" %
self.acceptableFailedFiles)
self.rwUpdatePeriod = gConfig.getValue(
self.configSection + "/RssRWUpdatePeriod", 600)
self.log.info("RSSUpdatePeriod = %s s" % self.rwUpdatePeriod)
self.rwUpdatePeriod = datetime.timedelta(seconds=self.rwUpdatePeriod)
## bandwithds
self.bandwidths = bandwidths if bandwidths else {}
## channels
self.channels = channels if channels else {}
## distinct failed files per channel
self.failedFiles = failedFiles if failedFiles else {}
## chosen strategy
self.chosenStrategy = 0
## fts graph
self.ftsGraph = None
## timestamp for last update
self.lastRssUpdate = datetime.datetime.now()
# dispatcher
self.strategyDispatcher = {
"MinimiseTotalWait": self.minimiseTotalWait,
"DynamicThroughput": self.dynamicThroughput,
"Simple": self.simple,
"Swarm": self.swarm
}
## own RSS client
self.resourceStatus = ResourceStatus()
## create fts graph
ftsGraph = self.setup(self.channels, self.bandwidths, self.failedFiles)
if not ftsGraph["OK"]:
raise SHGraphCreationError(ftsGraph["Message"])
self.log.info("%s has been constructed" % self.__class__.__name__)
def initialize(self):
super(MonitoringHistoryCorrector, self).initialize()
self.log = gLogger.getSubLogger("MonitoringHistoryCorrector")
return S_OK()
def __init__(self,ceType=''): """ Standard constructor """ self.ceType = ceType self.log = gLogger.getSubLogger( self.ceType )
def setParameters(self, params):
""" Set the transformation parameters and extract transID """
self.params = params
self.transID = params['TransformationID']
self.log = gLogger.getSubLogger(self.plugin + self.transInThread.get(
self.transID, ' [NoThread] [%d] ' % self.transID))
def __init__(self, catalogName):
""" Standard constructor
"""
self.fileName = catalogName
self.name = COMPONENT_NAME
self.log = gLogger.getSubLogger(self.name)
def __init__(self):
""" Standard constructor
"""
self.log = gLogger.getSubLogger('ProcessMonitor')
self.osType = platform.uname()
def moveFilesToDerivedTransformation(self, transDict, resetUnused=True):
""" move files input to a transformation, to the derived one
"""
prod = transDict['TransformationID']
parentProd = int(transDict.get('InheritedFrom', 0))
movedFiles = {}
log = gLogger.getSubLogger(
"[None] [%d] .moveFilesToDerivedTransformation:" % prod)
if not parentProd:
log.warn("Transformation was not derived...")
return S_OK((parentProd, movedFiles))
# get the lfns in status Unused/MaxReset of the parent production
res = self.getTransformationFiles(condDict={
'TransformationID': parentProd,
'Status': ['Unused', 'MaxReset']
})
if not res['OK']:
log.error(" Error getting Unused files from transformation",
"%d: %s" % (parentProd, res['Message']))
return res
parentFiles = res['Value']
lfns = [lfnDict['LFN'] for lfnDict in parentFiles]
if not lfns:
log.info(" No files found to be moved from transformation",
"%d" % parentProd)
return S_OK((parentProd, movedFiles))
# get the lfns of the derived production that were Unused/MaxReset in the parent one
res = self.getTransformationFiles(condDict={
'TransformationID': prod,
'LFN': lfns
})
if not res['OK']:
log.error(" Error getting files from derived transformation:",
res['Message'])
return res
derivedFiles = res['Value']
derivedStatusDict = dict((derivedDict['LFN'], derivedDict['Status'])
for derivedDict in derivedFiles)
newStatusFiles = {}
parentStatusFiles = {}
badStatusFiles = {}
for parentDict in parentFiles:
lfn = parentDict['LFN']
derivedStatus = derivedStatusDict.get(lfn)
if derivedStatus:
parentStatus = parentDict['Status']
# By default move to the parent status (which is Unused or MaxReset)
status = parentStatus
moveStatus = parentStatus
# For MaxReset, set Unused if requested
if parentStatus == 'MaxReset':
if resetUnused:
status = 'Unused'
moveStatus = 'Unused from MaxReset'
else:
status = 'MaxReset-inherited'
if derivedStatus.endswith('-inherited'):
# This is the general case
newStatusFiles.setdefault((status, parentStatus),
[]).append(lfn)
movedFiles[moveStatus] = movedFiles.setdefault(
moveStatus, 0) + 1
else:
badStatusFiles[derivedStatus] = badStatusFiles.setdefault(
derivedStatus, 0) + 1
if parentStatus == 'Unused':
# If the file was Unused, set it NotProcessed in parent
parentStatusFiles.setdefault('NotProcessed',
[]).append(lfn)
else:
parentStatusFiles.setdefault('Moved', []).append(lfn)
for status, count in badStatusFiles.items(): # can be an iterator
log.warn(
'Files found in an unexpected status in derived transformation',
': %d files in status %s' % (count, status))
# Set the status in the parent transformation first
for status, lfnList in parentStatusFiles.items(): # can be an iterator
for lfnChunk in breakListIntoChunks(lfnList, 5000):
res = self.setFileStatusForTransformation(
parentProd, status, lfnChunk)
if not res['OK']:
log.error(
" Error setting status in transformation",
"%d: status %s for %d files - %s" %
(parentProd, status, len(lfnList), res['Message']))
# Set the status in the new transformation
for (status, oldStatus
), lfnList in newStatusFiles.items(): # can be an iterator
for lfnChunk in breakListIntoChunks(lfnList, 5000):
res = self.setFileStatusForTransformation(
prod, status, lfnChunk)
if not res['OK']:
log.error(
" Error setting status in transformation",
"%d: status %s for %d files; resetting them %s" %
(parentProd, status, len(lfnChunk), oldStatus),
res['Message'])
res = self.setFileStatusForTransformation(
parentProd, oldStatus, lfnChunk)
if not res['OK']:
log.error(
" Error setting status in transformation",
" %d: status %s for %d files" %
(parentProd, oldStatus, len(lfnChunk)),
res['Message'])
else:
log.info(
'Successfully moved files',
": %d files from %s to %s" %
(len(lfnChunk), oldStatus, status))
# If files were Assigned or Unused at the time of derivation, try and update them as jobs may have run since then
res = self.getTransformationFiles(
condDict={
'TransformationID': prod,
'Status': ['Assigned-inherited', 'Unused-inherited']
})
if res['OK']:
assignedFiles = res['Value']
if assignedFiles:
lfns = [lfnDict['LFN'] for lfnDict in assignedFiles]
res = self.getTransformationFiles(condDict={
'TransformationID': parentProd,
'LFN': lfns
})
if res['OK']:
parentFiles = res['Value']
processedLfns = [
lfnDict['LFN'] for lfnDict in parentFiles
if lfnDict['Status'] == 'Processed'
]
if processedLfns:
res = self.setFileStatusForTransformation(
prod, 'Processed-inherited', processedLfns)
if res['OK']:
log.info(
'Successfully set files status',
": %d files to status %s" %
(len(processedLfns), 'Processed-inherited'))
if not res['OK']:
log.error("Error setting status for Assigned derived files",
res['Message'])
return S_OK((parentProd, movedFiles))
def __init__(self, parameters=None): self.log = gLogger.getSubLogger(self.__class__.__name__) self.parameters = parameters
def __init__(self):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger(self.__class__.__name__, True)
# # final states tuple
self.finalStates = ('Canceled', 'Failed', 'Hold', 'Finished',
'FinishedDirty')
# # failed states tuple
self.failedStates = ('Canceled', 'Failed', 'Hold', 'FinishedDirty')
# # successful states tuple
self.successfulStates = ('Finished', 'Done')
# # all file states tuple
self.fileStates = ('Done', 'Active', 'Pending', 'Ready', 'Canceled',
'Failed', 'Finishing', 'Finished', 'Submitted',
'Hold', 'Waiting')
self.statusSummary = {}
# # request status
self.requestStatus = 'Unknown'
# # dict for FTS job files
self.fileDict = {}
# # dict for replicas information
self.catalogReplicas = {}
# # dict for metadata information
self.catalogMetadata = {}
# # dict for files that failed to register
self.failedRegistrations = {}
# # placehoder for FileCatalog reference
self.oCatalog = None
# # submit timestamp
self.submitTime = ''
# # placeholder FTS job GUID
self.ftsGUID = ''
# # placeholder for FTS server URL
self.ftsServer = ''
# # flag marking FTS job completness
self.isTerminal = False
# # completness percentage
self.percentageComplete = 0.0
# # source SE name
self.sourceSE = ''
# # flag marking source SE validity
self.sourceValid = False
# # source space token
self.sourceToken = ''
# # target SE name
self.targetSE = ''
# # flag marking target SE validity
self.targetValid = False
# # target space token
self.targetToken = ''
# # placeholder for target StorageElement
self.oTargetSE = None
# # placeholder for source StorageElement
self.oSourceSE = None
# # checksum type, set it to default
self.__cksmType = self.__defaultCksmType
# # disable checksum test by default
self.__cksmTest = False
# # statuses that prevent submitting to FTS
self.noSubmitStatus = ('Failed', 'Done', 'Staging')
# # were sources resolved?
self.sourceResolved = False
# # Number of file transfers actually submitted
self.submittedFiles = 0
self.transferTime = 0
self.submitCommand = Operations().getValue(
'DataManagement/FTSPlacement/FTS2/SubmitCommand',
'glite-transfer-submit')
self.monitorCommand = Operations().getValue(
'DataManagement/FTSPlacement/FTS2/MonitorCommand',
'glite-transfer-status')
self.ftsVersion = Operations().getValue('DataManagement/FTSVersion',
'FTS2')
self.ftsJob = None
self.ftsFiles = []
def __init__(self, baseURL=""):
self.__baseURL = baseURL.strip("/")
self.__routes = []
self.__handlers = []
self.__setupGroupRE = r"(?:/s:([\w-]*)/g:([\w-]*))?"
self.log = gLogger.getSubLogger("Routing")
def __init__(self,
requestJSON,
handlersDict,
csPath,
agentName,
standalone=False,
requestClient=None,
rmsMonitoring=False):
"""c'tor
:param self: self reference
:param str requestJSON: request serialized to JSON
:param dict opHandlers: operation handlers
"""
self.request = Request(requestJSON)
# # csPath
self.csPath = csPath
# # agent name
self.agentName = agentName
# # standalone flag
self.standalone = standalone
# # handlers dict
self.handlersDict = handlersDict
# # handlers class def
self.handlers = {}
# # own sublogger
self.log = gLogger.getSubLogger(
"pid_%s/%s" % (os.getpid(), self.request.RequestName))
# # get shifters info
self.__managersDict = {}
shifterProxies = self.__setupManagerProxies()
if not shifterProxies["OK"]:
self.log.error("Cannot setup shifter proxies",
shifterProxies["Message"])
# This flag which is set and sent from the RequestExecutingAgent and is False by default.
self.rmsMonitoring = rmsMonitoring
if self.rmsMonitoring:
self.rmsMonitoringReporter = MonitoringReporter(
monitoringType="RMSMonitoring")
else:
# # initialize gMonitor
gMonitor.setComponentType(gMonitor.COMPONENT_AGENT)
gMonitor.setComponentName(self.agentName)
gMonitor.initialize()
# # own gMonitor activities
gMonitor.registerActivity("RequestAtt", "Requests processed",
"RequestExecutingAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RequestFail", "Requests failed",
"RequestExecutingAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RequestOK", "Requests done",
"RequestExecutingAgent", "Requests/min",
gMonitor.OP_SUM)
if requestClient is None:
self.requestClient = ReqClient()
else:
self.requestClient = requestClient
def __init__(self):
""" Standard constructor
"""
self.log = gLogger.getSubLogger('EndpointFactory')
def enforce(self, decisionParams):
"""Given a dictionary with decisionParams, it is passed to the PDP, which
will return ( in case there is a/are positive match/es ) a dictionary containing
three key-pair values: the original decisionParams ( `decisionParams` ), all
the policies evaluated ( `singlePolicyResults` ) and the computed final result
( `policyCombinedResult` ).
To know more about decisionParams, please read PDP.setup where the decisionParams
are sanitized.
examples:
>>> pep.enforce( { 'element' : 'Site', 'name' : 'MySite' } )
>>> pep.enforce( { 'element' : 'Resource', 'name' : 'myce.domain.ch' } )
:Parameters:
**decisionParams** - `dict`
dictionary with the parameters that will be used to match policies.
"""
if not decisionParams:
self.log.warn("No decision params...?")
return S_OK()
standardParamsDict = {
"element": None,
"name": None,
"elementType": None,
"statusType": None,
"status": None,
"reason": None,
"tokenOwner": None,
# Last parameter allows policies to be de-activated
"active": "Active",
}
standardParamsDict.update(decisionParams)
if standardParamsDict["element"] is not None:
self.log = gLogger.getSubLogger(
f"{self.__class__.__name__}/{standardParamsDict['element']}")
if standardParamsDict["name"] is not None:
self.log = gLogger.getSubLogger(
f"{self.__class__.__name__}/{standardParamsDict['element']}/{standardParamsDict['name']}"
)
self.log.verbose("Enforce - statusType: %s, status: %s" %
(standardParamsDict["statusType"],
standardParamsDict["status"]))
decisionParams = dict(standardParamsDict)
# Setup PDP with new parameters dictionary
self.pdp.setup(decisionParams)
# Run policies, get decision, get actions to apply
resDecisions = self.pdp.takeDecision()
if not resDecisions["OK"]:
self.log.error("Something went wrong, not enforcing policies",
"%s" % decisionParams)
return resDecisions
resDecisions = resDecisions["Value"]
# We take from PDP the decision parameters used to find the policies
decisionParams = resDecisions["decisionParams"]
policyCombinedResult = resDecisions["policyCombinedResult"]
singlePolicyResults = resDecisions["singlePolicyResults"]
# We have run the actions and at this point, we are about to execute the actions.
# One more final check before proceeding
isNotUpdated = self.__isNotUpdated(decisionParams)
if not isNotUpdated["OK"]:
return isNotUpdated
for policyActionName, policyActionType in policyCombinedResult[
"PolicyAction"]:
try:
actionMod = Utils.voimport(
"DIRAC.ResourceStatusSystem.PolicySystem.Actions.%s" %
policyActionType)
except ImportError:
self.log.error("Error importing %s action" % policyActionType)
continue
try:
action = getattr(actionMod, policyActionType)
except AttributeError:
self.log.error("Error importing %s action class" %
policyActionType)
continue
actionObj = action(policyActionName, decisionParams,
policyCombinedResult, singlePolicyResults,
self.clients)
self.log.debug((policyActionName, policyActionType))
actionResult = actionObj.run()
if not actionResult["OK"]:
self.log.error(actionResult["Message"])
return S_OK(resDecisions)
def __init__(self, name, plugins=None, vo=None, hideExceptions=False):
""" c'tor
:param str name: SE name
:param list plugins: requested storage plugins
:param: vo
"""
self.methodName = None
if vo:
self.vo = vo
else:
result = getVOfromProxyGroup()
if not result['OK']:
return
self.vo = result['Value']
self.opHelper = Operations(vo=self.vo)
proxiedProtocols = gConfig.getValue(
'/LocalSite/StorageElements/ProxyProtocols', "").split(',')
useProxy = (gConfig.getValue(
"/Resources/StorageElements/%s/AccessProtocol.1/Protocol" % name,
"UnknownProtocol") in proxiedProtocols)
if not useProxy:
useProxy = gConfig.getValue(
'/LocalSite/StorageElements/%s/UseProxy' % name, False)
if not useProxy:
useProxy = self.opHelper.getValue(
'/Services/StorageElements/%s/UseProxy' % name, False)
self.valid = True
if plugins == None:
res = StorageFactory(useProxy=useProxy, vo=self.vo).getStorages(
name, pluginList=[], hideExceptions=hideExceptions)
else:
res = StorageFactory(useProxy=useProxy, vo=self.vo).getStorages(
name, pluginList=plugins, hideExceptions=hideExceptions)
if not res['OK']:
self.valid = False
self.name = name
self.errorReason = res['Message']
else:
factoryDict = res['Value']
self.name = factoryDict['StorageName']
self.options = factoryDict['StorageOptions']
self.localPlugins = factoryDict['LocalPlugins']
self.remotePlugins = factoryDict['RemotePlugins']
self.storages = factoryDict['StorageObjects']
self.protocolOptions = factoryDict['ProtocolOptions']
self.turlProtocols = factoryDict['TurlProtocols']
for storage in self.storages:
storage.setStorageElement(self)
self.log = gLogger.getSubLogger("SE[%s]" % self.name)
self.useCatalogURL = gConfig.getValue(
'/Resources/StorageElements/%s/UseCatalogURL' % self.name, False)
# 'getTransportURL',
self.readMethods = [
'getFile', 'prestageFile', 'prestageFileStatus', 'getDirectory'
]
self.writeMethods = [
'retransferOnlineFile', 'putFile', 'replicateFile', 'pinFile',
'releaseFile', 'createDirectory', 'putDirectory'
]
self.removeMethods = ['removeFile', 'removeDirectory']
self.checkMethods = [
'exists',
'getDirectoryMetadata',
'getDirectorySize',
'getFileSize',
'getFileMetadata',
'listDirectory',
'isDirectory',
'isFile',
]
self.okMethods = [
'getLocalProtocols', 'getProtocols', 'getRemoteProtocols',
'getStorageElementName', 'getStorageParameters', 'getTransportURL',
'isLocalSE'
]
self.__fileCatalog = None
def __init__(self, catalogs=None, vo=None):
""" Default constructor
"""
self.valid = True
self.timeout = 180
self.ro_methods = set()
self.write_methods = set()
self.no_lfn_methods = set()
self.readCatalogs = []
self.writeCatalogs = []
self.rootConfigPath = '/Resources/FileCatalogs'
self.vo = vo if vo else getVOfromProxyGroup().get('Value', None)
self.log = gLogger.getSubLogger("FileCatalog")
self.opHelper = Operations(vo=self.vo)
catalogList = []
if isinstance(catalogs, basestring):
catalogList = [catalogs]
elif isinstance(catalogs, (list, tuple)):
catalogList = list(catalogs)
if catalogList:
result = self._getEligibleCatalogs()
if not result['OK']:
self.log.error("Failed to get eligible catalog")
return
eligibleFileCatalogs = result['Value']
catalogCheck = True
for catalog in catalogList:
if catalog not in eligibleFileCatalogs:
self.log.error("Specified catalog is not eligible",
catalog)
catalogCheck = False
if catalogCheck:
result = self._getSelectedCatalogs(catalogList)
else:
result = S_ERROR("Specified catalog is not eligible")
else:
result = self._getCatalogs()
if not result['OK']:
self.log.error("Failed to create catalog objects")
self.valid = False
elif (len(self.readCatalogs) == 0) and (len(self.writeCatalogs) == 0):
self.log.error("No catalog object created")
self.valid = False
result = self.getMasterCatalogNames()
masterCatalogs = result['Value']
# There can not be more than one master catalog
haveMaster = False
if len(masterCatalogs) > 1:
self.log.error("More than one master catalog created")
self.valid = False
elif len(masterCatalogs) == 1:
haveMaster = True
# Get the list of write methods
if haveMaster:
# All the write methods must be present in the master
_catalogName, oCatalog, _master = self.writeCatalogs[0]
_roList, writeList, nolfnList = oCatalog.getInterfaceMethods()
self.write_methods.update(writeList)
self.no_lfn_methods.update(nolfnList)
else:
for _catalogName, oCatalog, _master in self.writeCatalogs:
_roList, writeList, nolfnList = oCatalog.getInterfaceMethods()
self.write_methods.update(writeList)
self.no_lfn_methods.update(nolfnList)
# Get the list of read methods
for _catalogName, oCatalog, _master in self.readCatalogs:
roList, _writeList, nolfnList = oCatalog.getInterfaceMethods()
self.ro_methods.update(roList)
self.no_lfn_methods.update(nolfnList)
self.condParser = FCConditionParser(vo=self.vo,
ro_methods=self.ro_methods)
def setUp( self ): gLogger.showHeaders( True ) self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.processPool = ProcessPool( 4, 8, 8 ) self.processPool.daemonize()
def __init__(self, rpcClient=False, transferClient=False):
self.rpcClient = rpcClient
self.transferClient = transferClient
self.log = gLogger.getSubLogger("BundleDelivery")
def __init__( self, dbname, fullname, maxQueueSize ):
self.database_name = dbname
self.fullname = fullname
self.cs_path = getDatabaseSection( fullname )
self.log = gLogger.getSubLogger( self.database_name )
self.dbHost = ''
result = gConfig.getOption( self.cs_path + '/Host' )
if not result['OK']:
self.log.fatal( 'Failed to get the configuration parameters: Host' )
return
self.dbHost = result['Value']
# Check if the host is the local one and then set it to 'localhost' to use
# a socket connection
if self.dbHost != 'localhost':
localHostName = socket.getfqdn()
if localHostName == self.dbHost:
self.dbHost = 'localhost'
self.dbUser = ''
result = gConfig.getOption( self.cs_path + '/User' )
if not result['OK']:
# No individual user name found, try at the common place
result = gConfig.getOption( '/Systems/Databases/User' )
if not result['OK']:
self.log.fatal( 'Failed to get the configuration parameters: User' )
return
self.dbUser = result['Value']
self.dbPass = ''
result = gConfig.getOption( self.cs_path + '/Password' )
if not result['OK']:
# No individual password found, try at the common place
result = gConfig.getOption( '/Systems/Databases/Password' )
if not result['OK']:
self.log.fatal( 'Failed to get the configuration parameters: Password' )
return
self.dbPass = result['Value']
self.dbName = ''
result = gConfig.getOption( self.cs_path + '/DBName' )
if not result['OK']:
self.log.fatal( 'Failed to get the configuration parameters: DBName' )
return
self.dbName = result['Value']
self.maxQueueSize = maxQueueSize
result = gConfig.getOption( self.cs_path + '/MaxQueueSize' )
if result['OK']:
self.maxQueueSize = int( result['Value'] )
MySQL.__init__( self, self.dbHost, self.dbUser, self.dbPass,
self.dbName, maxQueueSize = maxQueueSize )
if not self._connected:
err = 'Can not connect to DB, exiting...'
self.log.fatal( err )
sys.exit( err )
self.log.info( "==================================================" )
#self.log.info("SystemInstance: "+self.system)
self.log.info( "User: "******"Host: " + self.dbHost )
#self.log.info("Password: "******"DBName: " + self.dbName )
self.log.info( "MaxQueue: " + str(self.maxQueueSize) )
self.log.info( "==================================================" )
def __init__( self, taskID, timeWait, raiseException=False ): self.log = gLogger.getSubLogger( self.__class__.__name__ + "/%s" % taskID ) self.taskID = taskID self.timeWait = timeWait self.raiseException = raiseException
def __init__(self, name, protocols=None, vo=None):
""" c'tor
:param str name: SE name
:param list protocols: requested protocols
"""
if vo:
self.vo = vo
else:
result = getVOfromProxyGroup()
if not result['OK']:
return
self.vo = result['Value']
self.opHelper = Operations(vo=self.vo)
useProxy = gConfig.getValue(
'/LocalSite/StorageElements/%s/UseProxy' % name, False)
if not useProxy:
useProxy = self.opHelper.getValue(
'/Services/StorageElements/%s/UseProxy' % name, False)
self.valid = True
if protocols == None:
res = StorageFactory(useProxy).getStorages(name, protocolList=[])
else:
res = StorageFactory(useProxy).getStorages(name,
protocolList=protocols)
if not res['OK']:
self.valid = False
self.name = name
self.errorReason = res['Message']
else:
factoryDict = res['Value']
self.name = factoryDict['StorageName']
self.options = factoryDict['StorageOptions']
self.localProtocols = factoryDict['LocalProtocols']
self.remoteProtocols = factoryDict['RemoteProtocols']
self.storages = factoryDict['StorageObjects']
self.protocolOptions = factoryDict['ProtocolOptions']
self.turlProtocols = factoryDict['TurlProtocols']
self.log = gLogger.getSubLogger("SE[%s]" % self.name)
self.readMethods = [
'getFile', 'getAccessUrl', 'getTransportURL', 'prestageFile',
'prestageFileStatus', 'getDirectory'
]
self.writeMethods = [
'retransferOnlineFile', 'putFile', 'replicateFile', 'pinFile',
'releaseFile', 'createDirectory', 'putDirectory'
]
self.removeMethods = ['removeFile', 'removeDirectory']
self.checkMethods = [
'exists',
'getDirectoryMetadata',
'getDirectorySize',
'getFileSize',
'getFileMetadata',
'listDirectory',
'isDirectory',
'isFile',
]
self.okMethods = [
'getLocalProtocols', 'getPfnForProtocol', 'getPfnForLfn',
'getPfnPath', 'getProtocols', 'getRemoteProtocols',
'getStorageElementName', 'getStorageElementOption',
'getStorageParameters', 'isLocalSE'
]
def __init__(self,
pid,
exeThread,
spObject,
jobCPUTime,
memoryLimit=0,
processors=1,
systemFlag='linux',
jobArgs={}):
""" Constructor, takes system flag as argument.
"""
self.stopSigStartSeconds = int(jobArgs.get('StopSigStartSeconds',
1800)) # 30 minutes
self.stopSigFinishSeconds = int(
jobArgs.get('StopSigFinishSeconds', 1800)) # 30 minutes
self.stopSigNumber = int(jobArgs.get('StopSigNumber', 2)) # SIGINT
self.stopSigRegex = jobArgs.get('StopSigRegex', None)
self.stopSigSent = False
self.log = gLogger.getSubLogger("Watchdog")
self.systemFlag = systemFlag
self.exeThread = exeThread
self.wrapperPID = pid
self.appPID = self.exeThread.getCurrentPID()
self.spObject = spObject
self.jobCPUTime = jobCPUTime
self.memoryLimit = memoryLimit
self.calibration = 0
self.initialValues = {}
self.parameters = {}
self.peekFailCount = 0
self.peekRetry = 5
self.profiler = Profiler(pid)
self.checkError = ''
self.currentStats = {}
self.initialized = False
self.count = 0
# defaults
self.testWallClock = 1
self.testDiskSpace = 1
self.testLoadAvg = 1
self.maxWallClockTime = 3 * 24 * 60 * 60
self.testCPUConsumed = 1
self.testCPULimit = 0
self.testMemoryLimit = 0
self.testTimeLeft = 1
self.pollingTime = 10 # 10 seconds
self.checkingTime = 30 * 60 # 30 minute period
self.minCheckingTime = 20 * 60 # 20 mins
self.wallClockCheckSeconds = 5 * 60 # 5 minutes
self.maxWallClockTime = 3 * 24 * 60 * 60 # e.g. 4 days
self.jobPeekFlag = 1 # on / off
self.minDiskSpace = 10 # MB
self.loadAvgLimit = 1000 # > 1000 and jobs killed
self.sampleCPUTime = 30 * 60 # e.g. up to 20mins sample
self.jobCPUMargin = 20 # %age buffer before killing job
self.minCPUWallClockRatio = 5 # ratio %age
self.nullCPULimit = 5 # After 5 sample times return null CPU consumption kill job
self.checkCount = 0
self.wallClockCheckCount = 0
self.nullCPUCount = 0
self.grossTimeLeftLimit = 10 * self.checkingTime
self.timeLeftUtil = TimeLeft()
self.timeLeft = 0
self.littleTimeLeft = False
self.scaleFactor = 1.0
self.processors = processors
