def getValidStatusTypes(): ''' Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/Resources ''' DEFAULTS = { 'Site' : { 'StatusType' : "''" }, 'Service' : { 'StatusType' : "''" }, 'Resource' : { 'StatusType' : "''" }, 'StorageElement': { 'StatusType' : [ 'Read', 'Write', 'Remove', 'Check' ] } } opHelper = Operations() sections = opHelper.getSections( 'RSSConfiguration/GeneralConfig/Resources' ) if not sections[ 'OK' ]: return DEFAULTS result = {} for section in sections[ 'Value' ]: res = opHelper.getValue( 'RSSConfiguration/GeneralConfig/Resources/%s/StatusType' % section ) if res is None: if DEFAULTS.has_key( section ): result[ section ] = { 'StatusType' : DEFAULTS[ section ] } else: result[ section ] = { 'StatusType' : None } else: result[ section ] = { 'StatusType' : Utils.getTypedList( res ) } return result
def resolveDeps(sysconfig, appli, appversion): """ Resolve the dependencies :param str sysconfig: system configuration :param str appli: application name :param str appversion: application version :return: list of dictionaries """ log = gLogger.getSubLogger("resolveDeps") ops = Operations() deps = ops.getSections('/AvailableTarBalls/%s/%s/%s/Dependencies' % (sysconfig, appli, appversion), '') depsarray = [] if deps['OK']: for dep in deps['Value']: vers = ops.getValue('/AvailableTarBalls/%s/%s/%s/Dependencies/%s/version' % (sysconfig, appli, appversion, dep), '') depvers = '' if vers: depvers = vers else: log.error("Retrieving dependency version for %s failed, skipping to next !" % (dep)) continue log.verbose("Found dependency %s %s" % (dep, depvers)) depdict = {} depdict["app"] = dep depdict["version"] = depvers depsarray.append(depdict) ##resolve recursive dependencies depsofdeps = resolveDeps(sysconfig, dep, depvers) depsarray.extend(depsofdeps) else: log.verbose("Could not find any dependency for %s %s, ignoring" % (appli, appversion)) return depsarray
def web_getLaunchpadSetupWithLFNs(self): """ Method obtain launchpad setup with pre-selected LFNs as input data parameter, the caller js client will use setup to open an new Launchpad """ # On the fly file catalog for advanced launchpad if not hasattr(self, 'fc'): userData = self.getSessionData() group = str(userData["user"]["group"]) vo = getVOForGroup(group) self.fc = FileCatalog(vo=vo) self.set_header('Content-type', 'text/plain') arguments = self.request.arguments gLogger.always("submit: incoming arguments %s to getLaunchpadSetupWithLFNs" % arguments) lfnList = str(arguments['path'][0]).split(',') # Modified for Eiscat # Checks if the experiments folder in lfn list has a rtg_def.m file at some subfolder gLogger.always("submit: checking if some rtg_def.m", arguments) processed = [] metaDict = {'type': 'info'} for lfn in lfnList: pos_relative = lfn.find("/") pos_relative = lfn.find("/", pos_relative + 1) pos_relative = lfn.find("/", pos_relative + 1) pos_relative = lfn.find("/", pos_relative + 1) pos_relative = lfn.find("/", pos_relative + 1) experiment_lfn = lfn[0:pos_relative] if experiment_lfn in processed: continue processed.append(experiment_lfn) gLogger.always( "checking rtg_def.m in %s" % experiment_lfn ) result = self.fc.findFilesByMetadata( metaDict, path=str(experiment_lfn) ) if not result['OK'] or not result['Value']: gLogger.error( "Failed to get type info from $s, %s" % (experiment_lfn,result[ "Message" ]) ) continue for candidate_lfn in result['Value']: if candidate_lfn.find('rtg_def.m') > 0: lfnList.append(candidate_lfn) # End modified ptlfn = '' for lfn in lfnList: ptlfn += (', ' + lfn) if ptlfn else lfn params = self.defaultParams.copy() params["InputData"] = [1, ptlfn] obj = Operations(vo=vo) predefinedSets = {} launchpadSections = obj.getSections("Launchpad") if launchpadSections['OK']: for section in launchpadSections["Value"]: predefinedSets[section] = {} sectionOptions = obj.getOptionsDict("Launchpad/" + section) pprint.pprint(sectionOptions) if sectionOptions['OK']: predefinedSets[section] = sectionOptions["Value"] self.write({"success": "true", "result": params, "predefinedSets": predefinedSets})
def web_getLaunchpadOpts(self): defaultParams = {"JobName" : [1, 'DIRAC'], "Executable" : [1, "/bin/ls"], "Arguments" : [1, "-ltrA"], "OutputSandbox" : [1, "std.out, std.err"], "InputData" : [0, ""], "OutputData" : [0, ""], "OutputSE" : [0, "DIRAC-USER"], "OutputPath": [0, ""], "CPUTime" : [0, "86400"], "Site" : [0, ""], "BannedSite" : [0, ""], "Platform" : [0, "Linux_x86_64_glibc-2.5"], "Priority" : [0, "5"], "StdError" : [0, "std.err"], "StdOutput" : [0, "std.out"], "Parameters" : [0, "0"], "ParameterStart" : [0, "0"], "ParameterStep" : [0, "1"]} delimiter = gConfig.getValue("/Website/Launchpad/ListSeparator" , ',') options = self.__getOptionsFromCS(delimiter=delimiter) # platform = self.__getPlatform() # if platform and options: # if not options.has_key("Platform"): # options[ "Platform" ] = platform # else: # csPlatform = list(options[ "Platform" ]) # allPlatforms = csPlatform + platform # platform = uniqueElements(allPlatforms) # options[ "Platform" ] = platform gLogger.debug("Combined options from CS: %s" % options) override = gConfig.getValue("/Website/Launchpad/OptionsOverride" , False) gLogger.info("end __getLaunchpadOpts") # Updating the default values from OptionsOverride configuration branch for key in options: if key not in defaultParams: defaultParams[key] = [ 0, "" ] defaultParams[key][1] = options[key][0] # Reading of the predefined sets of launchpad parameters values obj = Operations() predefinedSets = {} launchpadSections = obj.getSections("Launchpad") import pprint if launchpadSections['OK']: for section in launchpadSections["Value"]: predefinedSets[section] = {} sectionOptions = obj.getOptionsDict("Launchpad/" + section) pprint.pprint(sectionOptions) if sectionOptions['OK']: predefinedSets[section] = sectionOptions["Value"] self.write({"success":"true", "result":defaultParams, "predefinedSets":predefinedSets})
def __setupManagerProxies(self): """ setup grid proxy for all defined managers """ oHelper = Operations() shifters = oHelper.getSections("Shifter") if not shifters["OK"]: self.log.error(shifters["Message"]) return shifters shifters = shifters["Value"] for shifter in shifters: shifterDict = oHelper.getOptionsDict("Shifter/%s" % shifter) if not shifterDict["OK"]: self.log.error(shifterDict["Message"]) continue userName = shifterDict["Value"].get("User", "") userGroup = shifterDict["Value"].get("Group", "") userDN = CS.getDNForUsername(userName) if not userDN["OK"]: self.log.error(userDN["Message"]) continue userDN = userDN["Value"][0] vomsAttr = CS.getVOMSAttributeForGroup(userGroup) if vomsAttr: self.log.debug( "getting VOMS [%s] proxy for shifter %s@%s (%s)" % (vomsAttr, userName, userGroup, userDN)) getProxy = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup, requiredTimeLeft=1200, cacheTime=4 * 43200) else: self.log.debug("getting proxy for shifter %s@%s (%s)" % (userName, userGroup, userDN)) getProxy = gProxyManager.downloadProxyToFile( userDN, userGroup, requiredTimeLeft=1200, cacheTime=4 * 43200) if not getProxy["OK"]: self.log.error(getProxy["Message"]) return S_ERROR("unable to setup shifter proxy for %s: %s" % (shifter, getProxy["Message"])) chain = getProxy["chain"] fileName = getProxy["Value"] self.log.debug("got %s: %s %s" % (shifter, userName, userGroup)) self.__managersDict[shifter] = { "ShifterDN": userDN, "ShifterName": userName, "ShifterGroup": userGroup, "Chain": chain, "ProxyFile": fileName } return S_OK()
def _getCatalogs( self ): # Get the eligible catalogs first # First, look in the Operations, if nothing defined look in /Resources for backward compatibility result = getVOfromProxyGroup() if not result['OK']: return result vo = result['Value'] opHelper = Operations( vo = vo ) result = opHelper.getSections( '/Services/FileCatalogs' ) fileCatalogs = [] operationsFlag = False if result['OK']: fileCatalogs = result['Value'] operationsFlag = True else: res = gConfig.getSections( self.rootConfigPath, listOrdered = True ) if not res['OK']: errStr = "FileCatalog._getCatalogs: Failed to get file catalog configuration." gLogger.error( errStr, res['Message'] ) return S_ERROR( errStr ) fileCatalogs = res['Value'] # Get the catalogs now for catalogName in fileCatalogs: res = self._getCatalogConfigDetails( catalogName ) if not res['OK']: return res catalogConfig = res['Value'] if operationsFlag: result = opHelper.getOptionsDict( '/Services/FileCatalogs/%s' % catalogName ) if not result['OK']: return result catalogConfig.update( result['Value'] ) if catalogConfig['Status'] == 'Active': res = self._generateCatalogObject( catalogName ) if not res['OK']: return res oCatalog = res['Value'] master = catalogConfig['Master'] # If the catalog is read type if re.search( 'Read', catalogConfig['AccessType'] ): if master: self.readCatalogs.insert( 0, ( catalogName, oCatalog, master ) ) else: self.readCatalogs.append( ( catalogName, oCatalog, master ) ) # If the catalog is write type if re.search( 'Write', catalogConfig['AccessType'] ): if master: self.writeCatalogs.insert( 0, ( catalogName, oCatalog, master ) ) else: self.writeCatalogs.append( ( catalogName, oCatalog, master ) ) return S_OK()
def __setupManagerProxies( self ): """ setup grid proxy for all defined managers """ oHelper = Operations() shifters = oHelper.getSections( "Shifter" ) if not shifters["OK"]: self.log.error( shifters["Message"] ) return shifters shifters = shifters["Value"] for shifter in shifters: shifterDict = oHelper.getOptionsDict( "Shifter/%s" % shifter ) if not shifterDict["OK"]: self.log.error( shifterDict["Message"] ) continue userName = shifterDict["Value"].get( "User", "" ) userGroup = shifterDict["Value"].get( "Group", "" ) userDN = CS.getDNForUsername( userName ) if not userDN["OK"]: self.log.error( userDN["Message"] ) continue userDN = userDN["Value"][0] vomsAttr = CS.getVOMSAttributeForGroup( userGroup ) if vomsAttr: self.log.debug( "getting VOMS [%s] proxy for shifter %s@%s (%s)" % ( vomsAttr, userName, userGroup, userDN ) ) getProxy = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup, requiredTimeLeft = 1200, cacheTime = 4 * 43200 ) else: self.log.debug( "getting proxy for shifter %s@%s (%s)" % ( userName, userGroup, userDN ) ) getProxy = gProxyManager.downloadProxyToFile( userDN, userGroup, requiredTimeLeft = 1200, cacheTime = 4 * 43200 ) if not getProxy["OK"]: self.log.error( getProxy["Message" ] ) return S_ERROR( "unable to setup shifter proxy for %s: %s" % ( shifter, getProxy["Message"] ) ) chain = getProxy["chain"] fileName = getProxy["Value" ] self.log.debug( "got %s: %s %s" % ( shifter, userName, userGroup ) ) self.__managersDict[shifter] = { "ShifterDN" : userDN, "ShifterName" : userName, "ShifterGroup" : userGroup, "Chain" : chain, "ProxyFile" : fileName } return S_OK()
def resolveDeps(sysconfig, appli, appversion): """ Resolve the dependencies :param string sysconfig: system configuration :param string appli: application name :param string appversion: application version :return: list of dictionaries """ log = gLogger.getSubLogger("resolveDeps") ops = Operations() deps = ops.getSections( '/AvailableTarBalls/%s/%s/%s/Dependencies' % (sysconfig, appli, appversion), '') depsarray = [] if deps['OK']: for dep in deps['Value']: vers = ops.getValue( '/AvailableTarBalls/%s/%s/%s/Dependencies/%s/version' % (sysconfig, appli, appversion, dep), '') depvers = '' if vers: depvers = vers else: log.error( "Retrieving dependency version for %s failed, skipping to next !" % (dep)) continue log.verbose("Found dependency %s %s" % (dep, depvers)) depdict = {} depdict["app"] = dep depdict["version"] = depvers depsarray.append(depdict) ##resolve recursive dependencies depsofdeps = resolveDeps(sysconfig, dep, depvers) depsarray.extend(depsofdeps) else: log.verbose("Could not find any dependency for %s %s, ignoring" % (appli, appversion)) return depsarray
def matchQueue(jobJDL, queueDict, fullMatch=False): """ Match the job description to the queue definition :param str job: JDL job description :param bool fullMatch: test matching on all the criteria :param dict queueDict: queue parameters dictionary :return: S_OK/S_ERROR, Value - result of matching, S_OK if matched or S_ERROR with the reason for no match """ # Check the job description validity job = ClassAd(jobJDL) if not job.isOK(): return S_ERROR("Invalid job description") noMatchReasons = [] # Check job requirements to resource # 1. CPUTime cpuTime = job.getAttributeInt("CPUTime") if not cpuTime: cpuTime = 84600 if cpuTime > int(queueDict.get("CPUTime", 0)): noMatchReasons.append("Job CPUTime requirement not satisfied") if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 2. Multi-value match requirements for parameter in ["Site", "GridCE", "Platform", "JobType"]: if parameter in queueDict: valueSet = set(job.getListFromExpression(parameter)) if not valueSet: valueSet = set(job.getListFromExpression("%ss" % parameter)) queueSet = set(fromChar(queueDict[parameter])) if valueSet and queueSet and not valueSet.intersection(queueSet): valueToPrint = ",".join(valueSet) if len(valueToPrint) > 20: valueToPrint = "%s..." % valueToPrint[:20] noMatchReasons.append("Job %s %s requirement not satisfied" % (parameter, valueToPrint)) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 3. Banned multi-value match requirements for par in ["Site", "GridCE", "Platform", "JobType"]: parameter = "Banned%s" % par if par in queueDict: valueSet = set(job.getListFromExpression(parameter)) if not valueSet: valueSet = set(job.getListFromExpression("%ss" % parameter)) queueSet = set(fromChar(queueDict[par])) if valueSet and queueSet and valueSet.issubset(queueSet): valueToPrint = ",".join(valueSet) if len(valueToPrint) > 20: valueToPrint = "%s..." % valueToPrint[:20] noMatchReasons.append("Job %s %s requirement not satisfied" % (parameter, valueToPrint)) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 4. Tags tags = set(job.getListFromExpression("Tag")) nProc = job.getAttributeInt("NumberOfProcessors") if nProc and nProc > 1: tags.add("MultiProcessor") wholeNode = job.getAttributeString("WholeNode") if wholeNode: tags.add("WholeNode") queueTags = set(queueDict.get("Tag", [])) if not tags.issubset(queueTags): noMatchReasons.append("Job Tag %s not satisfied" % ",".join(tags)) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 4. MultiProcessor requirements if nProc and nProc > int(queueDict.get("NumberOfProcessors", 1)): noMatchReasons.append("Job NumberOfProcessors %d requirement not satisfied" % nProc) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 5. RAM ram = job.getAttributeInt("RAM") # If MaxRAM is not specified in the queue description, assume 2GB if ram and ram > int(queueDict.get("MaxRAM", 2048) / 1024): noMatchReasons.append("Job RAM %d requirement not satisfied" % ram) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # Check resource requirements to job # 1. OwnerGroup - rare case but still if "OwnerGroup" in queueDict: result = getProxyInfo(disableVOMS=True) if not result["OK"]: return S_ERROR("No valid proxy available") ownerGroup = result["Value"]["group"] if ownerGroup != queueDict["OwnerGroup"]: noMatchReasons.append("Resource OwnerGroup %s requirement not satisfied" % queueDict["OwnerGroup"]) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 2. Required tags requiredTags = set(queueDict.get("RequiredTags", [])) if not requiredTags.issubset(tags): noMatchReasons.append("Resource RequiredTags %s not satisfied" % ",".join(requiredTags)) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 3. RunningLimit site = queueDict["Site"] ce = queueDict.get("GridCE") opsHelper = Operations() result = opsHelper.getSections("JobScheduling/RunningLimit") if result["OK"] and site in result["Value"]: result = opsHelper.getSections("JobScheduling/RunningLimit/%s" % site) if result["OK"]: for parameter in result["Value"]: value = job.getAttributeString(parameter) if ( value and ( opsHelper.getValue("JobScheduling/RunningLimit/%s/%s/%s" % (site, parameter, value), 1) or opsHelper.getValue( "JobScheduling/RunningLimit/%s/CEs/%s/%s/%s" % (site, ce, parameter, value), 1 ) ) == 0 ): noMatchReasons.append("Resource operational %s requirement not satisfied" % parameter) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) return S_OK({"Match": not bool(noMatchReasons), "Reason": noMatchReasons})
class WorkflowTasks(TaskBase): """ Handles jobs """ def __init__(self, transClient=None, logger=None, submissionClient=None, jobMonitoringClient=None, outputDataModule=None, jobClass=None, opsH=None, destinationPlugin=None, ownerDN=None, ownerGroup=None): """ Generates some default objects. jobClass is by default "DIRAC.Interfaces.API.Job.Job". An extension of it also works: VOs can pass in their job class extension, if present """ if not logger: logger = gLogger.getSubLogger('WorkflowTasks') super(WorkflowTasks, self).__init__(transClient, logger) useCertificates = True if (bool(ownerDN) and bool(ownerGroup)) else False if not submissionClient: self.submissionClient = WMSClient(useCertificates=useCertificates, delegatedDN=ownerDN, delegatedGroup=ownerGroup) else: self.submissionClient = submissionClient if not jobMonitoringClient: self.jobMonitoringClient = JobMonitoringClient() else: self.jobMonitoringClient = jobMonitoringClient if not jobClass: self.jobClass = Job else: self.jobClass = jobClass if not opsH: self.opsH = Operations() else: self.opsH = opsH if not outputDataModule: self.outputDataModule = self.opsH.getValue( "Transformations/OutputDataModule", "") else: self.outputDataModule = outputDataModule if not destinationPlugin: self.destinationPlugin = self.opsH.getValue( 'Transformations/DestinationPlugin', 'BySE') else: self.destinationPlugin = destinationPlugin self.destinationPlugin_o = None self.outputDataModule_o = None def prepareTransformationTasks(self, transBody, taskDict, owner='', ownerGroup='', ownerDN='', bulkSubmissionFlag=False): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB jobClass is by default "DIRAC.Interfaces.API.Job.Job". An extension of it also works. :param str transBody: transformation job template :param dict taskDict: dictionary of per task parameters :param str owner: owner of the transformation :param str ownerGroup: group of the owner of the transformation :param str ownerDN: DN of the owner of the transformation :param bool bulkSubmissionFlag: flag for using bulk submission or not :return: S_OK/S_ERROR with updated taskDict """ if (not owner) or (not ownerGroup): res = getProxyInfo(False, False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername(owner) if not res['OK']: return res ownerDN = res['Value'][0] if bulkSubmissionFlag: return self.__prepareTasksBulk(transBody, taskDict, owner, ownerGroup, ownerDN) # not a bulk submission return self.__prepareTasks(transBody, taskDict, owner, ownerGroup, ownerDN) def __prepareTasksBulk(self, transBody, taskDict, owner, ownerGroup, ownerDN): """ Prepare transformation tasks with a single job object for bulk submission :param str transBody: transformation job template :param dict taskDict: dictionary of per task parameters :param str owner: owner of the transformation :param str ownerGroup: group of the owner of the transformation :param str ownerDN: DN of the owner of the transformation :return: S_OK/S_ERROR with updated taskDict """ if taskDict: transID = taskDict.values()[0]['TransformationID'] else: return S_OK({}) method = '__prepareTasksBulk' startTime = time.time() # Prepare the bulk Job object with common parameters oJob = self.jobClass(transBody) self._logVerbose('Setting job owner:group to %s:%s' % (owner, ownerGroup), transID=transID, method=method) oJob.setOwner(owner) oJob.setOwnerGroup(ownerGroup) oJob.setOwnerDN(ownerDN) try: site = oJob.workflow.findParameter('Site').getValue() except AttributeError: site = None jobType = oJob.workflow.findParameter('JobType').getValue() transGroup = str(transID).zfill(8) # Verify that the JOB_ID parameter is added to the workflow if not oJob.workflow.findParameter('JOB_ID'): oJob._addParameter(oJob.workflow, 'JOB_ID', 'string', '00000000', "Initial JOB_ID") if oJob.workflow.findParameter('PRODUCTION_ID'): oJob._setParamValue('PRODUCTION_ID', str(transID).zfill(8)) # pylint: disable=protected-access else: oJob._addParameter( oJob.workflow, # pylint: disable=protected-access 'PRODUCTION_ID', 'string', str(transID).zfill(8), "Production ID") oJob.setType(jobType) self._logVerbose('Adding default transformation group of %s' % (transGroup), transID=transID, method=method) oJob.setJobGroup(transGroup) clinicPath = self._checkSickTransformations(transID) if clinicPath: self._handleHospital(oJob, clinicPath) # Collect per job parameters sequences paramSeqDict = {} # tasks must be sorted because we use bulk submission and we must find the correspondance for taskID in sorted(taskDict): paramsDict = taskDict[taskID] seqDict = {} if site is not None: paramsDict['Site'] = site paramsDict['JobType'] = jobType # Handle destination site sites = self._handleDestination(paramsDict) if not sites: self._logError('Could not get a list a sites', transID=transID, method=method) return S_ERROR(ETSUKN, "Can not evaluate destination site") else: self._logVerbose('Setting Site: ', str(sites), transID=transID, method=method) seqDict['Site'] = sites seqDict['JobName'] = self._transTaskName(transID, taskID) seqDict['JOB_ID'] = str(taskID).zfill(8) self._logDebug('TransID: %s, TaskID: %s, paramsDict: %s' % (transID, taskID, str(paramsDict)), transID=transID, method=method) # Handle Input Data inputData = paramsDict.get('InputData') if inputData: if isinstance(inputData, six.string_types): inputData = inputData.replace(' ', '').split(';') self._logVerbose('Setting input data to %s' % inputData, transID=transID, method=method) seqDict['InputData'] = inputData elif paramSeqDict.get('InputData') is not None: self._logError( "Invalid mixture of jobs with and without input data") return S_ERROR( ETSDATA, "Invalid mixture of jobs with and without input data") for paramName, paramValue in paramsDict.iteritems(): if paramName not in ('InputData', 'Site', 'TargetSE'): if paramValue: self._logVerbose('Setting %s to %s' % (paramName, paramValue), transID=transID, method=method) seqDict[paramName] = paramValue outputParameterList = [] if self.outputDataModule: res = self.getOutputData({ 'Job': oJob._toXML(), # pylint: disable=protected-access 'TransformationID': transID, 'TaskID': taskID, 'InputData': inputData }) if not res['OK']: self._logError("Failed to generate output data", res['Message'], transID=transID, method=method) continue for name, output in res['Value'].iteritems(): seqDict[name] = output outputParameterList.append(name) if oJob.workflow.findParameter(name): oJob._setParamValue(name, "%%(%s)s" % name) # pylint: disable=protected-access else: oJob._addParameter( oJob.workflow, # pylint: disable=protected-access name, 'JDL', "%%(%s)s" % name, name) for pName, seq in seqDict.iteritems(): paramSeqDict.setdefault(pName, []).append(seq) for paramName, paramSeq in paramSeqDict.iteritems(): if paramName in ['JOB_ID', 'PRODUCTION_ID', 'InputData' ] + outputParameterList: res = oJob.setParameterSequence(paramName, paramSeq, addToWorkflow=paramName) else: res = oJob.setParameterSequence(paramName, paramSeq) if not res['OK']: return res if taskDict: self._logInfo('Prepared %d tasks' % len(taskDict), transID=transID, method=method, reftime=startTime) taskDict['BulkJobObject'] = oJob return S_OK(taskDict) def __prepareTasks(self, transBody, taskDict, owner, ownerGroup, ownerDN): """ Prepare transformation tasks with a job object per task :param str transBody: transformation job template :param dict taskDict: dictionary of per task parameters :param owner: owner of the transformation :param str ownerGroup: group of the owner of the transformation :param str ownerDN: DN of the owner of the transformation :return: S_OK/S_ERROR with updated taskDict """ if taskDict: transID = taskDict.values()[0]['TransformationID'] else: return S_OK({}) method = '__prepareTasks' startTime = time.time() oJobTemplate = self.jobClass(transBody) oJobTemplate.setOwner(owner) oJobTemplate.setOwnerGroup(ownerGroup) oJobTemplate.setOwnerDN(ownerDN) try: site = oJobTemplate.workflow.findParameter('Site').getValue() except AttributeError: site = None jobType = oJobTemplate.workflow.findParameter('JobType').getValue() templateOK = False getOutputDataTiming = 0. for taskID, paramsDict in taskDict.iteritems(): # Create a job for each task and add it to the taskDict if not templateOK: templateOK = True # Update the template with common information self._logVerbose('Job owner:group to %s:%s' % (owner, ownerGroup), transID=transID, method=method) transGroup = str(transID).zfill(8) self._logVerbose('Adding default transformation group of %s' % (transGroup), transID=transID, method=method) oJobTemplate.setJobGroup(transGroup) if oJobTemplate.workflow.findParameter('PRODUCTION_ID'): oJobTemplate._setParamValue('PRODUCTION_ID', str(transID).zfill(8)) else: oJobTemplate._addParameter(oJobTemplate.workflow, 'PRODUCTION_ID', 'string', str(transID).zfill(8), "Production ID") if not oJobTemplate.workflow.findParameter('JOB_ID'): oJobTemplate._addParameter(oJobTemplate.workflow, 'JOB_ID', 'string', '00000000', "Initial JOB_ID") if site is not None: paramsDict['Site'] = site paramsDict['JobType'] = jobType # Now create the job from the template oJob = copy.deepcopy(oJobTemplate) constructedName = self._transTaskName(transID, taskID) self._logVerbose('Setting task name to %s' % constructedName, transID=transID, method=method) oJob.setName(constructedName) oJob._setParamValue('JOB_ID', str(taskID).zfill(8)) inputData = None self._logDebug('TransID: %s, TaskID: %s, paramsDict: %s' % (transID, taskID, str(paramsDict)), transID=transID, method=method) # These helper functions do the real job sites = self._handleDestination(paramsDict) if not sites: self._logError('Could not get a list a sites', transID=transID, method=method) paramsDict['TaskObject'] = '' continue else: self._logDebug('Setting Site: ', str(sites), transID=transID, method=method) res = oJob.setDestination(sites) if not res['OK']: self._logError('Could not set the site: %s' % res['Message'], transID=transID, method=method) paramsDict['TaskObject'] = '' continue self._handleInputs(oJob, paramsDict) self._handleRest(oJob, paramsDict) clinicPath = self._checkSickTransformations(transID) if clinicPath: self._handleHospital(oJob, clinicPath) paramsDict['TaskObject'] = '' if self.outputDataModule: getOutputDataTiming -= time.time() res = self.getOutputData({ 'Job': oJob._toXML(), 'TransformationID': transID, 'TaskID': taskID, 'InputData': inputData }) getOutputDataTiming += time.time() if not res['OK']: self._logError("Failed to generate output data", res['Message'], transID=transID, method=method) continue for name, output in res['Value'].iteritems(): oJob._addJDLParameter(name, ';'.join(output)) paramsDict['TaskObject'] = oJob if taskDict: self._logVerbose('Average getOutputData time: %.1f per task' % (getOutputDataTiming / len(taskDict)), transID=transID, method=method) self._logInfo('Prepared %d tasks' % len(taskDict), transID=transID, method=method, reftime=startTime) return S_OK(taskDict) ############################################################################# def _handleDestination(self, paramsDict): """ Handle Sites and TargetSE in the parameters """ try: sites = ['ANY'] if paramsDict['Site']: # 'Site' comes from the XML and therefore is ; separated sites = fromChar(paramsDict['Site'], sepChar=';') except KeyError: pass if self.destinationPlugin_o: destinationPlugin_o = self.destinationPlugin_o else: res = self.__generatePluginObject(self.destinationPlugin) if not res['OK']: self._logFatal( "Could not generate a destination plugin object") return res destinationPlugin_o = res['Value'] self.destinationPlugin_o = destinationPlugin_o destinationPlugin_o.setParameters(paramsDict) destSites = destinationPlugin_o.run() if not destSites: return sites # Now we need to make the AND with the sites, if defined if sites != ['ANY']: # Need to get the AND destSites &= set(sites) return list(destSites) def _handleInputs(self, oJob, paramsDict): """ set job inputs (+ metadata) """ inputData = paramsDict.get('InputData') transID = paramsDict['TransformationID'] if inputData: self._logVerbose('Setting input data to %s' % inputData, transID=transID, method='_handleInputs') res = oJob.setInputData(inputData) if not res['OK']: self._logError("Could not set the inputs: %s" % res['Message'], transID=transID, method='_handleInputs') def _handleRest(self, oJob, paramsDict): """ add as JDL parameters all the other parameters that are not for inputs or destination """ transID = paramsDict['TransformationID'] for paramName, paramValue in paramsDict.iteritems(): if paramName not in ('InputData', 'Site', 'TargetSE'): if paramValue: self._logDebug('Setting %s to %s' % (paramName, paramValue), transID=transID, method='_handleRest') oJob._addJDLParameter(paramName, paramValue) def _checkSickTransformations(self, transID): """ Check if the transformation is in the transformations to be processed at Hospital or Clinic """ transID = int(transID) clinicPath = "Hospital" if transID in set( int(x) for x in self.opsH.getValue( os.path.join(clinicPath, "Transformations"), [])): return clinicPath if "Clinics" in self.opsH.getSections("Hospital").get('Value', []): basePath = os.path.join("Hospital", "Clinics") clinics = self.opsH.getSections(basePath)['Value'] for clinic in clinics: clinicPath = os.path.join(basePath, clinic) if transID in set( int(x) for x in self.opsH.getValue( os.path.join(clinicPath, "Transformations"), [])): return clinicPath return None def _handleHospital(self, oJob, clinicPath): """ Optional handle of hospital/clinic jobs """ if not clinicPath: return oJob.setInputDataPolicy('download', dataScheduling=False) # Check first for a clinic, if not it must be the general hospital hospitalSite = self.opsH.getValue( os.path.join(clinicPath, "ClinicSite"), '') hospitalCEs = self.opsH.getValue(os.path.join(clinicPath, "ClinicCE"), []) # If not found, get the hospital parameters if not hospitalSite: hospitalSite = self.opsH.getValue("Hospital/HospitalSite", 'DIRAC.JobDebugger.ch') if not hospitalCEs: hospitalCEs = self.opsH.getValue("Hospital/HospitalCEs", []) oJob.setDestination(hospitalSite) if hospitalCEs: oJob._addJDLParameter('GridCE', hospitalCEs) def __generatePluginObject(self, plugin): """ This simply instantiates the TaskManagerPlugin class with the relevant plugin name """ method = '__generatePluginObject' try: plugModule = __import__(self.pluginLocation, globals(), locals(), ['TaskManagerPlugin']) except ImportError as e: self._logException("Failed to import 'TaskManagerPlugin' %s: %s" % (plugin, e), method=method) return S_ERROR() try: plugin_o = getattr(plugModule, 'TaskManagerPlugin')('%s' % plugin, operationsHelper=self.opsH) return S_OK(plugin_o) except AttributeError as e: self._logException("Failed to create %s(): %s." % (plugin, e), method=method) return S_ERROR() ############################################################################# def getOutputData(self, paramDict): """ Get the list of job output LFNs from the provided plugin """ if not self.outputDataModule_o: # Create the module object moduleFactory = ModuleFactory() moduleInstance = moduleFactory.getModule(self.outputDataModule, None) if not moduleInstance['OK']: return moduleInstance self.outputDataModule_o = moduleInstance['Value'] # This is the "argument" to the module, set it and then execute self.outputDataModule_o.paramDict = paramDict return self.outputDataModule_o.execute() def submitTransformationTasks(self, taskDict): """ Submit the tasks """ if 'BulkJobObject' in taskDict: return self.__submitTransformationTasksBulk(taskDict) return self.__submitTransformationTasks(taskDict) def __submitTransformationTasksBulk(self, taskDict): """ Submit jobs in one go with one parametric job """ if not taskDict: return S_OK(taskDict) startTime = time.time() method = '__submitTransformationTasksBulk' oJob = taskDict.pop('BulkJobObject') # we can only do this, once the job has been popped, or we _might_ crash transID = taskDict.values()[0]['TransformationID'] if oJob is None: self._logError('no bulk Job object found', transID=transID, method=method) return S_ERROR(ETSUKN, 'No bulk job object provided for submission') result = self.submitTaskToExternal(oJob) if not result['OK']: self._logError('Failed to submit tasks to external', transID=transID, method=method) return result jobIDList = result['Value'] if len(jobIDList) != len(taskDict): for task in taskDict.values(): task['Success'] = False return S_ERROR( ETSUKN, 'Submitted less number of jobs than requested tasks') # Get back correspondance with tasks sorted by ID for jobID, taskID in zip(jobIDList, sorted(taskDict)): taskDict[taskID]['ExternalID'] = jobID taskDict[taskID]['Success'] = True submitted = len(jobIDList) self._logInfo('Submitted %d tasks to WMS in %.1f seconds' % (submitted, time.time() - startTime), transID=transID, method=method) return S_OK(taskDict) def __submitTransformationTasks(self, taskDict): """ Submit jobs one by one """ method = '__submitTransformationTasks' submitted = 0 failed = 0 startTime = time.time() for task in taskDict.itervalues(): transID = task['TransformationID'] if not task['TaskObject']: task['Success'] = False failed += 1 continue res = self.submitTaskToExternal(task['TaskObject']) if res['OK']: task['ExternalID'] = res['Value'] task['Success'] = True submitted += 1 else: self._logError("Failed to submit task to WMS", res['Message'], transID=transID, method=method) task['Success'] = False failed += 1 if submitted: self._logInfo('Submitted %d tasks to WMS in %.1f seconds' % (submitted, time.time() - startTime), transID=transID, method=method) if failed: self._logError('Failed to submit %d tasks to WMS.' % (failed), transID=transID, method=method) return S_OK(taskDict) def submitTaskToExternal(self, job): """ Submits a single job (which can be a bulk one) to the WMS. """ if isinstance(job, six.string_types): try: oJob = self.jobClass(job) except Exception as x: # pylint: disable=broad-except self._logException("Failed to create job object", '', x) return S_ERROR("Failed to create job object") elif isinstance(job, self.jobClass): oJob = job else: self._logError("No valid job description found") return S_ERROR("No valid job description found") workflowFileObject = StringIO.StringIO(oJob._toXML()) jdl = oJob._toJDL(jobDescriptionObject=workflowFileObject) return self.submissionClient.submitJob(jdl, workflowFileObject) def updateTransformationReservedTasks(self, taskDicts): transID = None jobNames = [ self._transTaskName(taskDict['TransformationID'], taskDict['TaskID']) for taskDict in taskDicts ] res = self.jobMonitoringClient.getJobs({'JobName': jobNames}) if not res['OK']: self._logError("Failed to get task from WMS", res['Message'], transID=transID, method='updateTransformationReservedTasks') return res jobNameIDs = {} for wmsID in res['Value']: res = self.jobMonitoringClient.getJobPrimarySummary(int(wmsID)) if not res['OK']: self._logWarn("Failed to get task summary from WMS", res['Message'], transID=transID, method='updateTransformationReservedTasks') else: jobNameIDs[res['Value']['JobName']] = int(wmsID) noTask = list(set(jobNames) - set(jobNameIDs)) return S_OK({'NoTasks': noTask, 'TaskNameIDs': jobNameIDs}) def getSubmittedTaskStatus(self, taskDicts): """ Check the status of a list of tasks and return lists of taskIDs for each new status """ method = 'getSubmittedTaskStatus' if taskDicts: wmsIDs = [ int(taskDict['ExternalID']) for taskDict in taskDicts if int(taskDict['ExternalID']) ] transID = taskDicts[0]['TransformationID'] else: return S_OK({}) res = self.jobMonitoringClient.getJobsStatus(wmsIDs) if not res['OK']: self._logWarn("Failed to get job status from the WMS system", transID=transID, method=method) return res statusDict = res['Value'] updateDict = {} for taskDict in taskDicts: taskID = taskDict['TaskID'] wmsID = int(taskDict['ExternalID']) if not wmsID: continue oldStatus = taskDict['ExternalStatus'] newStatus = statusDict.get(wmsID, {}).get('Status', 'Removed') if oldStatus != newStatus: if newStatus == "Removed": self._logVerbose( 'Production/Job %d/%d removed from WMS while it is in %s status' % (transID, taskID, oldStatus), transID=transID, method=method) newStatus = "Failed" self._logVerbose( 'Setting job status for Production/Job %d/%d to %s' % (transID, taskID, newStatus), transID=transID, method=method) updateDict.setdefault(newStatus, []).append(taskID) return S_OK(updateDict) def getSubmittedFileStatus(self, fileDicts): """ Check the status of a list of files and return the new status of each LFN """ if not fileDicts: return S_OK({}) method = 'getSubmittedFileStatus' # All files are from the same transformation transID = fileDicts[0]['TransformationID'] taskFiles = {} for fileDict in fileDicts: jobName = self._transTaskName(transID, fileDict['TaskID']) taskFiles.setdefault(jobName, {})[fileDict['LFN']] = fileDict['Status'] res = self.updateTransformationReservedTasks(fileDicts) if not res['OK']: self._logWarn("Failed to obtain taskIDs for files", transID=transID, method=method) return res noTasks = res['Value']['NoTasks'] taskNameIDs = res['Value']['TaskNameIDs'] updateDict = {} for jobName in noTasks: for lfn, oldStatus in taskFiles[jobName].iteritems(): if oldStatus != 'Unused': updateDict[lfn] = 'Unused' res = self.jobMonitoringClient.getJobsStatus(taskNameIDs.values()) if not res['OK']: self._logWarn("Failed to get job status from the WMS system", transID=transID, method=method) return res statusDict = res['Value'] for jobName, wmsID in taskNameIDs.iteritems(): jobStatus = statusDict.get(wmsID, {}).get('Status') newFileStatus = { 'Done': 'Processed', 'Completed': 'Processed', 'Failed': 'Unused' }.get(jobStatus) if newFileStatus: for lfn, oldStatus in taskFiles[jobName].iteritems(): if newFileStatus != oldStatus: updateDict[lfn] = newFileStatus return S_OK(updateDict)
def configHelper(voList): """ A helper function to gather necessary Rucio client options from the CS. :param volist: list of VO names, or a VO name (str) :return: a dictionary of a form {vo: params, vo: params,} :rtype: dict """ log = gLogger.getLocalSubLogger("RucioSynchronizerHelper") if isinstance(voList, str): voList = [voList] clientConfig = {} log.debug("VO list to consider for synchronization: ", voList) # locate RucioFileCatalog type in resources first result = gConfig.getSections("/Resources/FileCatalogs") catNames = [] if result["OK"]: catalogs = result["Value"] log.debug("File catalogs defined in Resources", catalogs) for catalog in catalogs: result = gConfig.getOptionsDict(getCatalogPath(catalog)) if result["OK"]: options = result["Value"] log.debug("Rucio Catalog candidate options", options) if options.get("Status", None) == "Active" and options.get( "CatalogType", None) == "RucioFileCatalog": catNames.append(catalog) else: log.error("No catalogs defined in Resources.") return S_ERROR("No catalogs defined in Resources.") log.info( "Active FileCatalogs candidates of type RucioFileCatalog found in Resources:", catNames) # we found (possibly more that one) candidate, now we look for it in Operations # to find out which one is used by which VO. There can be only one # Rucio catalog per VO. for vo in voList: opHelper = Operations(vo=vo) result = opHelper.getSections("/Services/Catalogs") if result["OK"]: catSections = set(result["Value"]) else: log.warn("No Services/Catalogs section in Operations, for ", "VO=%s (skipped)" % vo) continue selectedCatalog = list(catSections.intersection(catNames)) if len(selectedCatalog) > 1: log.error( "VO %s: Services/Catalogs section mis-configured." " More that one Rucio file catalog", "[VO: %s, Catalogs: %s]" % (vo, selectedCatalog), ) continue if not selectedCatalog: log.warn("VO is not using RucioFileCatalog (VO skipped)", "[VO: %s]" % vo) continue # check if the section name is in the catalog list to use. # if the list is not empty it has to contain the selected catalog. fileCatalogs = opHelper.getValue("/Services/Catalogs/CatalogList", []) if fileCatalogs and selectedCatalog[0] not in fileCatalogs: log.warn( "VO is not using RucioFileCatalog - it is not in the catalog list", "[VO: %s]" % vo) continue # now collect Rucio specific parameters for the VO params = {} result = gConfig.getOptionsDict(getCatalogPath(selectedCatalog[0])) if result["OK"]: optDict = result["Value"] params["rucioHost"] = optDict.get("RucioHost", None) params["authHost"] = optDict.get("AuthHost", None) params["privilegedAccount"] = optDict.get("PrivilegedAccount", "root") clientConfig[vo] = params log.info("RSEs and users will be configured in Rucio for the VO:", vo) else: log.error(result["Message"]) return clientConfig
class FileCatalog: ro_methods = [ "exists", "isLink", "readLink", "isFile", "getFileMetadata", "getReplicas", "getReplicaStatus", "getFileSize", "isDirectory", "getDirectoryReplicas", "listDirectory", "getDirectoryMetadata", "getDirectorySize", "getDirectoryContents", "resolveDataset", "getPathPermissions", "getLFNForPFN", "getUsers", "getGroups", "getFileUserMetadata", ] write_methods = [ "createLink", "removeLink", "addFile", "setFileStatus", "addReplica", "removeReplica", "removeFile", "setReplicaStatus", "setReplicaHost", "createDirectory", "setDirectoryStatus", "removeDirectory", "removeDataset", "removeFileFromDataset", "createDataset", ] def __init__(self, catalogs=[], vo=None): """ Default constructor """ self.valid = True self.timeout = 180 self.readCatalogs = [] self.writeCatalogs = [] self.rootConfigPath = "/Resources/FileCatalogs" self.vo = vo if not vo: result = getVOfromProxyGroup() if not result["OK"]: return result self.vo = result["Value"] self.opHelper = Operations(vo=self.vo) if type(catalogs) in types.StringTypes: catalogs = [catalogs] if catalogs: res = self._getSelectedCatalogs(catalogs) else: res = self._getCatalogs() if not res["OK"]: self.valid = False elif (len(self.readCatalogs) == 0) and (len(self.writeCatalogs) == 0): self.valid = False def isOK(self): return self.valid def getReadCatalogs(self): return self.readCatalogs def getWriteCatalogs(self): return self.writeCatalogs def __getattr__(self, name): self.call = name if name in FileCatalog.write_methods: return self.w_execute elif name in FileCatalog.ro_methods: return self.r_execute else: raise AttributeError def __checkArgumentFormat(self, path): if type(path) in types.StringTypes: urls = {path: False} elif type(path) == types.ListType: urls = {} for url in path: urls[url] = False elif type(path) == types.DictType: urls = path else: return S_ERROR("FileCatalog.__checkArgumentFormat: Supplied path is not of the correct format.") return S_OK(urls) def w_execute(self, *parms, **kws): """ Write method executor. """ successful = {} failed = {} failedCatalogs = [] fileInfo = parms[0] res = self.__checkArgumentFormat(fileInfo) if not res["OK"]: return res fileInfo = res["Value"] allLfns = fileInfo.keys() for catalogName, oCatalog, master in self.writeCatalogs: method = getattr(oCatalog, self.call) res = method(fileInfo, **kws) if not res["OK"]: if master: # If this is the master catalog and it fails we dont want to continue with the other catalogs gLogger.error( "FileCatalog.w_execute: Failed to execute %s on master catalog %s." % (self.call, catalogName), res["Message"], ) return res else: # Otherwise we keep the failed catalogs so we can update their state later failedCatalogs.append((catalogName, res["Message"])) else: for lfn, message in res["Value"]["Failed"].items(): # Save the error message for the failed operations if not failed.has_key(lfn): failed[lfn] = {} failed[lfn][catalogName] = message if master: # If this is the master catalog then we should not attempt the operation on other catalogs fileInfo.pop(lfn) for lfn, result in res["Value"]["Successful"].items(): # Save the result return for each file for the successful operations if not successful.has_key(lfn): successful[lfn] = {} successful[lfn][catalogName] = result # This recovers the states of the files that completely failed i.e. when S_ERROR is returned by a catalog for catalogName, errorMessage in failedCatalogs: for lfn in allLfns: if not failed.has_key(lfn): failed[lfn] = {} failed[lfn][catalogName] = errorMessage resDict = {"Failed": failed, "Successful": successful} return S_OK(resDict) def r_execute(self, *parms, **kws): """ Read method executor. """ successful = {} failed = {} for _catalogName, oCatalog, _master in self.readCatalogs: method = getattr(oCatalog, self.call) res = method(*parms, **kws) if res["OK"]: if "Successful" in res["Value"]: for key, item in res["Value"]["Successful"].items(): if not successful.has_key(key): successful[key] = item if failed.has_key(key): failed.pop(key) for key, item in res["Value"]["Failed"].items(): if not successful.has_key(key): failed[key] = item if len(failed) == 0: resDict = {"Failed": failed, "Successful": successful} return S_OK(resDict) else: return res if (len(successful) == 0) and (len(failed) == 0): return S_ERROR("Failed to perform %s from any catalog" % self.call) resDict = {"Failed": failed, "Successful": successful} return S_OK(resDict) ########################################################################################### # # Below is the method for obtaining the objects instantiated for a provided catalogue configuration # def addCatalog(self, catalogName, mode="Write", master=False): """ Add a new catalog with catalogName to the pool of catalogs in mode: "Read","Write" or "ReadWrite" """ result = self._generateCatalogObject(catalogName) if not result["OK"]: return result oCatalog = result["Value"] if mode.lower().find("read") != -1: self.readCatalogs.append((catalogName, oCatalog, master)) if mode.lower().find("write") != -1: self.writeCatalogs.append((catalogName, oCatalog, master)) return S_OK() def removeCatalog(self, catalogName): """ Remove the specified catalog from the internal pool """ catalog_removed = False for i in range(len(self.readCatalogs)): catalog, _object, _master = self.readCatalogs[i] if catalog == catalogName: del self.readCatalogs[i] catalog_removed = True break for i in range(len(self.writeCatalogs)): catalog, _object, _master = self.writeCatalogs[i] if catalog == catalogName: del self.writeCatalogs[i] catalog_removed = True break if catalog_removed: return S_OK() else: return S_OK("Catalog does not exist") def _getSelectedCatalogs(self, desiredCatalogs): for catalogName in desiredCatalogs: res = self._generateCatalogObject(catalogName) if not res["OK"]: return res oCatalog = res["Value"] self.readCatalogs.append((catalogName, oCatalog, True)) self.writeCatalogs.append((catalogName, oCatalog, True)) return S_OK() def _getCatalogs(self): # Get the eligible catalogs first # First, look in the Operations, if nothing defined look in /Resources for backward compatibility result = self.opHelper.getSections("/Services/Catalogs") fileCatalogs = [] operationsFlag = False if result["OK"]: fileCatalogs = result["Value"] operationsFlag = True else: res = gConfig.getSections(self.rootConfigPath, listOrdered=True) if not res["OK"]: errStr = "FileCatalog._getCatalogs: Failed to get file catalog configuration." gLogger.error(errStr, res["Message"]) return S_ERROR(errStr) fileCatalogs = res["Value"] # Get the catalogs now for catalogName in fileCatalogs: res = self._getCatalogConfigDetails(catalogName) if not res["OK"]: return res catalogConfig = res["Value"] if operationsFlag: result = self.opHelper.getOptionsDict("/Services/Catalogs/%s" % catalogName) if not result["OK"]: return result catalogConfig.update(result["Value"]) if catalogConfig["Status"] == "Active": res = self._generateCatalogObject(catalogName) if not res["OK"]: return res oCatalog = res["Value"] master = catalogConfig["Master"] # If the catalog is read type if re.search("Read", catalogConfig["AccessType"]): if master: self.readCatalogs.insert(0, (catalogName, oCatalog, master)) else: self.readCatalogs.append((catalogName, oCatalog, master)) # If the catalog is write type if re.search("Write", catalogConfig["AccessType"]): if master: self.writeCatalogs.insert(0, (catalogName, oCatalog, master)) else: self.writeCatalogs.append((catalogName, oCatalog, master)) return S_OK() def _getCatalogConfigDetails(self, catalogName): # First obtain the options that are available catalogConfigPath = "%s/%s" % (self.rootConfigPath, catalogName) res = gConfig.getOptions(catalogConfigPath) if not res["OK"]: errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options." gLogger.error(errStr, catalogName) return S_ERROR(errStr) catalogConfig = {} for option in res["Value"]: configPath = "%s/%s" % (catalogConfigPath, option) optionValue = gConfig.getValue(configPath) catalogConfig[option] = optionValue # The 'Status' option should be defined (default = 'Active') if not catalogConfig.has_key("Status"): warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined." gLogger.warn(warnStr, catalogName) catalogConfig["Status"] = "Active" # The 'AccessType' option must be defined if not catalogConfig.has_key("AccessType"): errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined." gLogger.error(errStr, catalogName) return S_ERROR(errStr) # Anything other than 'True' in the 'Master' option means it is not if not catalogConfig.has_key("Master"): catalogConfig["Master"] = False elif catalogConfig["Master"] == "True": catalogConfig["Master"] = True else: catalogConfig["Master"] = False return S_OK(catalogConfig) def _generateCatalogObject(self, catalogName): """ Create a file catalog object from its name and CS description """ useProxy = gConfig.getValue("/LocalSite/Catalogs/%s/UseProxy" % catalogName, False) if not useProxy: useProxy = self.opHelper.getValue("/Services/Catalogs/%s/UseProxy" % catalogName, False) return FileCatalogFactory().createCatalog(catalogName, useProxy)
def web_getLaunchpadOpts(self): defaultParams = {"JobName" : [1, 'DIRAC'], "Executable" : [1, "/bin/ls"], "Arguments" : [1, "-ltrA"], "OutputSandbox" : [1, "std.out, std.err"], "JobGroup" : [0, "Unknown"], "InputData" : [0, ""], "OutputData" : [0, ""], "OutputSE" : [0, "DIRAC-USER"], "OutputPath": [0, ""], "CPUTime" : [0, "86400"], "Site" : [0, ""], "BannedSite" : [0, ""], "Platform" : [0, "Linux_x86_64_glibc-2.12"], "Priority" : [0, "5"], "StdError" : [0, "std.err"], "StdOutput" : [0, "std.out"], "Parameters" : [0, "0"], "ParameterStart" : [0, "0"], "ParameterStep" : [0, "1"], "ParameterFactor": [0, "0"]} delimiter = gConfig.getValue("/WebApp/Launchpad/ListSeparator" , ',') options = self.__getOptionsFromCS(delimiter=delimiter) # platform = self.__getPlatform() # if platform and options: # if not options.has_key("Platform"): # options[ "Platform" ] = platform # else: # csPlatform = list(options[ "Platform" ]) # allPlatforms = csPlatform + platform # platform = uniqueElements(allPlatforms) # options[ "Platform" ] = platform gLogger.debug("Combined options from CS: %s" % options) override = gConfig.getValue("/WebApp/Launchpad/OptionsOverride" , False) gLogger.info("end __getLaunchpadOpts") # Updating the default values from OptionsOverride configuration branch for key in options: if key not in defaultParams: defaultParams[key] = [ 0, "" ] defaultParams[key][1] = options[key][0] # Reading of the predefined sets of launchpad parameters values obj = Operations( vo = self.vo ) predefinedSets = {} launchpadSections = obj.getSections("Launchpad") import pprint if launchpadSections['OK']: for section in launchpadSections["Value"]: predefinedSets[section] = {} sectionOptions = obj.getOptionsDict("Launchpad/" + section) pprint.pprint(sectionOptions) if sectionOptions['OK']: predefinedSets[section] = sectionOptions["Value"] self.write({"success":"true", "result":defaultParams, "predefinedSets":predefinedSets})
class CombinedSoftwareInstallation(object): """ Combined means that it will try to install in the Shared area and in the LocalArea, depending on the user's rights """ def __init__(self, argumentsDict): """ Standard constructor Defines, from dictionary of job parameters passed, a set of members to hold e.g. the applications and the system config. Also determines the SharedArea and LocalArea. """ self.ops = Operations() self.job = argumentsDict.get('Job', {}) self.ce = argumentsDict.get('CE', {}) self.source = argumentsDict.get('Source', {}) apps = [] if 'SoftwarePackages' in self.job: if isinstance( self.job['SoftwarePackages'], basestring): apps = [self.job['SoftwarePackages']] elif isinstance( self.job['SoftwarePackages'], list ): apps = self.job['SoftwarePackages'] self.jobConfig = '' if 'SystemConfig' in self.job: self.jobConfig = self.job['SystemConfig'] elif 'Platform' in self.job: self.jobConfig = self.job['Platform'] else: self.jobConfig = natOS.CMTSupportedConfig()[0] self.apps = [] for app in apps: LOG.verbose('Requested Package %s' % app) app = tuple(app.split('.')) if len(app) > 2: tempapp = app app = [] app.append(tempapp[0]) app.append(".".join(tempapp[1:])) deps = resolveDeps(self.jobConfig, app[0], app[1]) for dep in deps: depapp = (dep['app'], dep['version']) self.apps.append(depapp) self.apps.append(app) self.ceConfigs = [] if 'CompatiblePlatforms' in self.ce: self.ceConfigs = self.ce['CompatiblePlatforms'] if isinstance( self.ceConfigs, basestring ): self.ceConfigs = [self.ceConfigs] #else: ### Use always the list of compatible platform. self.ceConfigs = natOS.CMTSupportedConfig() self.sharedArea = getSharedAreaLocation() LOG.info("SharedArea is %s" % self.sharedArea) self.localArea = getLocalAreaLocation() LOG.info("LocalArea is %s" % self.localArea) def execute(self): """ Main method of the class executed by DIRAC jobAgent Executes the following: - look for the compatible platforms in the CS, see if one matches request - install the applications, calls :mod:`~ILCDIRAC.Core.Utilities.TARsoft` :return: S_OK(), S_ERROR() """ if not self.apps: # There is nothing to do return DIRAC.S_OK() if not self.jobConfig: LOG.error('No architecture requested') return DIRAC.S_ERROR( 'No architecture requested' ) found_config = False LOG.info("Found CE Configs %s, compatible with system reqs %s" % (",".join(self.ceConfigs), self.jobConfig)) res = self.ops.getSections('/AvailableTarBalls') if not res['OK']: return res else: supported_systems = res['Value'] ###look in CS if specified platform has software available. Normally consistency check is done at submission time for ceConfig in self.ceConfigs: for supp_systems in supported_systems: if ceConfig == supp_systems: self.jobConfig = ceConfig found_config = True break if not found_config: if self.ceConfigs: # redundant check as this is done in the job agent, if locally running option might not be defined LOG.error('Requested architecture not supported by CE') return DIRAC.S_ERROR( 'Requested architecture not supported by CE' ) else: LOG.info('Assume locally running job, will install software in ') areas = [] ###Deal with shared/local area: first try to see if the Shared area exists and if not create it (try to). If it fails, fall back to local area if not self.sharedArea: if createSharedArea(): self.sharedArea = getSharedAreaLocation() if self.sharedArea: areas.append(self.sharedArea) LOG.info("Will attempt to install in shared area") else: areas.append(self.sharedArea) areas.append(self.localArea) for app in self.apps: res = checkCVMFS(self.jobConfig, app) if res['OK']: LOG.notice('Software %s is available on CVMFS, skipping' % ", ".join(app)) continue ## First install the original package in any of the areas resInstall = installInAnyArea(areas, app, self.jobConfig) if not resInstall['OK']: return resInstall ## Then install the dependencies, which can be in a different area resDeps = installDependencies(app, self.jobConfig, areas) if not resDeps['OK']: LOG.error("Failed to install dependencies: ", resDeps['Message']) return S_ERROR("Failed to install dependencies") if self.sharedArea: #List content listAreaDirectory(self.sharedArea) return DIRAC.S_OK()
class FileCatalog( object ): def __init__( self, catalogs = None, vo = None ): """ Default constructor """ self.valid = True self.timeout = 180 self.ro_methods = set() self.write_methods = set() self.no_lfn_methods = set() self.readCatalogs = [] self.writeCatalogs = [] self.rootConfigPath = '/Resources/FileCatalogs' self.vo = vo if vo else getVOfromProxyGroup().get( 'Value', None ) self.log = gLogger.getSubLogger( "FileCatalog" ) self.opHelper = Operations( vo = self.vo ) catalogList = [] if isinstance( catalogs, basestring ): catalogList = [catalogs] elif isinstance( catalogs, ( list, tuple ) ): catalogList = list( catalogs ) if catalogList: result = self._getEligibleCatalogs() if not result['OK']: self.log.error( "Failed to get eligible catalog" ) return eligibleFileCatalogs = result['Value'] catalogCheck = True for catalog in catalogList: if catalog not in eligibleFileCatalogs: self.log.error( "Specified catalog is not eligible", catalog ) catalogCheck = False if catalogCheck: result = self._getSelectedCatalogs( catalogList ) else: result = S_ERROR( "Specified catalog is not eligible" ) else: result = self._getCatalogs() if not result['OK']: self.log.error( "Failed to create catalog objects" ) self.valid = False elif ( len( self.readCatalogs ) == 0 ) and ( len( self.writeCatalogs ) == 0 ): self.log.error( "No catalog object created" ) self.valid = False result = self.getMasterCatalogNames() masterCatalogs = result['Value'] # There can not be more than one master catalog haveMaster = False if len( masterCatalogs ) > 1: self.log.error( "More than one master catalog created" ) self.valid = False elif len( masterCatalogs ) == 1: haveMaster = True # Get the list of write methods if haveMaster: # All the write methods must be present in the master _catalogName, oCatalog, _master = self.writeCatalogs[0] _roList, writeList, nolfnList = oCatalog.getInterfaceMethods() self.write_methods.update( writeList ) self.no_lfn_methods.update( nolfnList ) else: for _catalogName, oCatalog, _master in self.writeCatalogs: _roList, writeList, nolfnList = oCatalog.getInterfaceMethods() self.write_methods.update( writeList ) self.no_lfn_methods.update( nolfnList ) # Get the list of read methods for _catalogName, oCatalog, _master in self.readCatalogs: roList, _writeList, nolfnList = oCatalog.getInterfaceMethods() self.ro_methods.update( roList ) self.no_lfn_methods.update( nolfnList ) self.condParser = FCConditionParser( vo = self.vo, ro_methods = self.ro_methods ) def isOK( self ): return self.valid def getReadCatalogs( self ): return self.readCatalogs def getWriteCatalogs( self ): return self.writeCatalogs def getMasterCatalogNames( self ): """ Returns the list of names of the Master catalogs """ masterNames = [catalogName for catalogName, oCatalog, master in self.writeCatalogs if master] return S_OK( masterNames ) def __getattr__( self, name ): self.call = name if name in self.write_methods: return self.w_execute elif name in self.ro_methods: return self.r_execute else: raise AttributeError def w_execute( self, *parms, **kws ): """ Write method executor. If one of the LFNs given as input does not pass a condition defined for the master catalog, we return S_ERROR without trying anything else :param fcConditions: either a dict or a string, to be propagated to the FCConditionParser If it is a string, it is given for all catalogs If it is a dict, it has to be { catalogName: condition}, and only the specific condition for the catalog will be given CAUTION !!! If the method is a write no_lfn method, then the return value are completely different We only return the result of the master catalog """ successful = {} failed = {} failedCatalogs = {} successfulCatalogs = {} specialConditions = kws.pop( 'fcConditions' ) if 'fcConditions' in kws else None allLfns = [] lfnMapDict = {} masterResult = {} parms1 = [] if self.call not in self.no_lfn_methods: fileInfo = parms[0] result = checkArgumentFormat( fileInfo, generateMap = True ) if not result['OK']: return result fileInfo, lfnMapDict = result['Value'] # No need to check the LFNs again in the clients kws['LFNChecking'] = False allLfns = fileInfo.keys() parms1 = parms[1:] for catalogName, oCatalog, master in self.writeCatalogs: # Skip if the method is not implemented in this catalog # NOTE: it is impossible for the master since the write method list is populated # only from the master catalog, and if the method is not there, __getattr__ # would raise an exception if not oCatalog.hasCatalogMethod( self.call ): continue method = getattr( oCatalog, self.call ) if self.call in self.no_lfn_methods: result = method( *parms, **kws ) else: if isinstance( specialConditions, dict ): condition = specialConditions.get( catalogName ) else: condition = specialConditions # Check whether this catalog should be used for this method res = self.condParser( catalogName, self.call, fileInfo, condition = condition ) # condParser never returns S_ERROR condEvals = res['Value']['Successful'] # For a master catalog, ALL the lfns should be valid if master: if any([not valid for valid in condEvals.values()]): gLogger.error( "The master catalog is not valid for some LFNS", condEvals ) return S_ERROR( "The master catalog is not valid for some LFNS %s" % condEvals ) validLFNs = dict( ( lfn, fileInfo[lfn] ) for lfn in condEvals if condEvals[lfn] ) invalidLFNs = [lfn for lfn in condEvals if not condEvals[lfn]] if invalidLFNs: gLogger.debug( "Some LFNs are not valid for operation '%s' on catalog '%s' : %s" % ( self.call, catalogName, invalidLFNs ) ) result = method( validLFNs, *parms1, **kws ) if master: masterResult = result if not result['OK']: if master: # If this is the master catalog and it fails we don't want to continue with the other catalogs self.log.error( "Failed to execute call on master catalog", "%s on %s: %s" % ( self.call, catalogName, result['Message'] ) ) return result else: # Otherwise we keep the failed catalogs so we can update their state later failedCatalogs[catalogName] = result['Message'] else: successfulCatalogs[catalogName] = result['Value'] if allLfns: if result['OK']: for lfn, message in result['Value']['Failed'].items(): # Save the error message for the failed operations failed.setdefault( lfn, {} )[catalogName] = message if master: # If this is the master catalog then we should not attempt the operation on other catalogs fileInfo.pop( lfn, None ) for lfn, result in result['Value']['Successful'].items(): # Save the result return for each file for the successful operations successful.setdefault( lfn, {} )[catalogName] = result if allLfns: # This recovers the states of the files that completely failed i.e. when S_ERROR is returned by a catalog for catalogName, errorMessage in failedCatalogs.items(): for lfn in allLfns: failed.setdefault( lfn, {} )[catalogName] = errorMessage # Restore original lfns if they were changed by normalization if lfnMapDict: for lfn in failed.keys(): failed[lfnMapDict.get( lfn, lfn )] = failed.pop( lfn ) for lfn in successful.keys(): successful[lfnMapDict.get( lfn, lfn )] = successful.pop( lfn ) resDict = {'Failed':failed, 'Successful':successful} return S_OK( resDict ) else: # FIXME: Return just master result here. This is temporary as more detailed # per catalog result needs multiple fixes in various client calls return masterResult def r_execute( self, *parms, **kws ): """ Read method executor. """ successful = {} failed = {} for _catalogName, oCatalog, _master in self.readCatalogs: # Skip if the method is not implemented in this catalog if not oCatalog.hasCatalogMethod( self.call ): continue method = getattr( oCatalog, self.call ) res = method( *parms, **kws ) if res['OK']: if 'Successful' in res['Value']: for key, item in res['Value']['Successful'].items(): successful.setdefault( key, item ) failed.pop( key, None ) for key, item in res['Value']['Failed'].items(): if key not in successful: failed[key] = item else: return res if not successful and not failed: return S_ERROR( DErrno.EFCERR, "Failed to perform %s from any catalog" % self.call ) return S_OK( {'Failed':failed, 'Successful':successful} ) ########################################################################################### # # Below is the method for obtaining the objects instantiated for a provided catalogue configuration # def addCatalog( self, catalogName, mode = "Write", master = False ): """ Add a new catalog with catalogName to the pool of catalogs in mode: "Read","Write" or "ReadWrite" """ result = self._generateCatalogObject( catalogName ) if not result['OK']: return result oCatalog = result['Value'] if mode.lower().find( "read" ) != -1: self.readCatalogs.append( ( catalogName, oCatalog, master ) ) if mode.lower().find( "write" ) != -1: self.writeCatalogs.append( ( catalogName, oCatalog, master ) ) return S_OK() def removeCatalog( self, catalogName ): """ Remove the specified catalog from the internal pool """ catalog_removed = False for i in range( len( self.readCatalogs ) ): catalog, _object, _master = self.readCatalogs[i] if catalog == catalogName: del self.readCatalogs[i] catalog_removed = True break for i in range( len( self.writeCatalogs ) ): catalog, _object, _master = self.writeCatalogs[i] if catalog == catalogName: del self.writeCatalogs[i] catalog_removed = True break if catalog_removed: return S_OK() else: return S_OK( 'Catalog does not exist' ) def _getSelectedCatalogs( self, desiredCatalogs ): for catalogName in desiredCatalogs: result = self._getCatalogConfigDetails( catalogName ) if not result['OK']: return result catalogConfig = result['Value'] result = self._generateCatalogObject( catalogName ) if not result['OK']: return result oCatalog = result['Value'] if re.search( 'Read', catalogConfig['AccessType'] ): if catalogConfig['Master']: self.readCatalogs.insert( 0, ( catalogName, oCatalog, catalogConfig['Master'] ) ) else: self.readCatalogs.append( ( catalogName, oCatalog, catalogConfig['Master'] ) ) if re.search( 'Write', catalogConfig['AccessType'] ): if catalogConfig['Master']: self.writeCatalogs.insert( 0, ( catalogName, oCatalog, catalogConfig['Master'] ) ) else: self.writeCatalogs.append( ( catalogName, oCatalog, catalogConfig['Master'] ) ) return S_OK() def _getEligibleCatalogs( self ): """ Get a list of eligible catalogs :return: S_OK/S_ERROR, Value - a list of catalog names """ # First, look in the Operations, if nothing defined look in /Resources for backward compatibility fileCatalogs = self.opHelper.getValue( '/Services/Catalogs/CatalogList', [] ) if not fileCatalogs: result = self.opHelper.getSections( '/Services/Catalogs' ) if result['OK']: fileCatalogs = result['Value'] else: res = gConfig.getSections( self.rootConfigPath, listOrdered = True ) if not res['OK']: errStr = "FileCatalog._getEligibleCatalogs: Failed to get file catalog configuration." self.log.error( errStr, res['Message'] ) return S_ERROR( errStr ) fileCatalogs = res['Value'] return S_OK( fileCatalogs ) def _getCatalogs( self ): """ Updates self.readCatalogs and self.writeCatalogs with list of catalog objects as found in the CS """ # Get the eligible catalogs first result = self._getEligibleCatalogs() if not result['OK']: return result fileCatalogs = result['Value'] # Get the catalog objects now for catalogName in fileCatalogs: res = self._getCatalogConfigDetails( catalogName ) if not res['OK']: return res catalogConfig = res['Value'] if catalogConfig['Status'] == 'Active': res = self._generateCatalogObject( catalogName ) if not res['OK']: return res oCatalog = res['Value'] master = catalogConfig['Master'] # If the catalog is read type if re.search( 'Read', catalogConfig['AccessType'] ): if master: self.readCatalogs.insert( 0, ( catalogName, oCatalog, master ) ) else: self.readCatalogs.append( ( catalogName, oCatalog, master ) ) # If the catalog is write type if re.search( 'Write', catalogConfig['AccessType'] ): if master: self.writeCatalogs.insert( 0, ( catalogName, oCatalog, master ) ) else: self.writeCatalogs.append( ( catalogName, oCatalog, master ) ) return S_OK() def _getCatalogConfigDetails( self, catalogName ): # First obtain the options that are available catalogConfigPath = '%s/%s' % ( self.rootConfigPath, catalogName ) result = gConfig.getOptionsDict( catalogConfigPath ) if not result['OK']: errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options." self.log.error( errStr, catalogName ) return S_ERROR( errStr ) catalogConfig = result['Value'] result = self.opHelper.getOptionsDict( '/Services/Catalogs/%s' % catalogName ) if result['OK']: catalogConfig.update( result['Value'] ) # The 'Status' option should be defined (default = 'Active') if 'Status' not in catalogConfig: warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined." self.log.warn( warnStr, catalogName ) catalogConfig['Status'] = 'Active' # The 'AccessType' option must be defined if 'AccessType' not in catalogConfig: errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined." self.log.error( errStr, catalogName ) return S_ERROR( errStr ) # Anything other than 'True' in the 'Master' option means it is not catalogConfig['Master'] = ( catalogConfig.setdefault( 'Master', False ) == 'True' ) return S_OK( catalogConfig ) def _generateCatalogObject( self, catalogName ): """ Create a file catalog object from its name and CS description """ useProxy = gConfig.getValue( '/LocalSite/Catalogs/%s/UseProxy' % catalogName, False ) if not useProxy: useProxy = self.opHelper.getValue( '/Services/Catalogs/%s/UseProxy' % catalogName, False ) return FileCatalogFactory().createCatalog( catalogName, useProxy )
def matchQueue(jobJDL, queueDict, fullMatch=False): """ Match the job description to the queue definition :param str job: JDL job description :param bool fullMatch: test matching on all the criteria :param dict queueDict: queue parameters dictionary :return: S_OK/S_ERROR, Value - result of matching, S_OK if matched or S_ERROR with the reason for no match """ # Check the job description validity job = ClassAd(jobJDL) if not job.isOK(): return S_ERROR('Invalid job description') noMatchReasons = [] # Check job requirements to resource # 1. CPUTime cpuTime = job.getAttributeInt('CPUTime') if not cpuTime: cpuTime = 84600 if cpuTime > queueDict.get('CPUTime', 0.): noMatchReasons.append('Job CPUTime requirement not satisfied') if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 2. Multi-value match requirements for parameter in [ 'Site', 'GridCE', 'Platform', 'GridMiddleware', 'PilotType', 'SubmitPool', 'JobType' ]: if parameter in queueDict: valueSet = set(job.getListFromExpression(parameter)) if not valueSet: valueSet = set(job.getListFromExpression('%ss' % parameter)) queueSet = set(fromChar(queueDict[parameter])) if valueSet and queueSet and not valueSet.intersection(queueSet): valueToPrint = ','.join(valueSet) if len(valueToPrint) > 20: valueToPrint = "%s..." % valueToPrint[:20] noMatchReasons.append('Job %s %s requirement not satisfied' % (parameter, valueToPrint)) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 3. Banned multi-value match requirements for par in [ 'Site', 'GridCE', 'Platform', 'GridMiddleware', 'PilotType', 'SubmitPool', 'JobType' ]: parameter = "Banned%s" % par if par in queueDict: valueSet = set(job.getListFromExpression(parameter)) if not valueSet: valueSet = set(job.getListFromExpression('%ss' % parameter)) queueSet = set(fromChar(queueDict[par])) if valueSet and queueSet and valueSet.issubset(queueSet): valueToPrint = ','.join(valueSet) if len(valueToPrint) > 20: valueToPrint = "%s..." % valueToPrint[:20] noMatchReasons.append('Job %s %s requirement not satisfied' % (parameter, valueToPrint)) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 4. Tags tags = set(job.getListFromExpression('Tag')) nProc = job.getAttributeInt('NumberOfProcessors') if nProc and nProc > 1: tags.add('MultiProcessor') wholeNode = job.getAttributeString('WholeNode') if wholeNode: tags.add('WholeNode') queueTags = set(queueDict.get('Tags', [])) if not tags.issubset(queueTags): noMatchReasons.append('Job Tag %s not satisfied' % ','.join(tags)) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 4. MultiProcessor requirements if nProc and nProc > int(queueDict.get('NumberOfProcessors', 1)): noMatchReasons.append( 'Job NumberOfProcessors %d requirement not satisfied' % nProc) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 5. RAM ram = job.getAttributeInt('RAM') # If MaxRAM is not specified in the queue description, assume 2GB if ram and ram > int(queueDict.get('MaxRAM', 2048)) / 1024: noMatchReasons.append('Job RAM %d requirement not satisfied' % ram) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # Check resource requirements to job # 1. OwnerGroup - rare case but still if "OwnerGroup" in queueDict: result = getProxyInfo(disableVOMS=True) if not result['OK']: return S_ERROR('No valid proxy available') ownerGroup = result['Value']['group'] if ownerGroup != queueDict['OwnerGroup']: noMatchReasons.append( 'Resource OwnerGroup %s requirement not satisfied' % queueDict['OwnerGroup']) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 2. Required tags requiredTags = set(queueDict.get('RequiredTags', [])) if not requiredTags.issubset(tags): noMatchReasons.append('Resource RequiredTags %s not satisfied' % ','.join(requiredTags)) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 3. RunningLimit site = queueDict['Site'] opsHelper = Operations() result = opsHelper.getSections('JobScheduling/RunningLimit') if result['OK'] and site in result['Value']: result = opsHelper.getSections('JobScheduling/RunningLimit/%s' % site) if result['OK']: for parameter in result['Value']: value = job.getAttributeString(parameter) if value and opsHelper.getValue( 'JobScheduling/RunningLimit/%s/%s/%s' % (site, parameter, value), 1) == 0: noMatchReasons.append( 'Resource operational %s requirement not satisfied' % parameter) if not fullMatch: return S_OK({ 'Match': False, 'Reason': noMatchReasons[0] }) return S_OK({'Match': not bool(noMatchReasons), 'Reason': noMatchReasons})
class Limiter(object): def __init__(self, jobDB=None, opsHelper=None): """ Constructor """ self.__runningLimitSection = "JobScheduling/RunningLimit" self.__matchingDelaySection = "JobScheduling/MatchingDelay" self.csDictCache = DictCache() self.condCache = DictCache() self.delayMem = {} if jobDB: self.jobDB = jobDB else: self.jobDB = JobDB() self.log = gLogger.getSubLogger("Limiter") if opsHelper: self.__opsHelper = opsHelper else: self.__opsHelper = Operations() def getNegativeCond(self): """ Get negative condition for ALL sites """ orCond = self.condCache.get("GLOBAL") if orCond: return orCond negCond = {} # Run Limit result = self.__opsHelper.getSections(self.__runningLimitSection) sites = [] if result['OK']: sites = result['Value'] for siteName in sites: result = self.__getRunningCondition(siteName) if not result['OK']: continue data = result['Value'] if data: negCond[siteName] = data # Delay limit result = self.__opsHelper.getSections(self.__matchingDelaySection) sites = [] if result['OK']: sites = result['Value'] for siteName in sites: result = self.__getDelayCondition(siteName) if not result['OK']: continue data = result['Value'] if not data: continue if siteName in negCond: negCond[siteName] = self.__mergeCond(negCond[siteName], data) else: negCond[siteName] = data orCond = [] for siteName in negCond: negCond[siteName]['Site'] = siteName orCond.append(negCond[siteName]) self.condCache.add("GLOBAL", 10, orCond) return orCond def getNegativeCondForSite(self, siteName): """ Generate a negative query based on the limits set on the site """ # Check if Limits are imposed onto the site negativeCond = {} if self.__opsHelper.getValue("JobScheduling/CheckJobLimits", True): result = self.__getRunningCondition(siteName) if result['OK']: negativeCond = result['Value'] self.log.verbose('Negative conditions for site %s after checking limits are: %s' % (siteName, str(negativeCond))) if self.__opsHelper.getValue("JobScheduling/CheckMatchingDelay", True): result = self.__getDelayCondition(siteName) if result['OK']: delayCond = result['Value'] self.log.verbose('Negative conditions for site %s after delay checking are: %s' % (siteName, str(delayCond))) negativeCond = self.__mergeCond(negativeCond, delayCond) if negativeCond: self.log.info('Negative conditions for site %s are: %s' % (siteName, str(negativeCond))) return negativeCond def __mergeCond(self, negCond, addCond): """ Merge two negative dicts """ # Merge both negative dicts for attr in addCond: if attr not in negCond: negCond[attr] = [] for value in addCond[attr]: if value not in negCond[attr]: negCond[attr].append(value) return negCond def __extractCSData(self, section): """ Extract limiting information from the CS in the form: { 'JobType' : { 'Merge' : 20, 'MCGen' : 1000 } } """ stuffDict = self.csDictCache.get(section) if stuffDict: return S_OK(stuffDict) result = self.__opsHelper.getSections(section) if not result['OK']: return result attribs = result['Value'] stuffDict = {} for attName in attribs: result = self.__opsHelper.getOptionsDict("%s/%s" % (section, attName)) if not result['OK']: return result attLimits = result['Value'] try: attLimits = dict([(k, int(attLimits[k])) for k in attLimits]) except Exception as excp: errMsg = "%s/%s has to contain numbers: %s" % (section, attName, str(excp)) self.log.error(errMsg) return S_ERROR(errMsg) stuffDict[attName] = attLimits self.csDictCache.add(section, 300, stuffDict) return S_OK(stuffDict) def __getRunningCondition(self, siteName): """ Get extra conditions allowing site throttling """ siteSection = "%s/%s" % (self.__runningLimitSection, siteName) result = self.__extractCSData(siteSection) if not result['OK']: return result limitsDict = result['Value'] # limitsDict is something like { 'JobType' : { 'Merge' : 20, 'MCGen' : 1000 } } if not limitsDict: return S_OK({}) # Check if the site exceeding the given limits negCond = {} for attName in limitsDict: if attName not in self.jobDB.jobAttributeNames: self.log.error("Attribute %s does not exist. Check the job limits" % attName) continue cK = "Running:%s:%s" % (siteName, attName) data = self.condCache.get(cK) if not data: result = self.jobDB.getCounters( 'Jobs', [attName], { 'Site': siteName, 'Status': [ 'Running', 'Matched', 'Stalled']}) if not result['OK']: return result data = result['Value'] data = dict([(k[0][attName], k[1]) for k in data]) self.condCache.add(cK, 10, data) for attValue in limitsDict[attName]: limit = limitsDict[attName][attValue] running = data.get(attValue, 0) if running >= limit: self.log.verbose('Job Limit imposed at %s on %s/%s=%d,' ' %d jobs already deployed' % (siteName, attName, attValue, limit, running)) if attName not in negCond: negCond[attName] = [] negCond[attName].append(attValue) # negCond is something like : {'JobType': ['Merge']} return S_OK(negCond) def updateDelayCounters(self, siteName, jid): # Get the info from the CS siteSection = "%s/%s" % (self.__matchingDelaySection, siteName) result = self.__extractCSData(siteSection) if not result['OK']: return result delayDict = result['Value'] # limitsDict is something like { 'JobType' : { 'Merge' : 20, 'MCGen' : 1000 } } if not delayDict: return S_OK() attNames = [] for attName in delayDict: if attName not in self.jobDB.jobAttributeNames: self.log.error("Attribute %s does not exist in the JobDB. Please fix it!" % attName) else: attNames.append(attName) result = self.jobDB.getJobAttributes(jid, attNames) if not result['OK']: self.log.error("While retrieving attributes coming from %s: %s" % (siteSection, result['Message'])) return result atts = result['Value'] # Create the DictCache if not there if siteName not in self.delayMem: self.delayMem[siteName] = DictCache() # Update the counters delayCounter = self.delayMem[siteName] for attName in atts: attValue = atts[attName] if attValue in delayDict[attName]: delayTime = delayDict[attName][attValue] self.log.notice("Adding delay for %s/%s=%s of %s secs" % (siteName, attName, attValue, delayTime)) delayCounter.add((attName, attValue), delayTime) return S_OK() def __getDelayCondition(self, siteName): """ Get extra conditions allowing matching delay """ if siteName not in self.delayMem: return S_OK({}) lastRun = self.delayMem[siteName].getKeys() negCond = {} for attName, attValue in lastRun: if attName not in negCond: negCond[attName] = [] negCond[attName].append(attValue) return S_OK(negCond)
class FileCatalog(object): ro_methods = [ 'exists', 'isLink', 'readLink', 'isFile', 'getFileMetadata', 'getReplicas', 'getReplicaStatus', 'getFileSize', 'isDirectory', 'getDirectoryReplicas', 'listDirectory', 'getDirectoryMetadata', 'getDirectorySize', 'getDirectoryContents', 'resolveDataset', 'getPathPermissions', 'getLFNForPFN', 'getUsers', 'getGroups', 'getLFNForGUID' ] ro_meta_methods = [ 'getFileUserMetadata', 'getMetadataFields', 'findFilesByMetadata', 'getFileUserMetadata', 'findDirectoriesByMetadata', 'getReplicasByMetadata', 'findFilesByMetadataDetailed', 'findFilesByMetadataWeb', 'getCompatibleMetadata', 'getMetadataSet' ] ro_methods += ro_meta_methods write_methods = [ 'createLink', 'removeLink', 'addFile', 'setFileStatus', 'addReplica', 'removeReplica', 'removeFile', 'setReplicaStatus', 'setReplicaHost', 'setReplicaProblematic', 'createDirectory', 'setDirectoryStatus', 'removeDirectory', 'removeDataset', 'removeFileFromDataset', 'createDataset', 'changePathMode', 'changePathOwner', 'changePathGroup' ] write_meta_methods = [ 'addMetadataField', 'deleteMetadataField', 'setMetadata', 'setMetadataBulk', 'removeMetadata', 'addMetadataSet' ] write_methods += write_meta_methods def __init__(self, catalogs=None, vo=None): """ Default constructor """ self.valid = True self.timeout = 180 self.readCatalogs = [] self.writeCatalogs = [] self.metaCatalogs = [] self.rootConfigPath = '/Resources/FileCatalogs' self.vo = vo if vo else getVOfromProxyGroup().get('Value', None) self.opHelper = Operations(vo=self.vo) if catalogs is None: catalogList = [] elif type(catalogs) in types.StringTypes: catalogList = [catalogs] else: catalogList = catalogs if catalogList: res = self._getSelectedCatalogs(catalogList) else: res = self._getCatalogs() if not res['OK']: self.valid = False elif (len(self.readCatalogs) == 0) and (len(self.writeCatalogs) == 0): self.valid = False def isOK(self): return self.valid def getReadCatalogs(self): return self.readCatalogs def getWriteCatalogs(self): return self.writeCatalogs def getMasterCatalogNames(self): """ Returns the list of names of the Master catalogs """ masterNames = [ catalogName for catalogName, oCatalog, master in self.writeCatalogs if master ] return S_OK(masterNames) def __getattr__(self, name): self.call = name if name in FileCatalog.write_methods: return self.w_execute elif name in FileCatalog.ro_methods: return self.r_execute else: raise AttributeError def w_execute(self, *parms, **kws): """ Write method executor. """ successful = {} failed = {} failedCatalogs = [] fileInfo = parms[0] res = checkArgumentFormat(fileInfo) if not res['OK']: return res fileInfo = res['Value'] allLfns = fileInfo.keys() parms = parms[1:] for catalogName, oCatalog, master in self.writeCatalogs: # Skip if metadata related method on pure File Catalog if self.call in FileCatalog.write_meta_methods and not catalogName in self.metaCatalogs: continue method = getattr(oCatalog, self.call) res = method(fileInfo, *parms, **kws) if not res['OK']: if master: # If this is the master catalog and it fails we dont want to continue with the other catalogs gLogger.error( "FileCatalog.w_execute: Failed to execute %s on master catalog %s." % (self.call, catalogName), res['Message']) return res else: # Otherwise we keep the failed catalogs so we can update their state later failedCatalogs.append((catalogName, res['Message'])) else: for lfn, message in res['Value']['Failed'].items(): # Save the error message for the failed operations failed.setdefault(lfn, {})[catalogName] = message if master: # If this is the master catalog then we should not attempt the operation on other catalogs fileInfo.pop(lfn, None) for lfn, result in res['Value']['Successful'].items(): # Save the result return for each file for the successful operations successful.setdefault(lfn, {})[catalogName] = result # This recovers the states of the files that completely failed i.e. when S_ERROR is returned by a catalog for catalogName, errorMessage in failedCatalogs: for lfn in allLfns: failed.setdefault(lfn, {})[catalogName] = errorMessage resDict = {'Failed': failed, 'Successful': successful} return S_OK(resDict) def r_execute(self, *parms, **kws): """ Read method executor. """ successful = {} failed = {} for catalogName, oCatalog, _master in self.readCatalogs: # Skip if metadata related method on pure File Catalog if self.call in FileCatalog.ro_meta_methods and not catalogName in self.metaCatalogs: continue method = getattr(oCatalog, self.call) res = method(*parms, **kws) if res['OK']: if 'Successful' in res['Value']: for key, item in res['Value']['Successful'].items(): successful.setdefault(key, item) failed.pop(key, None) for key, item in res['Value']['Failed'].items(): if key not in successful: failed[key] = item else: return res if not successful and not failed: return S_ERROR("Failed to perform %s from any catalog" % self.call) return S_OK({'Failed': failed, 'Successful': successful}) ########################################################################################### # # Below is the method for obtaining the objects instantiated for a provided catalogue configuration # def addCatalog(self, catalogName, mode="Write", master=False): """ Add a new catalog with catalogName to the pool of catalogs in mode: "Read","Write" or "ReadWrite" """ result = self._generateCatalogObject(catalogName) if not result['OK']: return result oCatalog = result['Value'] if mode.lower().find("read") != -1: self.readCatalogs.append((catalogName, oCatalog, master)) if mode.lower().find("write") != -1: self.writeCatalogs.append((catalogName, oCatalog, master)) return S_OK() def removeCatalog(self, catalogName): """ Remove the specified catalog from the internal pool """ catalog_removed = False for i in range(len(self.readCatalogs)): catalog, _object, _master = self.readCatalogs[i] if catalog == catalogName: del self.readCatalogs[i] catalog_removed = True break for i in range(len(self.writeCatalogs)): catalog, _object, _master = self.writeCatalogs[i] if catalog == catalogName: del self.writeCatalogs[i] catalog_removed = True break if catalog_removed: return S_OK() else: return S_OK('Catalog does not exist') def _getSelectedCatalogs(self, desiredCatalogs): for catalogName in desiredCatalogs: res = self._getCatalogConfigDetails(catalogName) if not res['OK']: return res catalogConfig = res['Value'] res = self._generateCatalogObject(catalogName) if not res['OK']: return res oCatalog = res['Value'] self.readCatalogs.append((catalogName, oCatalog, True)) self.writeCatalogs.append((catalogName, oCatalog, True)) if catalogConfig.get('MetaCatalog') == 'True': self.metaCatalogs.append(catalogName) return S_OK() def _getCatalogs(self): # Get the eligible catalogs first # First, look in the Operations, if nothing defined look in /Resources for backward compatibility operationsFlag = False fileCatalogs = self.opHelper.getValue('/Services/Catalogs/CatalogList', []) if fileCatalogs: operationsFlag = True else: fileCatalogs = self.opHelper.getSections('/Services/Catalogs') result = self.opHelper.getSections('/Services/Catalogs') fileCatalogs = [] operationsFlag = False if result['OK']: fileCatalogs = result['Value'] operationsFlag = True else: res = gConfig.getSections(self.rootConfigPath, listOrdered=True) if not res['OK']: errStr = "FileCatalog._getCatalogs: Failed to get file catalog configuration." gLogger.error(errStr, res['Message']) return S_ERROR(errStr) fileCatalogs = res['Value'] # Get the catalogs now for catalogName in fileCatalogs: res = self._getCatalogConfigDetails(catalogName) if not res['OK']: return res catalogConfig = res['Value'] if operationsFlag: result = self.opHelper.getOptionsDict('/Services/Catalogs/%s' % catalogName) if not result['OK']: return result catalogConfig.update(result['Value']) if catalogConfig['Status'] == 'Active': res = self._generateCatalogObject(catalogName) if not res['OK']: return res oCatalog = res['Value'] master = catalogConfig['Master'] # If the catalog is read type if re.search('Read', catalogConfig['AccessType']): if master: self.readCatalogs.insert( 0, (catalogName, oCatalog, master)) else: self.readCatalogs.append( (catalogName, oCatalog, master)) # If the catalog is write type if re.search('Write', catalogConfig['AccessType']): if master: self.writeCatalogs.insert( 0, (catalogName, oCatalog, master)) else: self.writeCatalogs.append( (catalogName, oCatalog, master)) if catalogConfig.get('MetaCatalog') == 'True': self.metaCatalogs.append(catalogName) return S_OK() def _getCatalogConfigDetails(self, catalogName): # First obtain the options that are available catalogConfigPath = '%s/%s' % (self.rootConfigPath, catalogName) res = gConfig.getOptions(catalogConfigPath) if not res['OK']: errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options." gLogger.error(errStr, catalogName) return S_ERROR(errStr) catalogConfig = {} for option in res['Value']: configPath = '%s/%s' % (catalogConfigPath, option) optionValue = gConfig.getValue(configPath) catalogConfig[option] = optionValue # The 'Status' option should be defined (default = 'Active') if 'Status' not in catalogConfig: warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined." gLogger.warn(warnStr, catalogName) catalogConfig['Status'] = 'Active' # The 'AccessType' option must be defined if 'AccessType' not in catalogConfig: errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined." gLogger.error(errStr, catalogName) return S_ERROR(errStr) # Anything other than 'True' in the 'Master' option means it is not catalogConfig['Master'] = (catalogConfig.setdefault('Master', False) == 'True') return S_OK(catalogConfig) def _generateCatalogObject(self, catalogName): """ Create a file catalog object from its name and CS description """ useProxy = gConfig.getValue( '/LocalSite/Catalogs/%s/UseProxy' % catalogName, False) if not useProxy: useProxy = self.opHelper.getValue( '/Services/Catalogs/%s/UseProxy' % catalogName, False) return FileCatalogFactory().createCatalog(catalogName, useProxy)
class CombinedSoftwareInstallation(object): """ Combined means that it will try to install in the Shared area and in the LocalArea, depending on the user's rights """ def __init__(self, argumentsDict): """ Standard constructor Defines, from dictionary of job parameters passed, a set of members to hold e.g. the applications and the system config. Also determines the SharedArea and LocalArea. """ self.ops = Operations() self.job = argumentsDict.get('Job', {}) self.ce = argumentsDict.get('CE', {}) self.source = argumentsDict.get('Source', {}) apps = [] if 'SoftwarePackages' in self.job: if isinstance(self.job['SoftwarePackages'], basestring): apps = [self.job['SoftwarePackages']] elif isinstance(self.job['SoftwarePackages'], list): apps = self.job['SoftwarePackages'] self.jobConfig = '' if 'SystemConfig' in self.job: self.jobConfig = self.job['SystemConfig'] elif 'Platform' in self.job: self.jobConfig = self.job['Platform'] else: self.jobConfig = natOS.CMTSupportedConfig()[0] self.apps = [] for app in apps: DIRAC.gLogger.verbose('Requested Package %s' % app) app = tuple(app.split('.')) if len(app) > 2: tempapp = app app = [] app.append(tempapp[0]) app.append(".".join(tempapp[1:])) deps = resolveDeps(self.jobConfig, app[0], app[1]) for dep in deps: depapp = (dep['app'], dep['version']) self.apps.append(depapp) self.apps.append(app) self.ceConfigs = [] if 'CompatiblePlatforms' in self.ce: self.ceConfigs = self.ce['CompatiblePlatforms'] if isinstance(self.ceConfigs, basestring): self.ceConfigs = [self.ceConfigs] #else: ### Use always the list of compatible platform. self.ceConfigs = natOS.CMTSupportedConfig() self.sharedArea = getSharedAreaLocation() DIRAC.gLogger.info("SharedArea is %s" % self.sharedArea) self.localArea = getLocalAreaLocation() DIRAC.gLogger.info("LocalArea is %s" % self.localArea) def execute(self): """ Main method of the class executed by DIRAC jobAgent Executes the following: - look for the compatible platforms in the CS, see if one matches request - install the applications, calls :mod:`~ILCDIRAC.Core.Utilities.TARsoft` :return: S_OK(), S_ERROR() """ if not self.apps: # There is nothing to do return DIRAC.S_OK() if not self.jobConfig: DIRAC.gLogger.error('No architecture requested') return DIRAC.S_ERROR('No architecture requested') found_config = False DIRAC.gLogger.info( "Found CE Configs %s, compatible with system reqs %s" % (",".join(self.ceConfigs), self.jobConfig)) res = self.ops.getSections('/AvailableTarBalls') if not res['OK']: return res else: supported_systems = res['Value'] ###look in CS if specified platform has software available. Normally consistency check is done at submission time for ceConfig in self.ceConfigs: for supp_systems in supported_systems: if ceConfig == supp_systems: self.jobConfig = ceConfig found_config = True break if not found_config: if self.ceConfigs: # redundant check as this is done in the job agent, if locally running option might not be defined DIRAC.gLogger.error( 'Requested architecture not supported by CE') return DIRAC.S_ERROR( 'Requested architecture not supported by CE') else: DIRAC.gLogger.info( 'Assume locally running job, will install software in ') areas = [] ###Deal with shared/local area: first try to see if the Shared area exists and if not create it (try to). If it fails, fall back to local area if not self.sharedArea: if createSharedArea(): self.sharedArea = getSharedAreaLocation() if self.sharedArea: areas.append(self.sharedArea) DIRAC.gLogger.info( "Will attempt to install in shared area") else: areas.append(self.sharedArea) areas.append(self.localArea) for app in self.apps: res = checkCVMFS(self.jobConfig, app) if res['OK']: DIRAC.gLogger.notice( 'Software %s is available on CVMFS, skipping' % ", ".join(app)) continue ## First install the original package in any of the areas resInstall = installInAnyArea(areas, app, self.jobConfig) if not resInstall['OK']: return resInstall ## Then install the dependencies, which can be in a different area resDeps = installDependencies(app, self.jobConfig, areas) if not resDeps['OK']: DIRAC.gLogger.error("Failed to install dependencies: ", resDeps['Message']) return S_ERROR("Failed to install dependencies") if self.sharedArea: #List content listAreaDirectory(self.sharedArea) return DIRAC.S_OK()
class Limiter(object): # static variables shared between all instances of this class csDictCache = DictCache() condCache = DictCache() delayMem = {} def __init__(self, jobDB=None, opsHelper=None): """ Constructor """ self.__runningLimitSection = "JobScheduling/RunningLimit" self.__matchingDelaySection = "JobScheduling/MatchingDelay" if jobDB: self.jobDB = jobDB else: self.jobDB = JobDB() self.log = gLogger.getSubLogger("Limiter") if opsHelper: self.__opsHelper = opsHelper else: self.__opsHelper = Operations() def getNegativeCond(self): """ Get negative condition for ALL sites """ orCond = self.condCache.get("GLOBAL") if orCond: return orCond negCond = {} # Run Limit result = self.__opsHelper.getSections(self.__runningLimitSection) sites = [] if result['OK']: sites = result['Value'] for siteName in sites: result = self.__getRunningCondition(siteName) if not result['OK']: continue data = result['Value'] if data: negCond[siteName] = data # Delay limit result = self.__opsHelper.getSections(self.__matchingDelaySection) sites = [] if result['OK']: sites = result['Value'] for siteName in sites: result = self.__getDelayCondition(siteName) if not result['OK']: continue data = result['Value'] if not data: continue if siteName in negCond: negCond[siteName] = self.__mergeCond(negCond[siteName], data) else: negCond[siteName] = data orCond = [] for siteName in negCond: negCond[siteName]['Site'] = siteName orCond.append(negCond[siteName]) self.condCache.add("GLOBAL", 10, orCond) return orCond def getNegativeCondForSite(self, siteName): """ Generate a negative query based on the limits set on the site """ # Check if Limits are imposed onto the site negativeCond = {} if self.__opsHelper.getValue("JobScheduling/CheckJobLimits", True): result = self.__getRunningCondition(siteName) if result['OK']: negativeCond = result['Value'] self.log.verbose('Negative conditions for site', '%s after checking limits are: %s' % (siteName, str(negativeCond))) if self.__opsHelper.getValue("JobScheduling/CheckMatchingDelay", True): result = self.__getDelayCondition(siteName) if result['OK']: delayCond = result['Value'] self.log.verbose('Negative conditions for site', '%s after delay checking are: %s' % (siteName, str(delayCond))) negativeCond = self.__mergeCond(negativeCond, delayCond) if negativeCond: self.log.info('Negative conditions for site', '%s are: %s' % (siteName, str(negativeCond))) return negativeCond def __mergeCond(self, negCond, addCond): """ Merge two negative dicts """ # Merge both negative dicts for attr in addCond: if attr not in negCond: negCond[attr] = [] for value in addCond[attr]: if value not in negCond[attr]: negCond[attr].append(value) return negCond def __extractCSData(self, section): """ Extract limiting information from the CS in the form: { 'JobType' : { 'Merge' : 20, 'MCGen' : 1000 } } """ stuffDict = self.csDictCache.get(section) if stuffDict: return S_OK(stuffDict) result = self.__opsHelper.getSections(section) if not result['OK']: return result attribs = result['Value'] stuffDict = {} for attName in attribs: result = self.__opsHelper.getOptionsDict("%s/%s" % (section, attName)) if not result['OK']: return result attLimits = result['Value'] try: attLimits = dict([(k, int(attLimits[k])) for k in attLimits]) except Exception as excp: errMsg = "%s/%s has to contain numbers: %s" % (section, attName, str(excp)) self.log.error(errMsg) return S_ERROR(errMsg) stuffDict[attName] = attLimits self.csDictCache.add(section, 300, stuffDict) return S_OK(stuffDict) def __getRunningCondition(self, siteName): """ Get extra conditions allowing site throttling """ siteSection = "%s/%s" % (self.__runningLimitSection, siteName) result = self.__extractCSData(siteSection) if not result['OK']: return result limitsDict = result['Value'] # limitsDict is something like { 'JobType' : { 'Merge' : 20, 'MCGen' : 1000 } } if not limitsDict: return S_OK({}) # Check if the site exceeding the given limits negCond = {} for attName in limitsDict: if attName not in self.jobDB.jobAttributeNames: self.log.error("Attribute does not exist", "(%s). Check the job limits" % attName) continue cK = "Running:%s:%s" % (siteName, attName) data = self.condCache.get(cK) if not data: result = self.jobDB.getCounters( 'Jobs', [attName], { 'Site': siteName, 'Status': [ 'Running', 'Matched', 'Stalled']}) if not result['OK']: return result data = result['Value'] data = dict([(k[0][attName], k[1]) for k in data]) self.condCache.add(cK, 10, data) for attValue in limitsDict[attName]: limit = limitsDict[attName][attValue] running = data.get(attValue, 0) if running >= limit: self.log.verbose('Job Limit imposed', 'at %s on %s/%s=%d, %d jobs already deployed' % (siteName, attName, attValue, limit, running)) if attName not in negCond: negCond[attName] = [] negCond[attName].append(attValue) # negCond is something like : {'JobType': ['Merge']} return S_OK(negCond) def updateDelayCounters(self, siteName, jid): # Get the info from the CS siteSection = "%s/%s" % (self.__matchingDelaySection, siteName) result = self.__extractCSData(siteSection) if not result['OK']: return result delayDict = result['Value'] # limitsDict is something like { 'JobType' : { 'Merge' : 20, 'MCGen' : 1000 } } if not delayDict: return S_OK() attNames = [] for attName in delayDict: if attName not in self.jobDB.jobAttributeNames: self.log.error("Attribute does not exist in the JobDB. Please fix it!", "(%s)" % attName) else: attNames.append(attName) result = self.jobDB.getJobAttributes(jid, attNames) if not result['OK']: self.log.error("Error while retrieving attributes", "coming from %s: %s" % (siteSection, result['Message'])) return result atts = result['Value'] # Create the DictCache if not there if siteName not in self.delayMem: self.delayMem[siteName] = DictCache() # Update the counters delayCounter = self.delayMem[siteName] for attName in atts: attValue = atts[attName] if attValue in delayDict[attName]: delayTime = delayDict[attName][attValue] self.log.notice("Adding delay for %s/%s=%s of %s secs" % (siteName, attName, attValue, delayTime)) delayCounter.add((attName, attValue), delayTime) return S_OK() def __getDelayCondition(self, siteName): """ Get extra conditions allowing matching delay """ if siteName not in self.delayMem: return S_OK({}) lastRun = self.delayMem[siteName].getKeys() negCond = {} for attName, attValue in lastRun: if attName not in negCond: negCond[attName] = [] negCond[attName].append(attValue) return S_OK(negCond)
class OverlayInput(LCUtilityApplication): """ Helper call to define Overlay processor/driver inputs. Example: >>> over = OverlayInput() >>> over.setBXOverlay(300) >>> over.setGGToHadInt(3.2) >>> over.setNumberOfSignalEventsPerJob(10) >>> over.setBackgroundType("gghad") """ def __init__(self, paramdict=None): self._ops = Operations() self.bxToOverlay = None self.numberOfGGToHadronInteractions = 0 self.numberOfSignalEventsPerJob = 0 self.backgroundEventType = '' self.prodID = 0 self.machine = 'clic_cdr' self.detectorModel = '' self.useEnergyForFileLookup = True super(OverlayInput, self).__init__(paramdict) self.version = '1' self._modulename = "OverlayInput" self.appname = self._modulename self._moduledescription = 'Helper call to define Overlay processor/driver inputs' self.accountInProduction = False self._paramsToExclude.append('_ops') self.pathToOverlayFiles = '' self.processorName = '' def setMachine(self, machine): """ Define the machine to use, clic_cdr or ilc_dbd """ self._checkArgs({'machine': types.StringTypes}) self.machine = machine def setProdID(self, pid): """ Define the prodID to use as input, experts only """ self._checkArgs({'pid': types.IntType}) self.prodID = pid return S_OK() def setUseEnergyForFileLookup(self, useEnergyForFileLookup): """ Sets the flag to use the energy meta data in the search of the background files. Disable the energy when you want to use files created for a different energy than the signal events :param bool useEnergyForFileLookup: Use the Energy in the metadata search or not """ self._checkArgs({'useEnergyForFileLookup': types.BooleanType}) self.useEnergyForFileLookup = useEnergyForFileLookup return S_OK() def setOverlayBXPerSigEvt(self, bxoverlay): """ Define number bunch crossings to overlay for each signal event. This is used to determine the number of required overlay events. It does not modify any of the actual application parameters using the overly input. Alias for :func:`setBXOverlay` :param int bxoverlay: Bunch crossings to overlay. """ self._checkArgs({'bxoverlay': types.IntType}) self.bxToOverlay = bxoverlay return S_OK() def setBXOverlay(self, bxoverlay): """ Define number bunch crossings to overlay for each signal event. This is used to determine the number of required overlay events. It does not modify any of the actual application parameters using the overly input. :param int bxoverlay: Bunch crossings to overlay. """ return self.setOverlayBXPerSigEvt(bxoverlay) def setOverlayEvtsPerBX(self, ggtohadint): """ Define the number of overlay events per bunch crossing. This is used to determine the number of required overlay events. It does not modify any of the actual application parameters using the overly input. :param float ggtohadint: optional number of overlay events interactions per bunch crossing """ self._checkArgs({'ggtohadint': types.FloatType}) self.numberOfGGToHadronInteractions = ggtohadint return S_OK() def setGGToHadInt(self, ggtohadint): """ Define the number of overlay events per bunch crossing. This is used to determine the number of required overlay events. It does not modify any of the actual application parameters using the overly input. Alias for :func:`setOverlayEvtsPerBX` :param float ggtohadint: optional number of overlay events interactions per bunch crossing """ return self.setOverlayEvtsPerBX(ggtohadint) def setNbSigEvtsPerJob(self, nbsigevtsperjob): """ Set the number of signal events per job. This is used to determine the number of required overlay events. It does not modify any of the actual application parameters using the overly input. :param int nbsigevtsperjob: Number of signal events per job """ self._checkArgs({'nbsigevtsperjob': types.IntType}) self.numberOfSignalEventsPerJob = nbsigevtsperjob return S_OK() def setDetectorModel(self, detectormodel): """ Set the detector type for the background files. Files are defined in the ConfigurationSystem: Operations/Overlay/<Accelerator>/<energy>/<Detector> :param string detectormodel: Detector type. Must be 'CLIC_ILD_CDR' or 'CLIC_SID_CDR' or 'sidloi3' or 'ILD_o1_v05' """ self._checkArgs({'detectormodel': types.StringTypes}) self.detectorModel = detectormodel return S_OK() def setPathToFiles(self, path): """ Sets the path to where the overlay files are located. Setting this option will ignore all other settings! :param string path: LFN path to the folder containing the overlay files """ self._checkArgs({'path': types.StringTypes}) self.pathToOverlayFiles = path return S_OK() def setBkgEvtType(self, backgroundEventType): """ Define the background type. .. deprecated:: 23r0 Use :func:`setBackgroundType` instead :param string backgroundEventType: Background type. """ self._checkArgs({'backgroundEventType': types.StringTypes}) self.backgroundEventType = backgroundEventType return S_OK() def setBackgroundType(self, backgroundType): """Define the background type :param string backgroundType: Background type. """ return self.setBkgEvtType(backgroundType) def setProcessorName(self, processorName): """Set the processorName to set the input files for. Necessary if multiple invocations of the overlay processor happen in marlin for example. Different processors must use different background types :param string processorName: Name of the Processor these input files are for """ self._checkArgs({'processorName': types.StringTypes}) self.processorName = processorName return S_OK() def setNumberOfSignalEventsPerJob(self, numberSignalEvents): """Alternative to :func:`setNbSigEvtsPerJob` Number used to determine the number of background files needed. :param int numberSignalEvents: Number of signal events per job """ return self.setNbSigEvtsPerJob(numberSignalEvents) # def setProdIDToUse(self,prodid): # """ Optional parameter: Define the production ID to use as input # # :param int prodid: Production ID # """ # self._checkArgs({"prodid" : types.IntType}) # self.prodid = prodid # return S_OK() def _applicationModule(self): m1 = self._createModuleDefinition() m1.addParameter( Parameter("BXOverlay", 0, "float", "", "", False, False, "Bunch crossings to overlay")) m1.addParameter( Parameter( "ggtohadint", 0, "float", "", "", False, False, "Optional number of gamma gamma -> hadrons interactions per bunch crossing, default is 3.2" )) m1.addParameter( Parameter("NbSigEvtsPerJob", 0, "int", "", "", False, False, "Number of signal events per job")) m1.addParameter( Parameter("prodid", 0, "int", "", "", False, False, "ProdID to use")) m1.addParameter( Parameter("BkgEvtType", "", "string", "", "", False, False, "Background type.")) m1.addParameter( Parameter("detectormodel", "", "string", "", "", False, False, "Detector type.")) m1.addParameter( Parameter("machine", "", "string", "", "", False, False, "machine: clic_cdr or ilc_dbd")) m1.addParameter( Parameter("useEnergyForFileLookup", True, "bool", "", "", False, False, "useEnergy to look for background files: True or False")) m1.addParameter( Parameter("pathToOverlayFiles", "", "string", "", "", False, False, "use overlay files from this path")) m1.addParameter( Parameter("processorName", "", "string", "", "", False, False, "Processor Name")) m1.addParameter( Parameter("debug", False, "bool", "", "", False, False, "debug mode")) return m1 def _applicationModuleValues(self, moduleinstance): moduleinstance.setValue("BXOverlay", self.bxToOverlay) moduleinstance.setValue('ggtohadint', self.numberOfGGToHadronInteractions) moduleinstance.setValue('NbSigEvtsPerJob', self.numberOfSignalEventsPerJob) moduleinstance.setValue('prodid', self.prodID) moduleinstance.setValue('BkgEvtType', self.backgroundEventType) moduleinstance.setValue('detectormodel', self.detectorModel) moduleinstance.setValue('debug', self.debug) moduleinstance.setValue('machine', self.machine) moduleinstance.setValue('useEnergyForFileLookup', self.useEnergyForFileLookup) moduleinstance.setValue('pathToOverlayFiles', self.pathToOverlayFiles) moduleinstance.setValue('processorName', self.processorName) def _userjobmodules(self, stepdefinition): res1 = self._setApplicationModuleAndParameters(stepdefinition) if not res1["OK"]: return S_ERROR('userjobmodules failed') return S_OK() def _prodjobmodules(self, stepdefinition): res1 = self._setApplicationModuleAndParameters(stepdefinition) if not res1["OK"]: return S_ERROR('prodjobmodules failed') return S_OK() def _addParametersToStep(self, stepdefinition): res = self._addBaseParameters(stepdefinition) if not res["OK"]: return S_ERROR("Failed to set base parameters") return S_OK() def _checkConsistency(self, job=None): """ Checks that all needed parameters are set """ if self.pathToOverlayFiles: res = FileCatalogClient().findFilesByMetadata( {}, self.pathToOverlayFiles) if not res['OK']: return res self._log.notice("Found %i files in path %s" % (len(res['Value']), self.pathToOverlayFiles)) if len(res['Value']) == 0: return S_ERROR( "OverlayInput: PathToFiles is specified, but there are no files in that path" ) if not self.bxToOverlay: return S_ERROR("Number of overlay bunch crossings not defined") if not self.numberOfGGToHadronInteractions: return S_ERROR( "Number of background events per bunch crossing is not defined" ) if not self.backgroundEventType: return S_ERROR( "Background event type is not defined: Chose one gghad, aa_lowpt, ..." ) if self._jobtype == 'User': if not self.numberOfSignalEventsPerJob: return S_ERROR("Number of signal event per job is not defined") else: self.prodparameters['detectorModel'] = self.detectorModel self.prodparameters['BXOverlay'] = self.bxToOverlay self.prodparameters[ 'GGtoHadInt'] = self.numberOfGGToHadronInteractions return S_OK() def _checkFinalConsistency(self): """ Final check of consistency: the overlay files for the specifed energy must exist """ if self.pathToOverlayFiles: return S_OK() # can ignore other parameter if not self.energy: return S_ERROR("Energy MUST be specified for the overlay") res = self._ops.getSections('/Overlay') if not res['OK']: return S_ERROR( "Could not resolve the CS path to the overlay specifications") sections = res['Value'] if self.machine not in sections: return S_ERROR( "Machine %s does not have overlay data, use any of %s" % (self.machine, sections)) energytouse = energyWithLowerCaseUnit(self.energy) res = self._ops.getSections("/Overlay/%s" % self.machine) if energytouse not in res['Value']: return S_ERROR("No overlay files corresponding to %s" % energytouse) res = self._ops.getSections("/Overlay/%s/%s" % (self.machine, energytouse)) if not res['OK']: return S_ERROR("Could not find the detector models") if self.detectorModel not in res['Value']: return S_ERROR( "Detector model specified has no overlay data with that energy and machine" ) res = allowedBkg(self.backgroundEventType, energytouse, detectormodel=self.detectorModel, machine=self.machine) if not res['OK']: return res if res['Value'] < 0: return S_ERROR("No proper production ID found") return S_OK()
class OverlayDB ( DB ): def __init__( self, maxQueueSize = 10 ): """ """ self.ops = Operations() self.dbname = 'OverlayDB' self.logger = gLogger.getSubLogger('OverlayDB') DB.__init__( self, self.dbname, 'Overlay/OverlayDB', maxQueueSize ) self._createTables( { "OverlayData" : { 'Fields' : { 'Site' : "VARCHAR(256) UNIQUE NOT NULL", 'NumberOfJobs' : "INTEGER DEFAULT 0" }, 'PrimaryKey' : 'Site', 'Indexes': {'Index':['Site']} } } ) limits = self.ops.getValue("/Overlay/MaxConcurrentRunning", 200) self.limits = {} self.limits["default"] = limits res = self.ops.getSections("/Overlay/Sites/") sites = [] if res['OK']: sites = res['Value'] for tempsite in sites: res = self.ops.getValue("/Overlay/Sites/%s/MaxConcurrentRunning" % tempsite, 200) self.limits[tempsite] = res self.logger.info("Using the following restrictions : %s" % self.limits) ##################################################################### # Private methods def __getConnection( self, connection ): if connection: return connection res = self._getConnection() if res['OK']: return res['Value'] gLogger.warn( "Failed to get MySQL connection", res['Message'] ) return connection def _checkSite(self, site, connection = False ): """ Check the number of jos running at a given site. """ connection = self.__getConnection( connection ) req = "SELECT NumberOfJobs FROM OverlayData WHERE Site='%s';" % (site) res = self._query( req, connection ) if not res['OK']: return S_ERROR("Could not get site") if len(res['Value']): return res else: return S_ERROR("Could not find any site %s"%(site)) def _addSite(self, site, connection = False ): """ Add a new site to the DB """ connection = self.__getConnection( connection ) req = "INSERT INTO OverlayData (Site,NumberOfJobs) VALUES ('%s',1);" % site res = self._update( req, connection ) if not res['OK']: return res return res def _limitForSite(self, site): """ Get the current limit of jobs for a given site. """ if site in self.limits.keys(): return self.limits[site] return self.limits['default'] def _addNewJob(self, site, nbjobs, connection = False ): """ Add a new running job in the DB """ connection = self.__getConnection( connection ) nbjobs += 1 req = "UPDATE OverlayData SET NumberOfJobs=%s WHERE Site='%s';" % (nbjobs, site) self._update( req, connection ) return S_OK() ### Methods to fix the site def getSites(self, connection = False): """ Return the list of sites known to the service """ connection = self.__getConnection( connection ) req = 'SELECT Site From OverlayData;' res = self._query( req, connection ) if not res['OK']: return S_ERROR("Could not get sites") sites = [] for row in res['Value']: sites.append(row[0]) return S_OK(sites) def setJobsAtSites(self, sitedict, connection = False): """ As name suggests: set the number of jobs running at the site. """ connection = self.__getConnection( connection ) for site, nbjobs in sitedict.items(): req = "UPDATE OverlayData SET NumberOfJobs=%s WHERE Site='%s';" % (int(nbjobs), site) res = self._update( req, connection ) if not res['OK']: return S_ERROR("Could not set nb of jobs at site %s" % site) return S_OK() ### Useful methods for the users def getJobsAtSite(self, site, connection = False ): """ Get the number of jobs currently run """ connection = self.__getConnection( connection ) nbjobs = 0 res = self._checkSite(site, connection) if not res['OK']: return S_OK(nbjobs) nbjobs = res['Value'][0][0] return S_OK(nbjobs) ### Important methods def canRun(self, site, connection = False ): """ Can the job run at that site? """ connection = self.__getConnection( connection ) res = self._checkSite(site, connection) nbjobs = 0 if not res['OK']: self._addSite(site, connection) nbjobs = 1 else: nbjobs = res['Value'][0][0] if nbjobs < self._limitForSite(site): res = self._addNewJob(site, nbjobs, connection) if not res['OK']: return res return S_OK(True) else: return S_OK(False) def jobDone(self, site, connection = False ): """ Remove a job from the DB, should not remove a job from the DB if the Site does not exist, but this should never happen """ connection = self.__getConnection( connection ) res = self._checkSite(site, connection) if not res['OK']: return res nbjobs = res['Value'][0][0] if nbjobs == 1: return S_OK() nbjobs -= 1 req = "UPDATE OverlayData SET NumberOfJobs=%s WHERE Site='%s';" % (nbjobs, site) res = self._update( req, connection ) if not res['OK']: return res return S_OK()
class FileCatalog: ro_methods = [ 'exists', 'isLink', 'readLink', 'isFile', 'getFileMetadata', 'getReplicas', 'getReplicaStatus', 'getFileSize', 'isDirectory', 'getDirectoryReplicas', 'listDirectory', 'getDirectoryMetadata', 'getDirectorySize', 'getDirectoryContents', 'resolveDataset', 'getPathPermissions', 'getLFNForPFN', 'getUsers', 'getGroups', 'getFileUserMetadata' ] write_methods = [ 'createLink', 'removeLink', 'addFile', 'setFileStatus', 'addReplica', 'removeReplica', 'removeFile', 'setReplicaStatus', 'setReplicaHost', 'createDirectory', 'setDirectoryStatus', 'removeDirectory', 'removeDataset', 'removeFileFromDataset', 'createDataset' ] def __init__(self, catalogs=[], vo=None): """ Default constructor """ self.valid = True self.timeout = 180 self.readCatalogs = [] self.writeCatalogs = [] self.vo = vo if not vo: result = getVOfromProxyGroup() if not result['OK']: return result self.vo = result['Value'] self.opHelper = Operations(vo=self.vo) self.reHelper = Resources(vo=self.vo) if type(catalogs) in types.StringTypes: catalogs = [catalogs] if catalogs: res = self._getSelectedCatalogs(catalogs) else: res = self._getCatalogs() if not res['OK']: self.valid = False elif (len(self.readCatalogs) == 0) and (len(self.writeCatalogs) == 0): self.valid = False def isOK(self): return self.valid def getReadCatalogs(self): return self.readCatalogs def getWriteCatalogs(self): return self.writeCatalogs def __getattr__(self, name): self.call = name if name in FileCatalog.write_methods: return self.w_execute elif name in FileCatalog.ro_methods: return self.r_execute else: raise AttributeError def w_execute(self, *parms, **kws): """ Write method executor. """ successful = {} failed = {} failedCatalogs = [] fileInfo = parms[0] res = checkArgumentFormat(fileInfo) if not res['OK']: return res fileInfo = res['Value'] allLfns = fileInfo.keys() for catalogName, oCatalog, master in self.writeCatalogs: method = getattr(oCatalog, self.call) res = method(fileInfo, **kws) if not res['OK']: if master: # If this is the master catalog and it fails we dont want to continue with the other catalogs gLogger.error( "FileCatalog.w_execute: Failed to execute %s on master catalog %s." % (self.call, catalogName), res['Message']) return res else: # Otherwise we keep the failed catalogs so we can update their state later failedCatalogs.append((catalogName, res['Message'])) else: for lfn, message in res['Value']['Failed'].items(): # Save the error message for the failed operations if not failed.has_key(lfn): failed[lfn] = {} failed[lfn][catalogName] = message if master: # If this is the master catalog then we should not attempt the operation on other catalogs fileInfo.pop(lfn) for lfn, result in res['Value']['Successful'].items(): # Save the result return for each file for the successful operations if not successful.has_key(lfn): successful[lfn] = {} successful[lfn][catalogName] = result # This recovers the states of the files that completely failed i.e. when S_ERROR is returned by a catalog for catalogName, errorMessage in failedCatalogs: for lfn in allLfns: if not failed.has_key(lfn): failed[lfn] = {} failed[lfn][catalogName] = errorMessage resDict = {'Failed': failed, 'Successful': successful} return S_OK(resDict) def r_execute(self, *parms, **kws): """ Read method executor. """ successful = {} failed = {} for _catalogName, oCatalog, _master in self.readCatalogs: method = getattr(oCatalog, self.call) res = method(*parms, **kws) if res['OK']: if 'Successful' in res['Value']: for key, item in res['Value']['Successful'].items(): if not successful.has_key(key): successful[key] = item if failed.has_key(key): failed.pop(key) for key, item in res['Value']['Failed'].items(): if not successful.has_key(key): failed[key] = item if len(failed) == 0: resDict = {'Failed': failed, 'Successful': successful} return S_OK(resDict) else: return res if (len(successful) == 0) and (len(failed) == 0): return S_ERROR("Failed to perform %s from any catalog" % self.call) resDict = {'Failed': failed, 'Successful': successful} return S_OK(resDict) ########################################################################################### # # Below is the method for obtaining the objects instantiated for a provided catalogue configuration # def addCatalog(self, catalogName, mode="Write", master=False): """ Add a new catalog with catalogName to the pool of catalogs in mode: "Read","Write" or "ReadWrite" """ result = self._generateCatalogObject(catalogName) if not result['OK']: return result oCatalog = result['Value'] if mode.lower().find("read") != -1: self.readCatalogs.append((catalogName, oCatalog, master)) if mode.lower().find("write") != -1: self.writeCatalogs.append((catalogName, oCatalog, master)) return S_OK() def removeCatalog(self, catalogName): """ Remove the specified catalog from the internal pool """ catalog_removed = False for i in range(len(self.readCatalogs)): catalog, _object, _master = self.readCatalogs[i] if catalog == catalogName: del self.readCatalogs[i] catalog_removed = True break for i in range(len(self.writeCatalogs)): catalog, _object, _master = self.writeCatalogs[i] if catalog == catalogName: del self.writeCatalogs[i] catalog_removed = True break if catalog_removed: return S_OK() else: return S_OK('Catalog does not exist') def _getSelectedCatalogs(self, desiredCatalogs): for catalogName in desiredCatalogs: res = self._generateCatalogObject(catalogName) if not res['OK']: return res oCatalog = res['Value'] self.readCatalogs.append((catalogName, oCatalog, True)) self.writeCatalogs.append((catalogName, oCatalog, True)) return S_OK() def _getCatalogs(self): # Get the eligible catalogs first # First, look in the Operations, if nothing defined look in /Resources result = self.opHelper.getSections('/Services/Catalogs') fileCatalogs = [] operationsFlag = False optCatalogDict = {} if result['OK']: fcs = result['Value'] for fc in fcs: fName = self.opHelper.getValue( '/Services/Catalogs/%s/CatalogName' % fc, fc) fileCatalogs.append(fName) optCatalogDict[fName] = fc operationsFlag = True else: res = self.reHelper.getEligibleResources('Catalog') if not res['OK']: errStr = "FileCatalog._getCatalogs: Failed to get file catalog configuration." gLogger.error(errStr, res['Message']) return S_ERROR(errStr) fileCatalogs = res['Value'] # Get the catalogs now for catalogName in fileCatalogs: res = self._getCatalogConfigDetails(catalogName) if not res['OK']: return res catalogConfig = res['Value'] if operationsFlag: result = self.opHelper.getOptionsDict( '/Services/Catalogs/%s' % optCatalogDict[catalogName]) if not result['OK']: return result catalogConfig.update(result['Value']) if catalogConfig['Status'] == 'Active': res = self._generateCatalogObject(catalogName) if not res['OK']: return res oCatalog = res['Value'] master = catalogConfig['Master'] # If the catalog is read type if re.search('Read', catalogConfig['AccessType']): if master: self.readCatalogs.insert( 0, (catalogName, oCatalog, master)) else: self.readCatalogs.append( (catalogName, oCatalog, master)) # If the catalog is write type if re.search('Write', catalogConfig['AccessType']): if master: self.writeCatalogs.insert( 0, (catalogName, oCatalog, master)) else: self.writeCatalogs.append( (catalogName, oCatalog, master)) return S_OK() def _getCatalogConfigDetails(self, catalogName): # First obtain the options that are available result = self.reHelper.getCatalogOptionsDict(catalogName) if not result['OK']: errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options" gLogger.error(errStr, catalogName) return S_ERROR(errStr) catalogConfig = result['Value'] # The 'Status' option should be defined (default = 'Active') if not catalogConfig.has_key('Status'): warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined" gLogger.warn(warnStr, catalogName) catalogConfig['Status'] = 'Active' # The 'AccessType' option must be defined if not catalogConfig.has_key('AccessType'): errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined" gLogger.error(errStr, catalogName) return S_ERROR(errStr) # Anything other than 'True' in the 'Master' option means it is not if not catalogConfig.has_key('Master'): catalogConfig['Master'] = False elif catalogConfig['Master'] == 'True': catalogConfig['Master'] = True else: catalogConfig['Master'] = False return S_OK(catalogConfig) def _generateCatalogObject(self, catalogName): """ Create a file catalog object from its name and CS description """ useProxy = gConfig.getValue( '/LocalSite/Catalogs/%s/UseProxy' % catalogName, False) if not useProxy: useProxy = self.opHelper.getValue( '/Services/Catalogs/%s/UseProxy' % catalogName, False) return FileCatalogFactory().createCatalog(catalogName, useProxy)
def addShifter(self, shifters=None): """ Adds or modify one or more shifters. Also, adds the shifter section in case this is not present. Shifter identities are used in several places, mostly for running agents :param dict shifters: has to be in the form {'ShifterRole':{'User':'******', 'Group':'aDIRACGroup'}} :return: S_OK/S_ERROR """ def getOpsSection(): """ Where is the shifters section? """ vo = CSGlobals.getVO() setup = CSGlobals.getSetup() if vo: res = gConfig.getSections('/Operations/%s/%s/Shifter' % (vo, setup)) if res['OK']: return S_OK('/Operations/%s/%s/Shifter' % (vo, setup)) res = gConfig.getSections('/Operations/%s/Defaults/Shifter' % vo) if res['OK']: return S_OK('/Operations/%s/Defaults/Shifter' % vo) else: res = gConfig.getSections('/Operations/%s/Shifter' % setup) if res['OK']: return S_OK('/Operations/%s/Shifter' % setup) res = gConfig.getSections('/Operations/Defaults/Shifter') if res['OK']: return S_OK('/Operations/Defaults/Shifter') return S_ERROR("No shifter section") if shifters is None: shifters = {} if not self.__initialized['OK']: return self.__initialized # get current shifters opsH = Operations() currentShifterRoles = opsH.getSections('Shifter') if not currentShifterRoles['OK']: # we assume the shifter section is not present currentShifterRoles = [] else: currentShifterRoles = currentShifterRoles['Value'] currentShiftersDict = {} for currentShifterRole in currentShifterRoles: currentShifter = opsH.getOptionsDict('Shifter/%s' % currentShifterRole) if not currentShifter['OK']: return currentShifter currentShifter = currentShifter['Value'] currentShiftersDict[currentShifterRole] = currentShifter # Removing from shifters what does not need to be changed for sRole in shifters.keys(): # note the pop below if sRole in currentShiftersDict: if currentShiftersDict[sRole] == shifters[sRole]: shifters.pop(sRole) # get shifters section to modify section = getOpsSection() # Is this section present? if not section['OK']: if section['Message'] == "No shifter section": gLogger.warn(section['Message']) gLogger.info("Adding shifter section") vo = CSGlobals.getVO() if vo: section = '/Operations/%s/Defaults/Shifter' % vo else: section = '/Operations/Defaults/Shifter' res = self.__csMod.createSection(section) if not res: gLogger.error("Section %s not created" % section) return S_ERROR("Section %s not created" % section) else: gLogger.error(section['Message']) return section else: section = section['Value'] # add or modify shifters for shifter in shifters: self.__csMod.removeSection(section + '/' + shifter) self.__csMod.createSection(section + '/' + shifter) self.__csMod.createSection(section + '/' + shifter + '/' + 'User') self.__csMod.createSection(section + '/' + shifter + '/' + 'Group') self.__csMod.setOptionValue(section + '/' + shifter + '/' + 'User', shifters[shifter]['User']) self.__csMod.setOptionValue( section + '/' + shifter + '/' + 'Group', shifters[shifter]['Group']) self.csModified = True return S_OK(True)
class FileCatalog: ro_methods = ['exists', 'isLink', 'readLink', 'isFile', 'getFileMetadata', 'getReplicas', 'getReplicaStatus', 'getFileSize', 'isDirectory', 'getDirectoryReplicas', 'listDirectory', 'getDirectoryMetadata', 'getDirectorySize', 'getDirectoryContents', 'resolveDataset', 'getPathPermissions', 'getLFNForPFN', 'getUsers', 'getGroups', 'getFileUserMetadata'] write_methods = ['createLink', 'removeLink', 'addFile', 'setFileStatus', 'addReplica', 'removeReplica', 'removeFile', 'setReplicaStatus', 'setReplicaHost', 'createDirectory', 'setDirectoryStatus', 'removeDirectory', 'removeDataset', 'removeFileFromDataset', 'createDataset'] def __init__( self, catalogs = [], vo = None ): """ Default constructor """ self.valid = True self.timeout = 180 self.readCatalogs = [] self.writeCatalogs = [] self.rootConfigPath = '/Resources/FileCatalogs' self.vo = vo if vo else getVOfromProxyGroup().get( 'Value', None ) self.opHelper = Operations( vo = self.vo ) if type( catalogs ) in types.StringTypes: catalogs = [catalogs] if catalogs: res = self._getSelectedCatalogs( catalogs ) else: res = self._getCatalogs() if not res['OK']: self.valid = False elif ( len( self.readCatalogs ) == 0 ) and ( len( self.writeCatalogs ) == 0 ): self.valid = False def isOK( self ): return self.valid def getReadCatalogs( self ): return self.readCatalogs def getWriteCatalogs( self ): return self.writeCatalogs def __getattr__( self, name ): self.call = name if name in FileCatalog.write_methods: return self.w_execute elif name in FileCatalog.ro_methods: return self.r_execute else: raise AttributeError def w_execute( self, *parms, **kws ): """ Write method executor. """ successful = {} failed = {} failedCatalogs = [] fileInfo = parms[0] res = checkArgumentFormat( fileInfo ) if not res['OK']: return res fileInfo = res['Value'] allLfns = fileInfo.keys() for catalogName, oCatalog, master in self.writeCatalogs: method = getattr( oCatalog, self.call ) res = method( fileInfo, **kws ) if not res['OK']: if master: # If this is the master catalog and it fails we dont want to continue with the other catalogs gLogger.error( "FileCatalog.w_execute: Failed to execute %s on master catalog %s." % ( self.call, catalogName ), res['Message'] ) return res else: # Otherwise we keep the failed catalogs so we can update their state later failedCatalogs.append( ( catalogName, res['Message'] ) ) else: for lfn, message in res['Value']['Failed'].items(): # Save the error message for the failed operations failed.setdefault( lfn, {} )[catalogName] = message if master: # If this is the master catalog then we should not attempt the operation on other catalogs fileInfo.pop( lfn, None ) for lfn, result in res['Value']['Successful'].items(): # Save the result return for each file for the successful operations successful.setdefault( lfn, {} )[catalogName] = result # This recovers the states of the files that completely failed i.e. when S_ERROR is returned by a catalog for catalogName, errorMessage in failedCatalogs: for lfn in allLfns: failed.setdefault( lfn, {} )[catalogName] = errorMessage resDict = {'Failed':failed, 'Successful':successful} return S_OK( resDict ) def r_execute( self, *parms, **kws ): """ Read method executor. """ successful = {} failed = {} for _catalogName, oCatalog, _master in self.readCatalogs: method = getattr( oCatalog, self.call ) res = method( *parms, **kws ) if res['OK']: if 'Successful' in res['Value']: for key, item in res['Value']['Successful'].items(): successful.setdefault( key, item ) failed.pop( key, None ) for key, item in res['Value']['Failed'].items(): if key not in successful: failed[key] = item else: return res if not successful and not failed: return S_ERROR( "Failed to perform %s from any catalog" % self.call ) return S_OK( {'Failed':failed, 'Successful':successful} ) ########################################################################################### # # Below is the method for obtaining the objects instantiated for a provided catalogue configuration # def addCatalog( self, catalogName, mode = "Write", master = False ): """ Add a new catalog with catalogName to the pool of catalogs in mode: "Read","Write" or "ReadWrite" """ result = self._generateCatalogObject( catalogName ) if not result['OK']: return result oCatalog = result['Value'] if mode.lower().find( "read" ) != -1: self.readCatalogs.append( ( catalogName, oCatalog, master ) ) if mode.lower().find( "write" ) != -1: self.writeCatalogs.append( ( catalogName, oCatalog, master ) ) return S_OK() def removeCatalog( self, catalogName ): """ Remove the specified catalog from the internal pool """ catalog_removed = False for i in range( len( self.readCatalogs ) ): catalog, _object, _master = self.readCatalogs[i] if catalog == catalogName: del self.readCatalogs[i] catalog_removed = True break for i in range( len( self.writeCatalogs ) ): catalog, _object, _master = self.writeCatalogs[i] if catalog == catalogName: del self.writeCatalogs[i] catalog_removed = True break if catalog_removed: return S_OK() else: return S_OK( 'Catalog does not exist' ) def _getSelectedCatalogs( self, desiredCatalogs ): for catalogName in desiredCatalogs: res = self._generateCatalogObject( catalogName ) if not res['OK']: return res oCatalog = res['Value'] self.readCatalogs.append( ( catalogName, oCatalog, True ) ) self.writeCatalogs.append( ( catalogName, oCatalog, True ) ) return S_OK() def _getCatalogs( self ): # Get the eligible catalogs first # First, look in the Operations, if nothing defined look in /Resources for backward compatibility result = self.opHelper.getSections( '/Services/Catalogs' ) fileCatalogs = [] operationsFlag = False if result['OK']: fileCatalogs = result['Value'] operationsFlag = True else: res = gConfig.getSections( self.rootConfigPath, listOrdered = True ) if not res['OK']: errStr = "FileCatalog._getCatalogs: Failed to get file catalog configuration." gLogger.error( errStr, res['Message'] ) return S_ERROR( errStr ) fileCatalogs = res['Value'] # Get the catalogs now for catalogName in fileCatalogs: res = self._getCatalogConfigDetails( catalogName ) if not res['OK']: return res catalogConfig = res['Value'] if operationsFlag: result = self.opHelper.getOptionsDict( '/Services/Catalogs/%s' % catalogName ) if not result['OK']: return result catalogConfig.update( result['Value'] ) if catalogConfig['Status'] == 'Active': res = self._generateCatalogObject( catalogName ) if not res['OK']: return res oCatalog = res['Value'] master = catalogConfig['Master'] # If the catalog is read type if re.search( 'Read', catalogConfig['AccessType'] ): if master: self.readCatalogs.insert( 0, ( catalogName, oCatalog, master ) ) else: self.readCatalogs.append( ( catalogName, oCatalog, master ) ) # If the catalog is write type if re.search( 'Write', catalogConfig['AccessType'] ): if master: self.writeCatalogs.insert( 0, ( catalogName, oCatalog, master ) ) else: self.writeCatalogs.append( ( catalogName, oCatalog, master ) ) return S_OK() def _getCatalogConfigDetails( self, catalogName ): # First obtain the options that are available catalogConfigPath = '%s/%s' % ( self.rootConfigPath, catalogName ) res = gConfig.getOptions( catalogConfigPath ) if not res['OK']: errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options." gLogger.error( errStr, catalogName ) return S_ERROR( errStr ) catalogConfig = {} for option in res['Value']: configPath = '%s/%s' % ( catalogConfigPath, option ) optionValue = gConfig.getValue( configPath ) catalogConfig[option] = optionValue # The 'Status' option should be defined (default = 'Active') if 'Status' not in catalogConfig: warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined." gLogger.warn( warnStr, catalogName ) catalogConfig['Status'] = 'Active' # The 'AccessType' option must be defined if 'AccessType' not in catalogConfig: errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined." gLogger.error( errStr, catalogName ) return S_ERROR( errStr ) # Anything other than 'True' in the 'Master' option means it is not catalogConfig['Master'] = ( catalogConfig.setdefault( 'Master', False ) == 'True' ) return S_OK( catalogConfig ) def _generateCatalogObject( self, catalogName ): """ Create a file catalog object from its name and CS description """ useProxy = gConfig.getValue( '/LocalSite/Catalogs/%s/UseProxy' % catalogName, False ) if not useProxy: useProxy = self.opHelper.getValue( '/Services/Catalogs/%s/UseProxy' % catalogName, False ) return FileCatalogFactory().createCatalog( catalogName, useProxy )
class OverlayInput(LCUtilityApplication): """ Helper call to define Overlay processor/driver inputs. Example: >>> over = OverlayInput() >>> over.setBXOverlay(300) >>> over.setGGToHadInt(3.2) >>> over.setNumberOfSignalEventsPerJob(10) >>> over.setBackgroundType("gghad") """ def __init__(self, paramdict = None): self._ops = Operations() self.bxToOverlay = None self.numberOfGGToHadronInteractions = 0 self.numberOfSignalEventsPerJob = 0 self.backgroundEventType = '' self.prodID = 0 self.machine = 'clic_cdr' self.detectorModel = '' self.useEnergyForFileLookup = True super(OverlayInput, self).__init__( paramdict ) self.version = '1' self._modulename = "OverlayInput" self.appname = self._modulename self._moduledescription = 'Helper call to define Overlay processor/driver inputs' self.accountInProduction = False self._paramsToExclude.append('_ops') self.pathToOverlayFiles = '' self.processorName = '' def setMachine(self, machine): """ Define the machine to use, clic_cdr or ilc_dbd """ self._checkArgs( { 'machine' : types.StringTypes } ) self.machine = machine def setProdID(self, pid): """ Define the prodID to use as input, experts only """ self._checkArgs( {'pid': types.IntType}) self.prodID = pid return S_OK() def setUseEnergyForFileLookup(self, useEnergyForFileLookup): """ Sets the flag to use the energy meta data in the search of the background files. Disable the energy when you want to use files created for a different energy than the signal events :param bool useEnergyForFileLookup: Use the Energy in the metadata search or not """ self._checkArgs( {'useEnergyForFileLookup': types.BooleanType } ) self.useEnergyForFileLookup = useEnergyForFileLookup return S_OK() def setOverlayBXPerSigEvt( self, bxoverlay): """ Define number bunch crossings to overlay for each signal event. This is used to determine the number of required overlay events. It does not modify any of the actual application parameters using the overly input. Alias for :func:`setBXOverlay` :param int bxoverlay: Bunch crossings to overlay. """ self._checkArgs( { 'bxoverlay' : types.IntType } ) self.bxToOverlay = bxoverlay return S_OK() def setBXOverlay(self, bxoverlay): """ Define number bunch crossings to overlay for each signal event. This is used to determine the number of required overlay events. It does not modify any of the actual application parameters using the overly input. :param int bxoverlay: Bunch crossings to overlay. """ return self.setOverlayBXPerSigEvt( bxoverlay ) def setOverlayEvtsPerBX( self, ggtohadint ): """ Define the number of overlay events per bunch crossing. This is used to determine the number of required overlay events. It does not modify any of the actual application parameters using the overly input. :param float ggtohadint: optional number of overlay events interactions per bunch crossing """ self._checkArgs( { 'ggtohadint' : types.FloatType } ) self.numberOfGGToHadronInteractions = ggtohadint return S_OK() def setGGToHadInt(self, ggtohadint): """ Define the number of overlay events per bunch crossing. This is used to determine the number of required overlay events. It does not modify any of the actual application parameters using the overly input. Alias for :func:`setOverlayEvtsPerBX` :param float ggtohadint: optional number of overlay events interactions per bunch crossing """ return self.setOverlayEvtsPerBX( ggtohadint ) def setNbSigEvtsPerJob(self, nbsigevtsperjob): """ Set the number of signal events per job. This is used to determine the number of required overlay events. It does not modify any of the actual application parameters using the overly input. :param int nbsigevtsperjob: Number of signal events per job """ self._checkArgs( { 'nbsigevtsperjob' : types.IntType } ) self.numberOfSignalEventsPerJob = nbsigevtsperjob return S_OK() def setDetectorModel(self, detectormodel): """ Set the detector type for the background files. Files are defined in the ConfigurationSystem: Operations/Overlay/<Accelerator>/<energy>/<Detector> :param str detectormodel: Detector type. Must be 'CLIC_ILD_CDR' or 'CLIC_SID_CDR' or 'sidloi3' or 'ILD_o1_v05' """ self._checkArgs( { 'detectormodel' : types.StringTypes } ) self.detectorModel = detectormodel return S_OK() def setPathToFiles(self, path): """ Sets the path to where the overlay files are located. Setting this option will ignore all other settings! :param str path: LFN path to the folder containing the overlay files """ self._checkArgs( { 'path' : types.StringTypes } ) self.pathToOverlayFiles = path return S_OK() def setBkgEvtType(self, backgroundEventType): """ Define the background type. .. deprecated:: 23r0 Use :func:`setBackgroundType` instead :param str backgroundEventType: Background type. """ self._checkArgs( { 'backgroundEventType' : types.StringTypes } ) self.backgroundEventType = backgroundEventType return S_OK() def setBackgroundType(self, backgroundType): """Define the background type :param str backgroundType: Background type. """ return self.setBkgEvtType(backgroundType) def setProcessorName(self, processorName): """Set the processorName to set the input files for. Necessary if multiple invocations of the overlay processor happen in marlin for example. Different processors must use different background types :param str processorName: Name of the Processor these input files are for """ self._checkArgs( { 'processorName' : types.StringTypes } ) self.processorName = processorName return S_OK() def setNumberOfSignalEventsPerJob(self, numberSignalEvents): """Alternative to :func:`setNbSigEvtsPerJob` Number used to determine the number of background files needed. :param int numberSignalEvents: Number of signal events per job """ return self.setNbSigEvtsPerJob(numberSignalEvents) # def setProdIDToUse(self,prodid): # """ Optional parameter: Define the production ID to use as input # # :param int prodid: Production ID # """ # self._checkArgs({"prodid" : types.IntType}) # self.prodid = prodid # return S_OK() def _applicationModule(self): m1 = self._createModuleDefinition() m1.addParameter(Parameter("BXOverlay", 0, "float", "", "", False, False, "Bunch crossings to overlay")) m1.addParameter(Parameter("ggtohadint", 0, "float", "", "", False, False, "Optional number of gamma gamma -> hadrons interactions per bunch crossing, default is 3.2")) m1.addParameter(Parameter("NbSigEvtsPerJob", 0, "int", "", "", False, False, "Number of signal events per job")) m1.addParameter(Parameter("prodid", 0, "int", "", "", False, False, "ProdID to use")) m1.addParameter(Parameter("BkgEvtType", "", "string", "", "", False, False, "Background type.")) m1.addParameter(Parameter("detectormodel", "", "string", "", "", False, False, "Detector type.")) m1.addParameter(Parameter("machine", "", "string", "", "", False, False, "machine: clic_cdr or ilc_dbd")) m1.addParameter(Parameter("useEnergyForFileLookup", True, "bool", "", "", False, False, "useEnergy to look for background files: True or False")) m1.addParameter(Parameter("pathToOverlayFiles", "", "string", "", "", False, False, "use overlay files from this path")) m1.addParameter(Parameter("processorName", "", "string", "", "", False, False, "Processor Name")) m1.addParameter(Parameter("debug", False, "bool", "", "", False, False, "debug mode")) return m1 def _applicationModuleValues(self, moduleinstance): moduleinstance.setValue("BXOverlay", self.bxToOverlay) moduleinstance.setValue('ggtohadint', self.numberOfGGToHadronInteractions) moduleinstance.setValue('NbSigEvtsPerJob', self.numberOfSignalEventsPerJob) moduleinstance.setValue('prodid', self.prodID) moduleinstance.setValue('BkgEvtType', self.backgroundEventType) moduleinstance.setValue('detectormodel', self.detectorModel) moduleinstance.setValue('debug', self.debug) moduleinstance.setValue('machine', self.machine ) moduleinstance.setValue('useEnergyForFileLookup', self.useEnergyForFileLookup ) moduleinstance.setValue('pathToOverlayFiles', self.pathToOverlayFiles ) moduleinstance.setValue('processorName', self.processorName ) def _userjobmodules(self, stepdefinition): res1 = self._setApplicationModuleAndParameters(stepdefinition) if not res1["OK"] : return S_ERROR('userjobmodules failed') return S_OK() def _prodjobmodules(self, stepdefinition): res1 = self._setApplicationModuleAndParameters(stepdefinition) if not res1["OK"] : return S_ERROR('prodjobmodules failed') return S_OK() def _addParametersToStep(self, stepdefinition): res = self._addBaseParameters(stepdefinition) if not res["OK"]: return S_ERROR("Failed to set base parameters") return S_OK() def _checkConsistency(self, job=None): """ Checks that all needed parameters are set """ if self.pathToOverlayFiles: res = FileCatalogClient().findFilesByMetadata({}, self.pathToOverlayFiles) if not res['OK']: return res LOG.notice("Found %i files in path %s" % (len(res['Value']), self.pathToOverlayFiles)) if not res['Value']: return S_ERROR("OverlayInput: PathToFiles is specified, but there are no files in that path") if not self.bxToOverlay : return S_ERROR("Number of overlay bunch crossings not defined") if not self.numberOfGGToHadronInteractions : return S_ERROR("Number of background events per bunch crossing is not defined") if not self.backgroundEventType : return S_ERROR("Background event type is not defined: Chose one gghad, aa_lowpt, ...") if self._jobtype == 'User' : if not self.numberOfSignalEventsPerJob : return S_ERROR("Number of signal event per job is not defined") else: self.prodparameters['detectorModel'] = self.detectorModel self.prodparameters['BXOverlay'] = self.bxToOverlay self.prodparameters['GGtoHadInt'] = self.numberOfGGToHadronInteractions return S_OK() def _checkFinalConsistency(self): """ Final check of consistency: the overlay files for the specifed energy must exist """ if self.pathToOverlayFiles: return S_OK() # can ignore other parameter if not self.energy: return S_ERROR("Energy MUST be specified for the overlay") res = self._ops.getSections('/Overlay') if not res['OK']: return S_ERROR("Could not resolve the CS path to the overlay specifications") sections = res['Value'] if self.machine not in sections: return S_ERROR("Machine %s does not have overlay data, use any of %s" % (self.machine, sections)) energytouse = energyWithLowerCaseUnit( self.energy ) res = self._ops.getSections("/Overlay/%s" % self.machine) if energytouse not in res['Value']: return S_ERROR("No overlay files corresponding to %s" % energytouse) res = self._ops.getSections("/Overlay/%s/%s" % (self.machine, energytouse)) if not res['OK']: return S_ERROR("Could not find the detector models") if self.detectorModel not in res['Value']: return S_ERROR("Detector model specified has no overlay data with that energy and machine") res = allowedBkg(self.backgroundEventType, energytouse, detectormodel = self.detectorModel, machine = self.machine) if not res['OK']: return res if res['Value'] < 0: return S_ERROR("No proper production ID found") return S_OK()
class OverlayDB ( DB ): """ DB for OverlaySystem """ def __init__( self ): """ """ self.ops = Operations() self.dbname = 'OverlayDB' self.logger = gLogger.getSubLogger('OverlayDB') DB.__init__( self, self.dbname, 'Overlay/OverlayDB' ) self._createTables( { "OverlayData" : { 'Fields' : { 'Site' : "VARCHAR(255) UNIQUE NOT NULL", 'NumberOfJobs' : "INTEGER DEFAULT 0" }, 'PrimaryKey' : 'Site', 'Indexes': {'Index':['Site']} } } ) limits = self.ops.getValue("/Overlay/MaxConcurrentRunning", 200) self.limits = {} self.limits["default"] = limits res = self.ops.getSections("/Overlay/Sites/") sites = [] if res['OK']: sites = res['Value'] for tempsite in sites: res = self.ops.getValue("/Overlay/Sites/%s/MaxConcurrentRunning" % tempsite, 200) self.limits[tempsite] = res self.logger.info("Using the following restrictions : %s" % self.limits) ##################################################################### # Private methods def __getConnection( self, connection ): if connection: return connection res = self._getConnection() if res['OK']: return res['Value'] gLogger.warn( "Failed to get MySQL connection", res['Message'] ) return connection def _checkSite(self, site, connection = False ): """ Check the number of jobs running at a given site. """ connection = self.__getConnection( connection ) req = "SELECT NumberOfJobs FROM OverlayData WHERE Site='%s';" % (site) res = self._query( req, connection ) if not res['OK']: return S_ERROR("Could not get site") if len(res['Value']): return res else: return S_ERROR("Could not find any site %s"%(site)) def _addSite(self, site, connection = False ): """ Add a new site to the DB """ connection = self.__getConnection( connection ) req = "INSERT INTO OverlayData (Site,NumberOfJobs) VALUES ('%s',1);" % site res = self._update( req, connection ) if not res['OK']: return res return res def _limitForSite(self, site): """ Get the current limit of jobs for a given site. """ if site in self.limits.keys(): return self.limits[site] return self.limits['default'] def _addNewJob(self, site, nbjobs, connection = False ): """ Add a new running job in the DB """ connection = self.__getConnection( connection ) nbjobs += 1 req = "UPDATE OverlayData SET NumberOfJobs=%s WHERE Site='%s';" % (nbjobs, site) self._update( req, connection ) return S_OK() ### Methods to fix the site def getSites(self, connection = False): """ Return the list of sites known to the service """ connection = self.__getConnection( connection ) req = 'SELECT Site From OverlayData;' res = self._query( req, connection ) if not res['OK']: return S_ERROR("Could not get sites") sites = [] for row in res['Value']: sites.append(row[0]) return S_OK(sites) def setJobsAtSites(self, sitedict, connection = False): """ As name suggests: set the number of jobs running at the site. """ connection = self.__getConnection( connection ) for site, nbjobs in sitedict.items(): req = "UPDATE OverlayData SET NumberOfJobs=%i WHERE Site='%s';" % (int(nbjobs), site) res = self._update( req, connection ) if not res['OK']: return S_ERROR("Could not set number of jobs at site %s" % site) return S_OK() ### Useful methods for the users def getJobsAtSite(self, site, connection = False ): """ Get the number of jobs currently run """ connection = self.__getConnection( connection ) nbjobs = 0 res = self._checkSite(site, connection) if not res['OK']: return S_OK(nbjobs) nbjobs = res['Value'][0][0] return S_OK(nbjobs) ### Important methods def canRun(self, site, connection = False ): """ Can the job run at that site? """ connection = self.__getConnection( connection ) res = self._checkSite(site, connection) nbjobs = 0 if not res['OK']: self._addSite(site, connection) nbjobs = 1 else: nbjobs = res['Value'][0][0] if nbjobs < self._limitForSite(site): res = self._addNewJob(site, nbjobs, connection) if not res['OK']: return res return S_OK(True) else: return S_OK(False) def jobDone(self, site, connection = False ): """ Remove a job from the DB, should not remove a job from the DB if the Site does not exist, but this should never happen """ connection = self.__getConnection( connection ) res = self._checkSite(site, connection) if not res['OK']: return res nbjobs = res['Value'][0][0] if nbjobs == 1: return S_OK() nbjobs -= 1 req = "UPDATE OverlayData SET NumberOfJobs=%s WHERE Site='%s';" % (nbjobs, site) res = self._update( req, connection ) if not res['OK']: return res return S_OK()
class FileCatalog(object): def __init__(self, catalogs=None, vo=None): """Default constructor""" self.valid = True self.timeout = 180 self.ro_methods = set() self.write_methods = set() self.no_lfn_methods = set() self.readCatalogs = [] self.writeCatalogs = [] self.rootConfigPath = "/Resources/FileCatalogs" self.vo = vo if vo else getVOfromProxyGroup().get("Value", None) self.log = gLogger.getSubLogger(self.__class__.__name__) self.opHelper = Operations(vo=self.vo) catalogList = [] if isinstance(catalogs, six.string_types): catalogList = [catalogs] elif isinstance(catalogs, (list, tuple)): catalogList = list(catalogs) if catalogList: result = self._getEligibleCatalogs() if not result["OK"]: self.log.error("Failed to get eligible catalog") return eligibleFileCatalogs = result["Value"] catalogCheck = True for catalog in catalogList: if catalog not in eligibleFileCatalogs: self.log.error("Specified catalog is not eligible", catalog) catalogCheck = False if catalogCheck: result = self._getSelectedCatalogs(catalogList) else: result = S_ERROR("Specified catalog is not eligible") else: result = self._getCatalogs() if not result["OK"]: self.log.error("Failed to create catalog objects") self.valid = False elif (len(self.readCatalogs) == 0) and (len(self.writeCatalogs) == 0): self.log.error("No catalog object created") self.valid = False result = self.getMasterCatalogNames() masterCatalogs = result["Value"] # There can not be more than one master catalog haveMaster = False if len(masterCatalogs) > 1: self.log.error("More than one master catalog created") self.valid = False elif len(masterCatalogs) == 1: haveMaster = True # Get the list of write methods if haveMaster: # All the write methods must be present in the master _catalogName, oCatalog, _master = self.writeCatalogs[0] _roList, writeList, nolfnList = oCatalog.getInterfaceMethods() self.write_methods.update(writeList) self.no_lfn_methods.update(nolfnList) else: for _catalogName, oCatalog, _master in self.writeCatalogs: _roList, writeList, nolfnList = oCatalog.getInterfaceMethods() self.write_methods.update(writeList) self.no_lfn_methods.update(nolfnList) # Get the list of read methods for _catalogName, oCatalog, _master in self.readCatalogs: roList, _writeList, nolfnList = oCatalog.getInterfaceMethods() self.ro_methods.update(roList) self.no_lfn_methods.update(nolfnList) self.condParser = FCConditionParser(vo=self.vo, ro_methods=self.ro_methods) def isOK(self): return self.valid def getReadCatalogs(self): return self.readCatalogs def getWriteCatalogs(self): return self.writeCatalogs def getMasterCatalogNames(self): """Returns the list of names of the Master catalogs""" masterNames = [catalogName for catalogName, oCatalog, master in self.writeCatalogs if master] return S_OK(masterNames) def __getattr__(self, name): self.call = name if name in self.write_methods: return self.w_execute elif name in self.ro_methods: return self.r_execute else: raise AttributeError def w_execute(self, *parms, **kws): """Write method executor. If one of the LFNs given as input does not pass a condition defined for the master catalog, we return S_ERROR without trying anything else :param fcConditions: either a dict or a string, to be propagated to the FCConditionParser * If it is a string, it is given for all catalogs * If it is a dict, it has to be { catalogName: condition}, and only the specific condition for the catalog will be given .. warning :: If the method is a write no_lfn method, then the return value are completely different. We only return the result of the master catalog """ successful = {} failed = {} failedCatalogs = {} successfulCatalogs = {} specialConditions = kws.pop("fcConditions") if "fcConditions" in kws else None allLfns = [] lfnMapDict = {} masterResult = {} parms1 = [] if self.call not in self.no_lfn_methods: fileInfo = parms[0] result = checkArgumentFormat(fileInfo, generateMap=True) if not result["OK"]: return result fileInfo, lfnMapDict = result["Value"] # No need to check the LFNs again in the clients kws["LFNChecking"] = False allLfns = list(fileInfo) parms1 = parms[1:] for catalogName, oCatalog, master in self.writeCatalogs: # Skip if the method is not implemented in this catalog # NOTE: it is impossible for the master since the write method list is populated # only from the master catalog, and if the method is not there, __getattr__ # would raise an exception if not oCatalog.hasCatalogMethod(self.call): continue method = getattr(oCatalog, self.call) if self.call in self.no_lfn_methods: result = method(*parms, **kws) else: if isinstance(specialConditions, dict): condition = specialConditions.get(catalogName) else: condition = specialConditions # Check whether this catalog should be used for this method res = self.condParser(catalogName, self.call, fileInfo, condition=condition) # condParser never returns S_ERROR condEvals = res["Value"]["Successful"] # For a master catalog, ALL the lfns should be valid if master: if any([not valid for valid in condEvals.values()]): gLogger.error("The master catalog is not valid for some LFNS", condEvals) return S_ERROR("The master catalog is not valid for some LFNS %s" % condEvals) validLFNs = dict((lfn, fileInfo[lfn]) for lfn in condEvals if condEvals[lfn]) # We can skip the execution without worry, # since at this level it is for sure not a master catalog if not validLFNs: gLogger.debug("No valid LFN, skipping the call") continue invalidLFNs = [lfn for lfn in condEvals if not condEvals[lfn]] if invalidLFNs: gLogger.debug( "Some LFNs are not valid for operation '%s' on catalog '%s' : %s" % (self.call, catalogName, invalidLFNs) ) result = method(validLFNs, *parms1, **kws) if master: masterResult = result if not result["OK"]: if master: # If this is the master catalog and it fails we don't want to continue with the other catalogs self.log.error( "Failed to execute call on master catalog", "%s on %s: %s" % (self.call, catalogName, result["Message"]), ) return result else: # Otherwise we keep the failed catalogs so we can update their state later failedCatalogs[catalogName] = result["Message"] else: successfulCatalogs[catalogName] = result["Value"] if allLfns: if result["OK"]: for lfn, message in result["Value"]["Failed"].items(): # Save the error message for the failed operations failed.setdefault(lfn, {})[catalogName] = message if master: # If this is the master catalog then we should not attempt the operation on other catalogs fileInfo.pop(lfn, None) for lfn, result in result["Value"]["Successful"].items(): # Save the result return for each file for the successful operations successful.setdefault(lfn, {})[catalogName] = result if allLfns: # This recovers the states of the files that completely failed i.e. when S_ERROR is returned by a catalog for catalogName, errorMessage in failedCatalogs.items(): for lfn in allLfns: failed.setdefault(lfn, {})[catalogName] = errorMessage # Restore original lfns if they were changed by normalization if lfnMapDict: for lfn in list(failed): failed[lfnMapDict.get(lfn, lfn)] = failed.pop(lfn) for lfn in list(successful): successful[lfnMapDict.get(lfn, lfn)] = successful.pop(lfn) resDict = {"Failed": failed, "Successful": successful} return S_OK(resDict) else: # FIXME: Return just master result here. This is temporary as more detailed # per catalog result needs multiple fixes in various client calls return masterResult def r_execute(self, *parms, **kws): """Read method executor.""" successful = {} failed = {} for _catalogName, oCatalog, _master in self.readCatalogs: # Skip if the method is not implemented in this catalog if not oCatalog.hasCatalogMethod(self.call): continue method = getattr(oCatalog, self.call) res = method(*parms, **kws) if res["OK"]: if "Successful" in res["Value"]: for key, item in res["Value"]["Successful"].items(): successful.setdefault(key, item) failed.pop(key, None) for key, item in res["Value"]["Failed"].items(): if key not in successful: failed[key] = item else: return res if not successful and not failed: return S_ERROR(DErrno.EFCERR, "Failed to perform %s from any catalog" % self.call) return S_OK({"Failed": failed, "Successful": successful}) ########################################################################################### # # Below is the method for obtaining the objects instantiated for a provided catalogue configuration # def addCatalog(self, catalogName, mode="Write", master=False): """Add a new catalog with catalogName to the pool of catalogs in mode: "Read","Write" or "ReadWrite" """ result = self._generateCatalogObject(catalogName) if not result["OK"]: return result oCatalog = result["Value"] if mode.lower().find("read") != -1: self.readCatalogs.append((catalogName, oCatalog, master)) if mode.lower().find("write") != -1: self.writeCatalogs.append((catalogName, oCatalog, master)) return S_OK() def removeCatalog(self, catalogName): """Remove the specified catalog from the internal pool""" catalog_removed = False for i in range(len(self.readCatalogs)): catalog, _object, _master = self.readCatalogs[i] if catalog == catalogName: del self.readCatalogs[i] catalog_removed = True break for i in range(len(self.writeCatalogs)): catalog, _object, _master = self.writeCatalogs[i] if catalog == catalogName: del self.writeCatalogs[i] catalog_removed = True break if catalog_removed: return S_OK() else: return S_OK("Catalog does not exist") def _getSelectedCatalogs(self, desiredCatalogs): for catalogName in desiredCatalogs: result = self._getCatalogConfigDetails(catalogName) if not result["OK"]: return result catalogConfig = result["Value"] result = self._generateCatalogObject(catalogName) if not result["OK"]: return result oCatalog = result["Value"] if re.search("Read", catalogConfig["AccessType"]): if catalogConfig["Master"]: self.readCatalogs.insert(0, (catalogName, oCatalog, catalogConfig["Master"])) else: self.readCatalogs.append((catalogName, oCatalog, catalogConfig["Master"])) if re.search("Write", catalogConfig["AccessType"]): if catalogConfig["Master"]: self.writeCatalogs.insert(0, (catalogName, oCatalog, catalogConfig["Master"])) else: self.writeCatalogs.append((catalogName, oCatalog, catalogConfig["Master"])) return S_OK() def _getEligibleCatalogs(self): """Get a list of eligible catalogs :return: S_OK/S_ERROR, Value - a list of catalog names """ # First, look in the Operations, if nothing defined look in /Resources for backward compatibility fileCatalogs = self.opHelper.getValue("/Services/Catalogs/CatalogList", []) if not fileCatalogs: result = self.opHelper.getSections("/Services/Catalogs") if result["OK"]: fileCatalogs = result["Value"] else: res = gConfig.getSections(self.rootConfigPath, listOrdered=True) if not res["OK"]: errStr = "FileCatalog._getEligibleCatalogs: Failed to get file catalog configuration." self.log.error(errStr, res["Message"]) return S_ERROR(errStr) fileCatalogs = res["Value"] return S_OK(fileCatalogs) def _getCatalogs(self): """Updates self.readCatalogs and self.writeCatalogs with list of catalog objects as found in the CS""" # Get the eligible catalogs first result = self._getEligibleCatalogs() if not result["OK"]: return result fileCatalogs = result["Value"] # Get the catalog objects now for catalogName in fileCatalogs: res = self._getCatalogConfigDetails(catalogName) if not res["OK"]: return res catalogConfig = res["Value"] if catalogConfig["Status"] == "Active": res = self._generateCatalogObject(catalogName) if not res["OK"]: return res oCatalog = res["Value"] master = catalogConfig["Master"] # If the catalog is read type if re.search("Read", catalogConfig["AccessType"]): if master: self.readCatalogs.insert(0, (catalogName, oCatalog, master)) else: self.readCatalogs.append((catalogName, oCatalog, master)) # If the catalog is write type if re.search("Write", catalogConfig["AccessType"]): if master: self.writeCatalogs.insert(0, (catalogName, oCatalog, master)) else: self.writeCatalogs.append((catalogName, oCatalog, master)) return S_OK() def _getCatalogConfigDetails(self, catalogName): # First obtain the options that are available catalogConfigPath = "%s/%s" % (self.rootConfigPath, catalogName) result = gConfig.getOptionsDict(catalogConfigPath) if not result["OK"]: errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options." self.log.error(errStr, catalogName) return S_ERROR(errStr) catalogConfig = result["Value"] result = self.opHelper.getOptionsDict("/Services/Catalogs/%s" % catalogName) if result["OK"]: catalogConfig.update(result["Value"]) # The 'Status' option should be defined (default = 'Active') if "Status" not in catalogConfig: warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined." self.log.warn(warnStr, catalogName) catalogConfig["Status"] = "Active" # The 'AccessType' option must be defined if "AccessType" not in catalogConfig: errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined." self.log.error(errStr, catalogName) return S_ERROR(errStr) # Anything other than 'True' in the 'Master' option means it is not catalogConfig["Master"] = catalogConfig.setdefault("Master", False) == "True" return S_OK(catalogConfig) def _generateCatalogObject(self, catalogName): """Create a file catalog object from its name and CS description""" useProxy = gConfig.getValue("/LocalSite/Catalogs/%s/UseProxy" % catalogName, False) if not useProxy: useProxy = self.opHelper.getValue("/Services/Catalogs/%s/UseProxy" % catalogName, False) return FileCatalogFactory().createCatalog(catalogName, useProxy)
def addShifter( self, shifters = None ): """ Adds or modify one or more shifters. Also, adds the shifter section in case this is not present. Shifter identities are used in several places, mostly for running agents shifters should be in the form {'ShifterRole':{'User':'******', 'Group':'aDIRACGroup'}} :return: S_OK/S_ERROR """ def getOpsSection(): """ Where is the shifters section? """ vo = CSGlobals.getVO() setup = CSGlobals.getSetup() if vo: res = gConfig.getSections( '/Operations/%s/%s/Shifter' % (vo, setup) ) if res['OK']: return S_OK( '/Operations/%s/%s/Shifter' % ( vo, setup ) ) res = gConfig.getSections( '/Operations/%s/Defaults/Shifter' % vo ) if res['OK']: return S_OK( '/Operations/%s/Defaults/Shifter' % vo ) else: res = gConfig.getSections( '/Operations/%s/Shifter' % setup ) if res['OK']: return S_OK( '/Operations/%s/Shifter' % setup ) res = gConfig.getSections( '/Operations/Defaults/Shifter' ) if res['OK']: return S_OK( '/Operations/Defaults/Shifter' ) return S_ERROR( "No shifter section" ) if shifters is None: shifters = {} if not self.__initialized['OK']: return self.__initialized # get current shifters opsH = Operations( ) currentShifterRoles = opsH.getSections( 'Shifter' ) if not currentShifterRoles['OK']: # we assume the shifter section is not present currentShifterRoles = [] else: currentShifterRoles = currentShifterRoles['Value'] currentShiftersDict = {} for currentShifterRole in currentShifterRoles: currentShifter = opsH.getOptionsDict( 'Shifter/%s' % currentShifterRole ) if not currentShifter['OK']: return currentShifter currentShifter = currentShifter['Value'] currentShiftersDict[currentShifterRole] = currentShifter # Removing from shifters what does not need to be changed for sRole in shifters: if sRole in currentShiftersDict: if currentShiftersDict[sRole] == shifters[sRole]: shifters.pop( sRole ) # get shifters section to modify section = getOpsSection() # Is this section present? if not section['OK']: if section['Message'] == "No shifter section": gLogger.warn( section['Message'] ) gLogger.info( "Adding shifter section" ) vo = CSGlobals.getVO() if vo: section = '/Operations/%s/Defaults/Shifter' % vo else: section = '/Operations/Defaults/Shifter' res = self.__csMod.createSection( section ) if not res: gLogger.error( "Section %s not created" % section ) return S_ERROR( "Section %s not created" % section ) else: gLogger.error( section['Message'] ) return section else: section = section['Value'] #add or modify shifters for shifter in shifters: self.__csMod.removeSection( section + '/' + shifter ) self.__csMod.createSection( section + '/' + shifter ) self.__csMod.createSection( section + '/' + shifter + '/' + 'User' ) self.__csMod.createSection( section + '/' + shifter + '/' + 'Group' ) self.__csMod.setOptionValue( section + '/' + shifter + '/' + 'User', shifters[shifter]['User'] ) self.__csMod.setOptionValue( section + '/' + shifter + '/' + 'Group', shifters[shifter]['Group'] ) self.__csModified = True return S_OK( True )
class DMSHelpers(object): """ This class is used to get information about sites, SEs and their interrelations """ def __init__(self, vo=False): self.siteSEMapping = {} self.storageElementSet = set() self.siteSet = set() self.__opsHelper = Operations(vo=vo) self.failoverSEs = None self.archiveSEs = None self.notForJobSEs = None def getSiteSEMapping(self): """Returns a dictionary of all sites and their localSEs as a list, e.g. {'LCG.CERN.ch':['CERN-RAW','CERN-RDST',...]} """ if self.siteSEMapping: return S_OK(self.siteSEMapping) # Get the list of SEs and keep a mapping of those using an Alias or a # BaseSE storageElements = gConfig.getSections("Resources/StorageElements") if not storageElements["OK"]: gLogger.warn("Problem retrieving storage elements", storageElements["Message"]) return storageElements storageElements = storageElements["Value"] equivalentSEs = {} for se in storageElements: for option in ("BaseSE", "Alias"): originalSE = gConfig.getValue( "Resources/StorageElements/%s/%s" % (se, option)) if originalSE: equivalentSEs.setdefault(originalSE, []).append(se) break siteSEMapping = {} gridTypes = gConfig.getSections("Resources/Sites/") if not gridTypes["OK"]: gLogger.warn("Problem retrieving sections in /Resources/Sites", gridTypes["Message"]) return gridTypes gridTypes = gridTypes["Value"] gLogger.debug("Grid Types are: %s" % (", ".join(gridTypes))) # Get a list of sites and their local SEs siteSet = set() storageElementSet = set() siteSEMapping[LOCAL] = {} for grid in gridTypes: result = gConfig.getSections("/Resources/Sites/%s" % grid) if not result["OK"]: gLogger.warn("Problem retrieving /Resources/Sites/%s section" % grid) return result sites = result["Value"] siteSet.update(sites) for site in sites: candidateSEs = gConfig.getValue( "/Resources/Sites/%s/%s/SE" % (grid, site), []) if candidateSEs: candidateSEs += [ eqSE for se in candidateSEs for eqSE in equivalentSEs.get(se, []) ] siteSEMapping[LOCAL].setdefault(site, set()).update(candidateSEs) storageElementSet.update(candidateSEs) # Add Sites from the SiteSEMappingByProtocol in the CS siteSEMapping[PROTOCOL] = {} cfgLocalSEPath = cfgPath("SiteSEMappingByProtocol") result = self.__opsHelper.getOptionsDict(cfgLocalSEPath) if result["OK"]: sites = result["Value"] for site in sites: candidates = set( self.__opsHelper.getValue(cfgPath(cfgLocalSEPath, site), [])) ses = set(resolveSEGroup(candidates - siteSet)) | (candidates & siteSet) # If a candidate is a site, then all local SEs are eligible for candidate in ses & siteSet: ses.remove(candidate) ses.update(siteSEMapping[LOCAL][candidate]) siteSEMapping[PROTOCOL].setdefault(site, set()).update(ses) # Add Sites from the SiteSEMappingByDownload in the CS, else # SiteLocalSEMapping (old convention) siteSEMapping[DOWNLOAD] = {} cfgLocalSEPath = cfgPath("SiteSEMappingByDownload") result = self.__opsHelper.getOptionsDict(cfgLocalSEPath) if not result["OK"]: cfgLocalSEPath = cfgPath("SiteLocalSEMapping") result = self.__opsHelper.getOptionsDict(cfgLocalSEPath) if result["OK"]: sites = result["Value"] for site in sites: candidates = set( self.__opsHelper.getValue(cfgPath(cfgLocalSEPath, site), [])) ses = set(resolveSEGroup(candidates - siteSet)) | (candidates & siteSet) # If a candidate is a site, then all local SEs are eligible for candidate in ses & siteSet: ses.remove(candidate) ses.update(siteSEMapping[LOCAL][candidate]) siteSEMapping[DOWNLOAD].setdefault(site, set()).update(ses) self.siteSEMapping = siteSEMapping # Add storage elements that may not be associated with a site result = gConfig.getSections("/Resources/StorageElements") if not result["OK"]: gLogger.warn( "Problem retrieving /Resources/StorageElements section", result["Message"]) return result self.storageElementSet = storageElementSet | set(result["Value"]) self.siteSet = siteSet return S_OK(siteSEMapping) def getSites(self): """Get the list of known sites""" self.getSiteSEMapping() return sorted(self.siteSet) def getTiers(self, withStorage=False, tier=None): """Get the list of sites for a given (list of) Tier level""" sites = sorted( self.getShortSiteNames(withStorage=withStorage, tier=tier).values()) if sites and isinstance(sites[0], list): # List of lists, flatten it sites = [s for sl in sites for s in sl] return sites def getShortSiteNames(self, withStorage=True, tier=None): """Create a directory of short site names pointing to full site names""" siteDict = {} result = self.getSiteSEMapping() if result["OK"]: for site in self.siteSEMapping[ LOCAL] if withStorage else self.siteSet: grid, shortSite, _country = site.split(".") if isinstance(tier, six.integer_types) and ( grid != "LCG" or gConfig.getValue( "/Resources/Sites/%s/%s/MoUTierLevel" % (grid, site), 999) != tier): continue if isinstance(tier, (list, tuple, dict, set)) and (grid != "LCG" or gConfig.getValue( "/Resources/Sites/%s/%s/MoUTierLevel" % (grid, site), 999) not in tier): continue if withStorage or tier is not None: siteDict[shortSite] = site else: siteDict.setdefault(shortSite, []).append(site) return siteDict def getStorageElements(self): """Get the list of known SEs""" self.getSiteSEMapping() return sorted(self.storageElementSet) def isSEFailover(self, storageElement): """Is this SE a failover SE""" if self.failoverSEs is None: seList = resolveSEGroup( self.__opsHelper.getValue("DataManagement/SEsUsedForFailover", [])) self.failoverSEs = resolveSEGroup(seList) # FIXME: remove string test at some point return storageElement in self.failoverSEs or ( not self.failoverSEs and isinstance(storageElement, six.string_types) and "FAILOVER" in storageElement.upper()) def isSEForJobs(self, storageElement, checkSE=True): """Is this SE suitable for making jobs""" if checkSE: self.getSiteSEMapping() if storageElement not in self.storageElementSet: return False if self.notForJobSEs is None: seList = resolveSEGroup( self.__opsHelper.getValue( "DataManagement/SEsNotToBeUsedForJobs", [])) self.notForJobSEs = resolveSEGroup(seList) return storageElement not in self.notForJobSEs def isSEArchive(self, storageElement): """Is this SE an archive SE""" if self.archiveSEs is None: seList = resolveSEGroup( self.__opsHelper.getValue("DataManagement/SEsUsedForArchive", [])) self.archiveSEs = resolveSEGroup(seList) # FIXME: remove string test at some point return storageElement in self.archiveSEs or ( not self.archiveSEs and isinstance(storageElement, six.string_types) and "ARCHIVE" in storageElement.upper()) def getSitesForSE(self, storageElement, connectionLevel=None): """Get the (list of) sites for a given SE and a given connctivity""" connectionIndex = _getConnectionIndex(connectionLevel, default=DOWNLOAD) if connectionIndex == LOCAL: return self._getLocalSitesForSE(storageElement) if connectionIndex == PROTOCOL: return self.getProtocolSitesForSE(storageElement) if connectionIndex == DOWNLOAD: return self.getDownloadSitesForSE(storageElement) return S_ERROR("Unknown connection level") def getLocalSiteForSE(self, se): """Get the site at which the SE is""" sites = self._getLocalSitesForSE(se) if not sites["OK"]: return sites if not sites["Value"]: return S_OK(None) return S_OK(sites["Value"][0]) def _getLocalSitesForSE(self, se): """Extract the list of sites that declare this SE""" mapping = self.getSiteSEMapping() if not mapping["OK"]: return mapping if se not in self.storageElementSet: return S_ERROR("Non-existing SE") mapping = mapping["Value"][LOCAL] sites = [site for site in mapping if se in mapping[site]] if len(sites) > 1 and self.__opsHelper.getValue( "DataManagement/ForceSingleSitePerSE", True): return S_ERROR("SE is at more than one site") return S_OK(sites) def getProtocolSitesForSE(self, se): """Get sites that can access the SE by protocol""" mapping = self.getSiteSEMapping() if not mapping["OK"]: return mapping if se not in self.storageElementSet: return S_ERROR("Non-existing SE") mapping = mapping["Value"][PROTOCOL] sites = self._getLocalSitesForSE(se) if not sites["OK"]: return sites sites = set(sites["Value"]) sites.update([site for site in mapping if se in mapping[site]]) return S_OK(sorted(sites)) def getDownloadSitesForSE(self, se): """Get the list of sites that are allowed to download files""" mapping = self.getSiteSEMapping() if not mapping["OK"]: return mapping if se not in self.storageElementSet: return S_ERROR("Non-existing SE") mapping = mapping["Value"][DOWNLOAD] sites = self.getProtocolSitesForSE(se) if not sites["OK"]: return sites sites = set(sites["Value"]) sites.update([site for site in mapping if se in mapping[site]]) return S_OK(sorted(sites)) def getSEsForSite(self, site, connectionLevel=None): """Get all SEs accessible from a site, given a connectivity""" connectionIndex = _getConnectionIndex(connectionLevel, default=DOWNLOAD) if connectionIndex is None: return S_ERROR("Unknown connection level") if not self.siteSet: self.getSiteSEMapping() if site not in self.siteSet: siteList = [s for s in self.siteSet if ".%s." % site in s] else: siteList = [site] if not siteList: return S_ERROR("Unknown site") return self._getSEsForSItes(siteList, connectionIndex=connectionIndex) def _getSEsForSItes(self, siteList, connectionIndex): """Extract list of SEs for a connectivity""" mapping = self.getSiteSEMapping() if not mapping["OK"]: return mapping ses = [] for index in range(LOCAL, connectionIndex + 1): for site in siteList: ses += mapping["Value"][index].get(site, []) if not ses: return S_ERROR("No SE found") return S_OK(sorted(ses)) def getSEsAtSite(self, site): """Get local SEs""" return self.getSEsForSite(site, connectionLevel=LOCAL) def isSameSiteSE(self, se1, se2): """Are these 2 SEs at the same site""" res = self.getLocalSiteForSE(se1) if not res["OK"]: return res site1 = res["Value"] res = self.getLocalSiteForSE(se2) if not res["OK"]: return res site2 = res["Value"] return S_OK(site1 == site2) def getSEsAtCountry(self, country, connectionLevel=None): """Get all SEs at a given country""" connectionIndex = _getConnectionIndex(connectionLevel, default=DOWNLOAD) if connectionIndex is None: return S_ERROR("Unknown connection level") if not self.siteSet: self.getSiteSEMapping() siteList = [ site for site in self.siteSet if siteCountryName(site) == country.lower() ] if not siteList: return S_ERROR("No SEs found in country") return self._getSEsForSItes(siteList, connectionIndex) def getSEInGroupAtSite(self, seGroup, site): """Get the SE in a group or list of SEs that is present at a site""" seList = self.getAllSEsInGroupAtSite(seGroup, site) if not seList["OK"] or seList["Value"] is None: return seList return S_OK(seList["Value"][0]) def getAllSEsInGroupAtSite(self, seGroup, site): """Get all SEs in a group or list of SEs that are present at a site""" seList = resolveSEGroup(seGroup) if not seList: return S_ERROR("SEGroup does not exist") sesAtSite = self.getSEsAtSite(site) if not sesAtSite["OK"]: return sesAtSite foundSEs = set(seList) & set(sesAtSite["Value"]) if not foundSEs: gLogger.warn("No SE found at that site", "in group %s at %s" % (seGroup, site)) return S_OK() return S_OK(sorted(foundSEs)) def getRegistrationProtocols(self): """Returns the Favorite registration protocol defined in the CS, or 'srm' as default""" return self.__opsHelper.getValue( "DataManagement/RegistrationProtocols", ["srm", "dips"]) def getThirdPartyProtocols(self): """Returns the Favorite third party protocol defined in the CS, or 'srm' as default""" return self.__opsHelper.getValue("DataManagement/ThirdPartyProtocols", ["srm"]) def getAccessProtocols(self): """Returns the Favorite access protocol defined in the CS, or 'srm' as default""" return self.__opsHelper.getValue("DataManagement/AccessProtocols", ["srm", "dips"]) def getWriteProtocols(self): """Returns the Favorite Write protocol defined in the CS, or 'srm' as default""" return self.__opsHelper.getValue("DataManagement/WriteProtocols", ["srm", "dips"]) def getStageProtocols(self): """Returns the Favorite staging protocol defined in the CS. There are no default""" return self.__opsHelper.getValue("DataManagement/StageProtocols", list()) def getMultiHopMatrix(self): """ Returns the multi-hop matrix described in DataManagement/MultiHopMatrixOfShame. .. code-block :: python 'Default': { 'Default': 'MultiHopSEUsedForAllTransfer', 'Dst3' : 'MultiHopFromAnySourceToDst3', } 'Src1' : { 'Default' : 'DefaultMultiHopSEFromSrc1', 'Dst1': 'MultiHopSEFromSrc1ToDst1}, 'Src2' : { 'Default' : 'DefaultMultiHopSEFromSrc2', 'Dst2': 'MultiHopSEFromSrc1ToDst2} :returns: dict of dict for all the source se / dest SE defined. We user defaultdict to allow for the use of non existing source/dest. """ matrixBasePath = "DataManagement/MultiHopMatrixOfShame" multiHopMatrix = defaultdict(lambda: defaultdict(lambda: None)) allSrcSEs = self.__opsHelper.getSections(matrixBasePath).get( "Value", []) for src in allSrcSEs: srcDst = self.__opsHelper.getOptionsDict( cfgPath(matrixBasePath, src)).get("Value") if srcDst: multiHopMatrix[src].update(srcDst) return multiHopMatrix
def web_getLaunchpadSetupWithLFNs(self): #on the fly file catalog for advanced launchpad if not hasattr(self, 'fc'): userData = self.getSessionData() group = str(userData["user"]["group"]) vo = getVOForGroup(group) self.fc = FileCatalog(vo=vo) self.set_header('Content-type', 'text/plain') lfnList = [] arguments = self.request.arguments gLogger.always( "submit: incoming arguments %s to getLaunchpadSetupWithLFNs" % arguments) lfnStr = str(arguments['path'][0]) lfnList = lfnStr.split(',') #checks if the experiments folder in lfn list has a rtg_def.m file at some subfolder gLogger.always("submit: checking if some rtg_def.m" % arguments) processed = [] metaDict = {'type': 'info'} for lfn in lfnStr.split(','): pos_relative = lfn.find("/") pos_relative = lfn.find("/", pos_relative + 1) pos_relative = lfn.find("/", pos_relative + 1) pos_relative = lfn.find("/", pos_relative + 1) pos_relative = lfn.find("/", pos_relative + 1) experiment_lfn = lfn[0:pos_relative] if experiment_lfn in processed: continue processed.append(experiment_lfn) gLogger.always("checking rtg_def.m in %s" % experiment_lfn) result = self.fc.findFilesByMetadata(metaDict, path=str(experiment_lfn)) print "result" print result if not result['OK'] or not result['Value']: gLogger.error("Failed to get type info from $s, %s" % (experiment_lfn, result["Message"])) continue for candidate_lfn in result['Value']: if candidate_lfn.find('rtg_def.m') > 0: lfnList.append(candidate_lfn) totalfn = len(lfnList) ptlfn = '' current = 1 for lfn in lfnList: ptlfn = ptlfn + lfn if current < totalfn: ptlfn = ptlfn + ', ' current = current + 1 defaultParams = { "JobName": [1, 'Eiscat'], "Executable": [1, "/bin/ls"], "Arguments": [1, "-ltrA"], "OutputSandbox": [1, "std.out, std.err"], "InputData": [1, ptlfn], "OutputData": [0, ""], "OutputSE": [1, "EISCAT-disk"], "OutputPath": [0, ""], "CPUTime": [0, "86400"], "Site": [0, ""], "BannedSite": [0, ""], "Platform": [0, "Linux_x86_64_glibc-2.5"], "Priority": [0, "5"], "StdError": [0, "std.err"], "StdOutput": [0, "std.out"], "Parameters": [0, "0"], "ParameterStart": [0, "0"], "ParameterStep": [0, "1"] } delimiter = gConfig.getValue("/Website/Launchpad/ListSeparator", ',') options = self.__getOptionsFromCS(delimiter=delimiter) # platform = self.__getPlatform() # if platform and options: # if not options.has_key("Platform"): # options[ "Platform" ] = platform # else: # csPlatform = list(options[ "Platform" ]) # allPlatforms = csPlatform + platform # platform = uniqueElements(allPlatforms) # options[ "Platform" ] = platform gLogger.debug("Options from CS: %s" % options) override = gConfig.getValue("/Website/Launchpad/OptionsOverride", False) gLogger.info("end __getLaunchpadOpts") # Updating the default values from OptionsOverride configuration branch, for key in options: if key not in defaultParams: defaultParams[key] = [0, ""] defaultParams[key][1] = options[key][0] gLogger.info( "Default params + override from /Website/Launchpad/OptionsOverride -> %s" % defaultParams) # Reading of the predefined sets of launchpad parameters values obj = Operations() predefinedSets = {} launchpadSections = obj.getSections("Launchpad") import pprint if launchpadSections['OK']: for section in launchpadSections["Value"]: predefinedSets[section] = {} sectionOptions = obj.getOptionsDict("Launchpad/" + section) pprint.pprint(sectionOptions) if sectionOptions['OK']: predefinedSets[section] = sectionOptions["Value"] self.write({ "success": "true", "result": defaultParams, "predefinedSets": predefinedSets })
class CombinedSoftwareInstallation(object): """ Combined means that it will try to install in the Shared area and in the LocalArea, depending on the user's rights """ def __init__(self, argumentsDict): """ Standard constructor Defines, from dictionary of job parameters passed, a set of members to hold e.g. the applications and the system config. Also determines the SharedArea and LocalArea. """ self.ops = Operations() self.job = {} if argumentsDict.has_key('Job'): self.job = argumentsDict['Job'] self.ce = {} if argumentsDict.has_key('CE'): self.ce = argumentsDict['CE'] self.source = {} if argumentsDict.has_key('Source'): self.source = argumentsDict['Source'] apps = [] if self.job.has_key('SoftwarePackages'): if type( self.job['SoftwarePackages'] ) == type(''): apps = [self.job['SoftwarePackages']] elif type( self.job['SoftwarePackages'] ) == type([]): apps = self.job['SoftwarePackages'] self.apps = [] for app in apps: DIRAC.gLogger.verbose( 'Requested Package %s' % app ) app = tuple(app.split('.')) if len(app) > 2: tempapp = app app = [] app.append(tempapp[0]) app.append(string.join(tempapp[1:], ".")) self.apps.append(app) self.jobConfig = '' if self.job.has_key( 'SystemConfig' ): self.jobConfig = self.job['SystemConfig'] else: self.jobConfig = natOS.CMTSupportedConfig()[0] self.ceConfigs = [] if self.ce.has_key('CompatiblePlatforms'): self.ceConfigs = self.ce['CompatiblePlatforms'] if type(self.ceConfigs) == type(''): self.ceConfigs = [self.ceConfigs] #else: ### Use always the list of compatible platform. self.ceConfigs = natOS.CMTSupportedConfig() self.sharedArea = SharedArea() DIRAC.gLogger.info("SharedArea is %s" % self.sharedArea) self.localArea = LocalArea() DIRAC.gLogger.info("LocalArea is %s" % self.localArea) def execute(self): """ Main method of the class executed by DIRAC jobAgent Executes the following: - look for the compatible platforms in the CS, see if one matches request - install the applications, calls L{TARsoft} @return: S_OK(), S_ERROR() """ if not self.apps: # There is nothing to do return DIRAC.S_OK() if not self.jobConfig: DIRAC.gLogger.error( 'No architecture requested' ) return DIRAC.S_ERROR( 'No architecture requested' ) found_config = False DIRAC.gLogger.info("Found CE Configs %s, compatible with system reqs %s" % (string.join(self.ceConfigs, ","), self.jobConfig)) res = self.ops.getSections('/AvailableTarBalls') if not res['OK']: return res else: supported_systems = res['Value'] ###look in CS if specified platform has software available. Normally consistency check is done at submission time for ceConfig in self.ceConfigs: for supp_systems in supported_systems: if ceConfig == supp_systems: self.jobConfig = ceConfig found_config = True break if not found_config: if self.ceConfigs: # redundant check as this is done in the job agent, if locally running option might not be defined DIRAC.gLogger.error( 'Requested architecture not supported by CE' ) return DIRAC.S_ERROR( 'Requested architecture not supported by CE' ) else: DIRAC.gLogger.info( 'Assume locally running job, will install software in ' ) areas = [] ###Deal with shared/local area: first try to see if the Shared area exists and if not create it (try to). If it fails, fall back to local area if not self.sharedArea: if CreateSharedArea(): self.sharedArea = SharedArea() if self.sharedArea: areas.append(self.sharedArea) DIRAC.gLogger.info("Will attempt to install in shared area") else: areas.append(self.sharedArea) areas.append(self.localArea) for app in self.apps: failed = False for area in areas: DIRAC.gLogger.info('Attempting to install %s_%s for %s in %s' % (app[0], app[1], self.jobConfig, area)) res = TARinstall(app, self.jobConfig, area) if not res['OK']: DIRAC.gLogger.error('Failed to install software in %s: %s' % (area, res['Message']), '%s_%s' % (app[0], app[1])) failed = True continue else: DIRAC.gLogger.info('%s was successfully installed for %s in %s' % (app, self.jobConfig, area)) failed = False break if failed: return DIRAC.S_ERROR("Failed to install software") if self.sharedArea: #List content listAreaDirectory(self.sharedArea) return DIRAC.S_OK()