def export_getSiteSummarySelectors(cls): """ Get all the distinct selector values for the site summary web portal page :return: S_OK(dict)/S_ERROR() """ resultDict = {} statusList = ['Good', 'Fair', 'Poor', 'Bad', 'Idle'] resultDict['Status'] = statusList maskStatus = ['Active', 'Banned', 'NoMask', 'Reduced'] resultDict['MaskStatus'] = maskStatus res = getSites() if not res['OK']: return res siteList = res['Value'] countryList = [] for site in siteList: if site.find('.') != -1: country = site.split('.')[2].lower() if country not in countryList: countryList.append(country) countryList.sort() resultDict['Country'] = countryList siteList.sort() resultDict['Site'] = siteList return S_OK(resultDict)
def doMaster(self): ''' Master method, which looks little bit spaguetti code, sorry ! - It gets all Sites. - It gets all StorageElements As there is no bulk query, it compares with what we have on the database. It queries a portion of them. ''' sites = getSites() if not sites['OK']: return sites sites = sites['Value'] elementNames = sites + DMSHelpers().getStorageElements() # sourceQuery = self.rmClient.selectTransferCache( meta = { 'columns' : [ 'SourceName' ] } ) # if not sourceQuery[ 'OK' ]: # return sourceQuery # sourceQuery = [ element[0] for element in sourceQuery[ 'Value' ] ] # # sourceElementsToQuery = list( set( elementNames ).difference( set( sourceQuery ) ) ) self.log.info('Processing %s' % ', '.join(elementNames)) for metric in ['Quality', 'FailedTransfers']: for direction in ['Source', 'Destination']: # 2 hours of window result = self.doNew((2, elementNames, direction, metric)) if not result['OK']: self.metrics['failed'].append(result) return S_OK(self.metrics)
def doCommand(self): """ Returns failed jobs using the DIRAC accounting system for every site for the last self.args[0] hours :params: :attr:`sites`: list of sites (when not given, take every site) :returns: """ if 'hours' not in self.args: return S_ERROR('Number of hours not specified') hours = self.args['hours'] sites = None if 'sites' in self.args: sites = self.args['sites'] if sites is None: # FIXME: pointing to the CSHelper instead # sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} ) # if not sources[ 'OK' ]: # return sources # sources = [ si[0] for si in sources[ 'Value' ] ] sites = getSites() if not sites['OK']: return sites sites = sites['Value'] if not sites: return S_ERROR('Sites is empty') fromD = datetime.utcnow() - timedelta(hours=hours) toD = datetime.utcnow() failedPilots = self.rClient.getReport('Pilot', 'NumberOfPilots', fromD, toD, { 'GridStatus': ['Aborted'], 'Site': sites }, 'Site') if not failedPilots['OK']: return failedPilots failedPilots = failedPilots['Value'] if 'data' not in failedPilots: return S_ERROR('Missing data key') if 'granularity' not in failedPilots: return S_ERROR('Missing granularity key') singlePlots = {} for site, value in failedPilots['data'].iteritems(): if site in sites: plot = {} plot['data'] = {site: value} plot['granularity'] = failedPilots['granularity'] singlePlots[site] = plot return S_OK(singlePlots)
def export_getSiteSummarySelectors(cls): """Get all the distinct selector values for the site summary web portal page :return: S_OK(dict)/S_ERROR() """ resultDict = {} statusList = ["Good", "Fair", "Poor", "Bad", "Idle"] resultDict["Status"] = statusList maskStatus = ["Active", "Banned", "NoMask", "Reduced"] resultDict["MaskStatus"] = maskStatus res = getSites() if not res["OK"]: return res siteList = res["Value"] countryList = [] for site in siteList: if site.find(".") != -1: country = site.split(".")[2].lower() if country not in countryList: countryList.append(country) countryList.sort() resultDict["Country"] = countryList siteList.sort() resultDict["Site"] = siteList return S_OK(resultDict)
def doCommand(self): """ Returns successfull jobs using the DIRAC accounting system for every site for the last self.args[0] hours :params: :attr:`sites`: list of sites (when not given, take every site) :returns: """ if 'hours' not in self.args: return S_ERROR('Number of hours not specified') hours = self.args['hours'] sites = None if 'sites' in self.args: sites = self.args['sites'] if sites is None: # FIXME: pointing to the CSHelper instead # sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} ) # if not sources[ 'OK' ]: # return sources # sources = [ si[0] for si in sources[ 'Value' ] ] sites = getSites() if not sites['OK']: return sites sites = sites['Value'] if not sites: return S_ERROR('Sites is empty') fromD = datetime.utcnow() - timedelta(hours=hours) toD = datetime.utcnow() successfulJobs = self.rClient.getReport('Job', 'NumberOfJobs', fromD, toD, {'FinalStatus': ['Done'], 'Site': sites }, 'Site') if not successfulJobs['OK']: return successfulJobs successfulJobs = successfulJobs['Value'] if 'data' not in successfulJobs: return S_ERROR('Missing data key') if 'granularity' not in successfulJobs: return S_ERROR('Missing granularity key') singlePlots = {} for site, value in successfulJobs['data'].items(): if site in sites: plot = {} plot['data'] = {site: value} plot['granularity'] = successfulJobs['granularity'] singlePlots[site] = plot return S_OK(singlePlots)
def __checkSiteIsValid( self, site ): """Internal function to check that a site name is valid. """ result = getSites() if not result['OK']: return S_ERROR( 'Could not get site CE mapping' ) siteList = result['Value'] if site in siteList: return S_OK( '%s is valid' % site ) result = getSites( fullName = True ) if not result['OK']: return S_ERROR( 'Could not get site CE mapping' ) siteList = result['Value'] if site in siteList: return S_OK( '%s is valid' % site ) return S_ERROR( 'Specified site %s is not in list of defined sites' % site )
def doCommand(self): """ Returns running and runned jobs, querying the WMSHistory for the last self.args[0] hours :params: :attr:`sites`: list of sites (when not given, take every sites) :returns: """ if 'hours' not in self.args: return S_ERROR('Number of hours not specified') hours = self.args['hours'] sites = None if 'sites' in self.args: sites = self.args['sites'] if sites is None: # FIXME: pointing to the CSHelper instead # sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} ) # if not sources[ 'OK' ]: # return sources # sources = [ si[0] for si in sources[ 'Value' ] ] sites = getSites() if not sites['OK']: return sites sites = sites['Value'] if not sites: return S_ERROR('Sites is empty') fromD = datetime.utcnow() - timedelta(hours=hours) toD = datetime.utcnow() runJobs = self.rClient.getReport('WMSHistory', 'NumberOfJobs', fromD, toD, {}, 'Site') if not runJobs['OK']: return runJobs runJobs = runJobs['Value'] if 'data' not in runJobs: return S_ERROR('Missing data key') if 'granularity' not in runJobs: return S_ERROR('Missing granularity key') singlePlots = {} for site, value in runJobs['data'].items(): if site in sites: plot = {} plot['data'] = {site: value} plot['granularity'] = runJobs['granularity'] singlePlots[site] = plot return S_OK(singlePlots)
def export_getSites(self): """ Returns list of all sites considered by RSS :return: S_OK( [ sites ] ) | S_ERROR """ gLogger.info('getSites') return getSites()
def doCommand(self): """ Returns simple jobs efficiency :param args: - args[0]: string: should be a ValidElement - args[1]: string should be the name of the ValidElement :returns: { 'Result': 'Good'|'Fair'|'Poor'|'Idle'|'Bad' } """ if 'siteName' not in self.args: return self.returnERROR(S_ERROR('siteName is missing')) siteName = self.args['siteName'] # If siteName is None, we take all sites if siteName is None: siteName = getSites() if not siteName['OK']: return self.returnERROR(siteName) siteName = siteName['Value'] results = self.wmsAdmin.getSiteSummaryWeb({'Site': siteName}, [], 0, 500) if not results['OK']: return self.returnERROR(results) results = results['Value'] if 'ParameterNames' not in results: return self.returnERROR(S_ERROR('Malformed result dictionary')) params = results['ParameterNames'] if 'Records' not in results: return self.returnERROR(S_ERROR('Malformed result dictionary')) records = results['Records'] jobResults = [] for record in records: jobDict = dict(zip(params, record)) try: jobDict['Efficiency'] = float(jobDict['Efficiency']) except KeyError as e: return self.returnERROR(S_ERROR(e)) except ValueError as e: return self.returnERROR(S_ERROR(e)) jobResults.append(jobDict) return S_OK(jobResults)
def __query( self, queryType, parameters ): ''' This method is a rather important one. It will format the input for the DB queries, instead of doing it on a decorator. Two dictionaries must be passed to the DB. First one contains 'columnName' : value pairs, being the key lower camel case. The second one must have, at lease, a key named 'table' with the right table name. ''' # Functions we can call, just a light safety measure. _gateFunctions = [ 'insert', 'update', 'select', 'delete', 'addOrModify', 'modify', 'addIfNotThere' ] if not queryType in _gateFunctions: return S_ERROR( '"%s" is not a proper gate call' % queryType ) gateFunction = getattr( self.gate, queryType ) # If meta is None, we set it to {} meta = ( True and parameters.pop( 'meta' ) ) or {} # Remove self, added by locals() del parameters[ 'self' ] # This is an special case with the Element tables. #if tableName.startswith( 'Element' ): element = parameters.pop( 'element' ) if not element in self.validElements: gLogger.debug( '"%s" is not a valid element like %s' % ( element, self.validElements ) ) return S_ERROR( '"%s" is not a valid element like %s' % ( element, self.validElements ) ) # For Site elements always use the short names if element == "Site" and parameters['name'] is not None: if type( parameters['name'] ) in StringTypes: parameters['name'] = [parameters['name']] if type( parameters['name'] ) == ListType: result = getSites( parameters['name'] ) if not result['OK']: gLogger.debug( result['Message'] ) return result parameters['name'] = result['Value'] else: gLogger.debug( 'Invalid site name type: %s' % type( parameters['name'] ) ) return S_ERROR( 'Invalid site name type: %s' % type( parameters['name'] ) ) tableType = parameters.pop( 'tableType' ) #tableName = tableName.replace( 'Element', element ) tableName = '%s%s' % ( element, tableType ) meta[ 'table' ] = tableName gLogger.debug( 'Calling %s, with \n params %s \n meta %s' % ( queryType, parameters, meta ) ) userRes = gateFunction( parameters, meta ) return userRes
def _syncSites(self): ''' Sync sites: compares CS with DB and does the necessary modifications. ''' gLogger.info('-- Synchronizing sites --') # sites in CS res = getSites() if not res['OK']: return res sitesCS = res['Value'] gLogger.verbose('%s sites found in CS' % len(sitesCS)) # sites in RSS result = self.rStatus.selectStatusElement('Site', 'Status', meta={'columns': ['Name']}) if not result['OK']: return result sitesDB = [siteDB[0] for siteDB in result['Value']] # Sites that are in DB but not (anymore) in CS toBeDeleted = list(set(sitesDB).difference(set(sitesCS))) gLogger.verbose('%s sites to be deleted' % len(toBeDeleted)) # Delete sites for siteName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Site', siteName) gLogger.verbose('Deleting site %s' % siteName) if not deleteQuery['OK']: return deleteQuery # Sites that are in CS but not (anymore) in DB toBeAdded = list(set(sitesCS).difference(set(sitesDB))) gLogger.verbose('%s site entries to be added' % len(toBeAdded)) for site in toBeAdded: query = self.rStatus.addIfNotThereStatusElement('Site', 'Status', name=site, statusType='all', status=self.defaultStatus, elementType='Site', tokenOwner=self.tokenOwner, reason='Synchronized') if not query['OK']: return query return S_OK()
def __init__(self, script=None, stdout='std.out', stderr='std.err'): """Instantiates the Workflow object and some default parameters. """ super(Job, self).__init__() self.stepCount = 0 self.owner = 'NotSpecified' self.name = 'Name' self.type = 'User' self.priority = 1 vo = '' ret = getProxyInfo(disableVOMS=True) if ret['OK'] and 'group' in ret['Value']: vo = getVOForGroup(ret['Value']['group']) self.group = vo self.site = None # self.setup = 'Development' self.origin = 'DIRAC' self.stdout = stdout self.stderr = stderr self.logLevel = 'info' self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified # $DIRACROOT is set by the JobWrapper at execution time self.addToInputSandbox = [] self.addToOutputSandbox = [] self.addToInputData = [] # #Add member to handle Parametric jobs self.numberOfParameters = 0 self.parameterSeqs = {} self.wfArguments = {} self.parametricWFArguments = {} # loading the function that will be used to determine the platform (it can be VO specific) res = ObjectLoader().loadObject("ConfigurationSystem.Client.Helpers.Resources", 'getDIRACPlatforms') if not res['OK']: self.log.fatal(res['Message']) self.getDIRACPlatforms = res['Value'] self.script = script if not script: self.workflow = Workflow() self.__setJobDefaults() else: self.workflow = Workflow(script) self._siteSet = set(getSites().get('Value', []))
def doMaster(self): """ Master method. Gets all sites and calls doNew method. """ siteNames = getSites() if not siteNames['OK']: return siteNames siteNames = siteNames['Value'] jobsResults = self.doNew(siteNames) if not jobsResults['OK']: self.metrics['failed'].append(jobsResults['Message']) return S_OK(self.metrics)
def ftsSites(self): """ get fts site list """ sites = getSites() if not sites["OK"]: return sites sites = sites["Value"] ftsServers = getFTSServersForSites(sites) if not ftsServers["OK"]: return ftsServers ftsServers = ftsServers["Value"] ftsSites = [] for site, ftsServerURL in ftsServers.items(): ftsSite = FTSSite() ftsSite.Name, ftsSite.FTSServer = site, ftsServerURL ## should be read from CS as well ftsSite.MaxActiveJobs = 50 ftsSites.append(ftsSite) return S_OK(ftsSites)
def ftsSites( self ): """ get fts site list """ sites = getSites() if not sites["OK"]: return sites sites = sites["Value"] ftsServers = getFTSServersForSites( sites ) if not ftsServers["OK"]: return ftsServers ftsServers = ftsServers["Value"] ftsSites = [] for site, ftsServerURL in ftsServers.items(): ftsSite = FTSSite() ftsSite.Name, ftsSite.FTSServer = site, ftsServerURL ## should be read from CS as well ftsSite.MaxActiveJobs = 50 ftsSites.append( ftsSite ) return S_OK( ftsSites )
def export_getSiteMaskSummary(cls): """ Get the mask status for all the configured sites :return: S_OK(dict)/S_ERROR() """ # Get all the configured site names res = getSites() if not res['OK']: return res sites = res['Value'] # Get the current mask status result = cls.jobDB.getSiteMaskStatus() siteDict = result['Value'] for site in sites: if site not in siteDict: siteDict[site] = 'Unknown' return S_OK(siteDict)
def doMaster(self): siteNames = getSites() if not siteNames['OK']: return siteNames siteNames = siteNames['Value'] ces = CSHelpers.getComputingElements() if not ces['OK']: return ces ces = ces['Value'] pilotResults = self.doNew(('Site', siteNames)) if not pilotResults['OK']: self.metrics['failed'].append(pilotResults['Message']) pilotResults = self.doNew(('Resource', ces)) if not pilotResults['OK']: self.metrics['failed'].append(pilotResults['Message']) return S_OK(self.metrics)
def export_getSitesResources(self, siteNames): """ Returns dictionary with SEs and CEs for the given site(s). If siteNames is None, all sites are taken into account. :return: S_OK( { site1 : { ces : [ ces ], 'ses' : [ ses ] },... } ) | S_ERROR """ gLogger.info('getSitesResources') if siteNames is None: siteNames = getSites() if not siteNames['OK']: return siteNames siteNames = siteNames['Value'] if isinstance(siteNames, basestring): siteNames = [siteNames] sitesRes = {} for siteName in siteNames: res = {} res['ces'] = CSHelpers.getSiteComputingElements(siteName) # Convert StorageElements to host names res = DMSHelpers().getSiteSEMapping() if not res['OK']: return res ses = res['Value'][1].get(siteName, []) sesHosts = CSHelpers.getStorageElementsHosts(ses) if not sesHosts['OK']: return sesHosts # Remove duplicates res['ses'] = list(set(sesHosts['Value'])) sitesRes[siteName] = res return S_OK(sitesRes)
def _ByJobType( self ): """ By default, all sites are allowed to do every job. The actual rules are freely specified in the Operation JobTypeMapping section. The content of the section may look like this: User { Exclude = PAK Exclude += Ferrara Exclude += Bologna Exclude += Paris Exclude += CERN Exclude += IN2P3 Allow { Paris = IN2P3 CERN = CERN IN2P3 = IN2P3 } } DataReconstruction { Exclude = PAK Exclude += Ferrara Exclude += CERN Exclude += IN2P3 Allow { Ferrara = CERN CERN = CERN IN2P3 = IN2P3 IN2P3 += CERN } } Merge { Exclude = ALL Allow { CERN = CERN IN2P3 = IN2P3 } } The sites in the exclusion list will be removed. The allow section says where each site may help another site """ # 1. get sites list res = getSites() if not res['OK']: gLogger.error( "Could not get the list of sites", res['Message'] ) return res destSites = set( res['Value'] ) # 2. get JobTypeMapping "Exclude" value (and add autoAddedSites) gLogger.debug( "Getting JobTypeMapping 'Exclude' value (and add autoAddedSites)" ) jobType = self.params['JobType'] if not jobType: raise RuntimeError( "No jobType specified" ) excludedSites = self.opsH.getValue( 'JobTypeMapping/%s/Exclude' % jobType, [] ) gLogger.debug( "Explicitly excluded sites for %s task: %s" % ( jobType, ','.join( excludedSites ) ) ) excludedSites += self.opsH.getValue( 'JobTypeMapping/AutoAddedSites', [] ) gLogger.debug( "Full list of excluded sites for %s task: %s" % ( jobType, ','.join( excludedSites ) ) ) # 3. removing sites in Exclude if not excludedSites: pass elif 'ALL' in excludedSites: destSites = set() else: destSites = destSites.difference( set( excludedSites ) ) # 4. get JobTypeMapping "Allow" section res = self.opsH.getOptionsDict( 'JobTypeMapping/%s/Allow' % jobType ) if not res['OK']: gLogger.verbose( res['Message'] ) allowed = {} else: allowed = res['Value'] for site in allowed: allowed[site] = fromChar( allowed[site] ) # 5. add autoAddedSites, if requested autoAddedSites = self.opsH.getValue( 'JobTypeMapping/AutoAddedSites', [] ) if autoAddedSites: for autoAddedSite in autoAddedSites: allowed.setdefault( autoAddedSite, [autoAddedSite] ) if autoAddedSite not in allowed: allowed[autoAddedSite] = [autoAddedSite] else: allowed[autoAddedSite] = [autoAddedSite] + allowed[autoAddedSite] gLogger.debug( "Allowed sites for %s task: %s" % ( jobType, ','.join( allowed ) ) ) # 6. Allowing sites that should be allowed if not self.params['TargetSE'] or self.params['TargetSE'] == 'Unknown': gLogger.warn( "TargetSE is not set: the destination sites list will be incomplete" ) taskSiteDestination = self._BySE() for destSite, fromSites in allowed.iteritems(): for fromSite in fromSites: if taskSiteDestination: if fromSite in taskSiteDestination: destSites.add( destSite ) else: destSites.add( destSite ) gLogger.verbose( "Computed list of destination sites for %s task with TargetSE %s: %s" % ( jobType, self.params['TargetSE'], ','.join( destSites ) ) ) return destSites
def _ByJobType(self): """ By default, all sites are allowed to do every job. The actual rules are freely specified in the Operation JobTypeMapping section. The content of the section may look like this: User { Exclude = PAK Exclude += Ferrara Exclude += Bologna Exclude += Paris Exclude += CERN Exclude += IN2P3 Allow { Paris = IN2P3 CERN = CERN IN2P3 = IN2P3 } } DataReconstruction { Exclude = PAK Exclude += Ferrara Exclude += CERN Exclude += IN2P3 Allow { Ferrara = CERN CERN = CERN IN2P3 = IN2P3 IN2P3 += CERN } } Merge { Exclude = ALL Allow { CERN = CERN IN2P3 = IN2P3 } } The sites in the exclusion list will be removed. The allow section says where each site may help another site """ # 1. get sites list res = getSites() if not res['OK']: gLogger.error("Could not get the list of sites", res['Message']) return res destSites = set(res['Value']) # 2. get JobTypeMapping "Exclude" value (and add autoAddedSites) gLogger.debug("Getting JobTypeMapping 'Exclude' value (and add autoAddedSites)") jobType = self.params['JobType'] if not jobType: raise RuntimeError("No jobType specified") excludedSites = set(self.opsH.getValue('JobTypeMapping/%s/Exclude' % jobType, [])) gLogger.debug("Explicitly excluded sites for %s task: %s" % (jobType, ','.join(excludedSites))) autoAddedSites = self.opsH.getValue('JobTypeMapping/AutoAddedSites', []) if 'WithStorage' in autoAddedSites: # Add all sites with storage, such that jobs can run wherever data is autoAddedSites.remove('WithStorage') autoAddedSites += DMSHelpers().getTiers(withStorage=True, tier=(0, 1, 2)) # 3. removing sites in Exclude if not excludedSites: pass elif 'ALL' in excludedSites: destSites = set() else: destSites -= excludedSites # 4. get JobTypeMapping "Allow" section res = self.opsH.getOptionsDict('JobTypeMapping/%s/Allow' % jobType) if not res['OK']: gLogger.debug(res['Message']) allowed = {} else: allowed = dict((site, set(fromChar(fromSites))) for site, fromSites in res['Value'].iteritems()) autoAddedSites = set(self.opsH.getValue('JobTypeMapping/%s/AutoAddedSites' % jobType, autoAddedSites)) gLogger.debug("Auto-added sites for %s task: %s" % (jobType, ','.join(autoAddedSites))) # 5. add autoAddedSites, if requested for autoAddedSite in autoAddedSites: allowed.setdefault(autoAddedSite, set()).add(autoAddedSite) gLogger.debug("Allowed sites for %s task: %s" % (jobType, ','.join(allowed))) # 6. Allowing sites that should be allowed taskSiteDestination = self._BySE() for destSite, fromSites in allowed.iteritems(): for fromSite in fromSites: if not taskSiteDestination or fromSite in taskSiteDestination: destSites.add(destSite) gLogger.debug("Computed list of destination sites for %s task with TargetSE %s: %s" % (jobType, self.params['TargetSE'], ','.join(destSites))) return destSites
DIRAC.exit( -1 ) ftsClient = FTSClient() ftsSites = ftsClient.getFTSSitesList() if not ftsSites["OK"]: gLogger.error( "unable to read FTSSites: %s" % ftsSites["Message"] ) DIRAC.exit( -1 ) ftsSites = ftsSites["Value"] for site in ftsSites: if site.Name == ftsSite: gLogger.error( "FTSSite '%s' is present in FTSDB!!!" % ftsSite ) DIRAC.exit( -1 ) getSites = getSites() if not getSites["OK"]: gLogger.error( "unable to read sites defined in CS!!!" ) DIRAC.exit( -1 ) getSites = getSites["Value"] if "LCG." + ftsSite not in getSites: gLogger.error( "Site '%s' is not defined in CS Resources/Sites section !!!" % ( "LCG.%s" % ftsSite ) ) DIRAC.exit( -1 ) SEs = gConfig.getOption( "/Resources/Sites/LCG/LCG.%s/SE" % ftsSite , [] ) if not SEs["OK"]: gLogger.error( "unable to read SEs attached to site LCG.%s: %s" % ftsSite ) DIRAC.exit( -1 ) SEs = SEs["Value"]