def doCommand( self ): """ Returns running and runned jobs, querying the WMSHistory for the last self.args[0] hours :params: :attr:`sites`: list of sites (when not given, take every sites) :returns: """ if not 'hours' in self.args: return S_ERROR( 'Number of hours not specified' ) hours = self.args[ 'hours' ] sites = None if 'sites' in self.args: sites = self.args[ 'sites' ] if sites is None: #FIXME: pointing to the CSHelper instead # sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} ) # if not sources[ 'OK' ]: # return sources # sources = [ si[0] for si in sources[ 'Value' ] ] sites = Resources.getSites() if not sites[ 'OK' ]: return sites sites = sites[ 'Value' ] if not sites: return S_ERROR( 'Sites is empty' ) fromD = datetime.utcnow() - timedelta( hours = hours ) toD = datetime.utcnow() runJobs = self.rClient.getReport( 'WMSHistory', 'NumberOfJobs', fromD, toD, {}, 'Site') if not runJobs[ 'OK' ]: return runJobs runJobs = runJobs[ 'Value' ] if not 'data' in runJobs: return S_ERROR( 'Missing data key' ) if not 'granularity' in runJobs: return S_ERROR( 'Missing granularity key' ) singlePlots = {} for site, value in runJobs[ 'data' ].items(): if site in sites: plot = {} plot[ 'data' ] = { site: value } plot[ 'granularity' ] = runJobs[ 'granularity' ] singlePlots[ site ] = plot return S_OK( singlePlots ) ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def test_ResourcesGetters(): res = Resources.getSites() assert res['OK'] is True, res['Message'] assert res['Value'] == ['DIRAC.Jenkins.ch'], res['Value'] res = Resources.getSiteCEMapping() assert res['OK'] is True, res['Message'] assert res['Value'] == { 'DIRAC.Jenkins.ch': ['jenkins.cern.ch'] }, res['Value'] res = Resources.getCESiteMapping() assert res['OK'] is True, res['Message'] assert res['Value'] == { 'jenkins.cern.ch': 'DIRAC.Jenkins.ch' }, res['Value'] res = Resources.getCESiteMapping('jenkins.cern.ch') assert res['OK'] is True, res['Message'] assert res['Value'] == { 'jenkins.cern.ch': 'DIRAC.Jenkins.ch' }, res['Value'] res = Resources.getCESiteMapping('not-here') assert res['OK'] is True, res['Message'] assert res['Value'] == {}, res['Value']
def test_ResourcesGetters(): res = Resources.getSites() assert res["OK"] is True, res["Message"] assert res["Value"] == ["DIRAC.Jenkins.ch"], res["Value"] res = Resources.getSiteCEMapping() assert res["OK"] is True, res["Message"] assert res["Value"] == { "DIRAC.Jenkins.ch": ["jenkins.cern.ch"] }, res["Value"] res = Resources.getCESiteMapping() assert res["OK"] is True, res["Message"] assert res["Value"] == { "jenkins.cern.ch": "DIRAC.Jenkins.ch" }, res["Value"] res = Resources.getCESiteMapping("jenkins.cern.ch") assert res["OK"] is True, res["Message"] assert res["Value"] == { "jenkins.cern.ch": "DIRAC.Jenkins.ch" }, res["Value"] res = Resources.getCESiteMapping("not-here") assert res["OK"] is True, res["Message"] assert res["Value"] == {}, res["Value"]
def export_getSitesResources( self, siteNames ): resources = Resources.Resources() if siteNames is None: siteNames = Resources.getSites() if not siteNames[ 'OK' ]: return siteNames siteNames = siteNames[ 'Value' ] if isinstance( siteNames, str ): siteNames = [ siteNames ] sitesRes = {} for siteName in siteNames: res = {} res[ 'ces' ] = resources.getEligibleResources( 'Computing', { 'Site': siteName } ) ses = resources.getEligibleStorageElements( { 'Site': siteName } ) sesHosts = CSHelpers.getStorageElementsHosts( ses ) if not sesHosts[ 'OK' ]: return sesHosts res[ 'ses' ] = list( set( sesHosts[ 'Value' ] ) ) sitesRes[ siteName ] = res return S_OK( sitesRes )
def doCommand(self): """ Returns failed jobs using the DIRAC accounting system for every site for the last self.args[0] hours :params: :attr:`sites`: list of sites (when not given, take every site) :returns: """ if not 'hours' in self.args: return S_ERROR('Number of hours not specified') hours = self.args['hours'] sites = None if 'sites' in self.args: sites = self.args['sites'] if sites is None: #FIXME: pointing to the CSHelper instead # sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} ) # if not sources[ 'OK' ]: # return sources # sources = [ si[0] for si in sources[ 'Value' ] ] sites = Resources.getSites() if not sites['OK']: return sites sites = sites['Value'] if not sites: return S_ERROR('Sites is empty') fromD = datetime.utcnow() - timedelta(hours=hours) toD = datetime.utcnow() failedPilots = self.rClient.getReport('Pilot', 'NumberOfPilots', fromD, toD, { 'GridStatus': ['Aborted'], 'Site': sites }, 'Site') if not failedPilots['OK']: return failedPilots failedPilots = failedPilots['Value'] if not 'data' in failedPilots: return S_ERROR('Missing data key') if not 'granularity' in failedPilots: return S_ERROR('Missing granularity key') singlePlots = {} for site, value in failedPilots['data'].items(): if site in sites: plot = {} plot['data'] = {site: value} plot['granularity'] = failedPilots['granularity'] singlePlots[site] = plot return S_OK(singlePlots)
def doCommand( self ): """ Returns failed jobs using the DIRAC accounting system for every site for the last self.args[0] hours :params: :attr:`sites`: list of sites (when not given, take every site) :returns: """ if not 'hours' in self.args: return S_ERROR( 'Number of hours not specified' ) hours = self.args[ 'hours' ] sites = None if 'sites' in self.args: sites = self.args[ 'sites' ] if sites is None: #FIXME: pointing to the CSHelper instead # sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} ) # if not sources[ 'OK' ]: # return sources # sources = [ si[0] for si in sources[ 'Value' ] ] sites = Resources.getSites() if not sites[ 'OK' ]: return sites sites = sites[ 'Value' ] if not sites: return S_ERROR( 'Sites is empty' ) fromD = datetime.utcnow() - timedelta( hours = hours ) toD = datetime.utcnow() failedPilots = self.rClient.getReport( 'Pilot', 'NumberOfPilots', fromD, toD, { 'GridStatus' : [ 'Aborted' ], 'Site' : sites }, 'Site' ) if not failedPilots[ 'OK' ]: return failedPilots failedPilots = failedPilots[ 'Value' ] if not 'data' in failedPilots: return S_ERROR( 'Missing data key' ) if not 'granularity' in failedPilots: return S_ERROR( 'Missing granularity key' ) singlePlots = {} for site, value in failedPilots[ 'data' ].items(): if site in sites: plot = {} plot[ 'data' ] = { site: value } plot[ 'granularity' ] = failedPilots[ 'granularity' ] singlePlots[ site ] = plot return S_OK( singlePlots )
def getGOCSites(diracSites=None): # FIXME: THIS SHOULD GO INTO Resources HELPER if diracSites is None: diracSites = Resources.getSites() if not diracSites["OK"]: return diracSites diracSites = diracSites["Value"] gocSites = [] for diracSite in diracSites: gocSite = getGOCSiteName(diracSite) if not gocSite["OK"]: continue gocSites.append(gocSite["Value"]) return S_OK(list(set(gocSites)))
def getGOCSites(diracSites=None): #FIXME: THIS SHOULD GO INTO Resources HELPER if diracSites is None: diracSites = Resources.getSites() if not diracSites['OK']: return diracSites diracSites = diracSites['Value'] gocSites = [] for diracSite in diracSites: gocSite = getGOCSiteName(diracSite) if not gocSite['OK']: continue gocSites.append(gocSite['Value']) return S_OK(list(set(gocSites)))
def doMaster( self ): ''' Master method, which looks little bit spaguetti code, sorry ! - It gets all Sites. - It gets all StorageElements As there is no bulk query, it compares with what we have on the database. It queries a portion of them. ''' sites = Resources.getSites() if not sites[ 'OK' ]: return sites sites = sites[ 'Value' ] ses = self.resources.getEligibleStorageElements() if not ses[ 'OK' ]: return ses ses = ses[ 'Value' ] elementNames = sites + ses # sourceQuery = self.rmClient.selectTransferCache( meta = { 'columns' : [ 'SourceName' ] } ) # if not sourceQuery[ 'OK' ]: # return sourceQuery # sourceQuery = [ element[0] for element in sourceQuery[ 'Value' ] ] # # sourceElementsToQuery = list( set( elementNames ).difference( set( sourceQuery ) ) ) gLogger.info( 'Processing %s' % ', '.join( elementNames ) ) for metric in [ 'Quality', 'FailedTransfers' ]: for direction in [ 'Source', 'Destination' ]: # 2 hours of window result = self.doNew( ( 2, elementNames, direction, metric ) ) if not result[ 'OK' ]: self.metrics[ 'failed' ].append( result ) return S_OK( self.metrics ) ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def doCommand(self): """ # Returns simple pilots efficiency # # :attr:`args`: # - args[0]: string - should be a ValidElement # # - args[1]: string - should be the name of the ValidElement # # returns: # { # 'Result': 'Good'|'Fair'|'Poor'|'Idle'|'Bad' # } """ if not 'element' in self.args: return self.returnERROR(S_ERROR('element is missing')) element = self.args['element'] if not 'siteName' in self.args: return self.returnERROR(S_ERROR('siteName is missing')) siteName = self.args['siteName'] # If siteName is None, we take all sites if siteName is None: siteName = Resources.getSites() if not siteName['OK']: return self.returnERROR(siteName) siteName = siteName['Value'] if element == 'Site': results = self.wmsAdmin.getPilotSummaryWeb({'GridSite': siteName}, [], 0, 300) elif element == 'Resource': results = self.wmsAdmin.getPilotSummaryWeb( {'ExpandSite': siteName}, [], 0, 300) else: return self.returnERROR(S_ERROR('%s is a wrong element' % element)) if not results['OK']: return self.returnERROR(results) results = results['Value'] if not 'ParameterNames' in results: return self.returnERROR(S_ERROR('Malformed result dictionary')) params = results['ParameterNames'] if not 'Records' in results: return self.returnERROR(S_ERROR('Malformed result dictionary')) records = results['Records'] pilotResults = [] for record in records: pilotDict = dict(zip(params, record)) try: pilotDict['PilotsPerJob'] = float(pilotDict['PilotsPerJob']) pilotDict['PilotsJobEff'] = float(pilotDict['PilotsJobEff']) except KeyError, e: return self.returnERROR(S_ERROR(e)) except ValueError, e: return self.returnERROR(S_ERROR(e))
def doCommand(self): """ Returns running and runned jobs, querying the WMSHistory for the last self.args[0] hours :params: :attr:`sites`: list of sites (when not given, take every sites) :returns: """ if not 'hours' in self.args: return S_ERROR('Number of hours not specified') hours = self.args['hours'] sites = None if 'sites' in self.args: sites = self.args['sites'] if sites is None: #FIXME: pointing to the CSHelper instead # sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} ) # if not sources[ 'OK' ]: # return sources # sources = [ si[0] for si in sources[ 'Value' ] ] sites = Resources.getSites() if not sites['OK']: return sites sites = sites['Value'] if not sites: return S_ERROR('Sites is empty') fromD = datetime.utcnow() - timedelta(hours=hours) toD = datetime.utcnow() runJobs = self.rClient.getReport('WMSHistory', 'NumberOfJobs', fromD, toD, {}, 'Site') if not runJobs['OK']: return runJobs runJobs = runJobs['Value'] if not 'data' in runJobs: return S_ERROR('Missing data key') if not 'granularity' in runJobs: return S_ERROR('Missing granularity key') singlePlots = {} for site, value in runJobs['data'].items(): if site in sites: plot = {} plot['data'] = {site: value} plot['granularity'] = runJobs['granularity'] singlePlots[site] = plot return S_OK(singlePlots) ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def export_getSites( self ): ''' Returns list of all sites considered by RSS ''' gLogger.info( 'getSites' ) return Resources.getSites()
def doCommand( self ): """ # Returns simple pilots efficiency # # :attr:`args`: # - args[0]: string - should be a ValidElement # # - args[1]: string - should be the name of the ValidElement # # returns: # { # 'Result': 'Good'|'Fair'|'Poor'|'Idle'|'Bad' # } """ if not 'element' in self.args: return self.returnERROR( S_ERROR( 'element is missing' ) ) element = self.args[ 'element' ] if not 'siteName' in self.args: return self.returnERROR( S_ERROR( 'siteName is missing' ) ) siteName = self.args[ 'siteName' ] # If siteName is None, we take all sites if siteName is None: siteName = Resources.getSites() if not siteName[ 'OK' ]: return self.returnERROR( siteName ) siteName = siteName[ 'Value' ] if element == 'Site': results = self.wmsAdmin.getPilotSummaryWeb( { 'GridSite' : siteName }, [], 0, 300 ) elif element == 'Resource': results = self.wmsAdmin.getPilotSummaryWeb( { 'ExpandSite' : siteName }, [], 0, 300 ) else: return self.returnERROR( S_ERROR( '%s is a wrong element' % element ) ) if not results[ 'OK' ]: return self.returnERROR( results ) results = results[ 'Value' ] if not 'ParameterNames' in results: return self.returnERROR( S_ERROR( 'Malformed result dictionary' ) ) params = results[ 'ParameterNames' ] if not 'Records' in results: return self.returnERROR( S_ERROR( 'Malformed result dictionary' ) ) records = results[ 'Records' ] pilotResults = [] for record in records: pilotDict = dict( zip( params , record )) try: pilotDict[ 'PilotsPerJob' ] = float( pilotDict[ 'PilotsPerJob' ] ) pilotDict[ 'PilotsJobEff' ] = float( pilotDict[ 'PilotsJobEff' ] ) except KeyError, e: return self.returnERROR( S_ERROR( e ) ) except ValueError, e: return self.returnERROR( S_ERROR( e ) )