def doCommand(self): """ Returns failed jobs using the DIRAC accounting system for every site for the last self.args[0] hours :params: :attr:`sites`: list of sites (when not given, take every site) :returns: """ if 'hours' not in self.args: return S_ERROR('Number of hours not specified') hours = self.args['hours'] sites = None if 'sites' in self.args: sites = self.args['sites'] if sites is None: #FIXME: pointing to the CSHelper instead # sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} ) # if not sources[ 'OK' ]: # return sources # sources = [ si[0] for si in sources[ 'Value' ] ] sites = CSHelpers.getSites() if not sites['OK']: return sites sites = sites['Value'] if not sites: return S_ERROR('Sites is empty') fromD = datetime.utcnow() - timedelta(hours=hours) toD = datetime.utcnow() failedPilots = self.rClient.getReport('Pilot', 'NumberOfPilots', fromD, toD, { 'GridStatus': ['Aborted'], 'Site': sites }, 'Site') if not failedPilots['OK']: return failedPilots failedPilots = failedPilots['Value'] if not 'data' in failedPilots: return S_ERROR('Missing data key') if not 'granularity' in failedPilots: return S_ERROR('Missing granularity key') singlePlots = {} for site, value in failedPilots['data'].items(): if site in sites: plot = {} plot['data'] = {site: value} plot['granularity'] = failedPilots['granularity'] singlePlots[site] = plot return S_OK(singlePlots)
def doCommand( self ): """ Returns running and runned jobs, querying the WMSHistory for the last self.args[0] hours :params: :attr:`sites`: list of sites (when not given, take every sites) :returns: """ if not 'hours' in self.args: return S_ERROR( 'Number of hours not specified' ) hours = self.args[ 'hours' ] sites = None if 'sites' in self.args: sites = self.args[ 'sites' ] if sites is None: #FIXME: pointing to the CSHelper instead # sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} ) # if not sources[ 'OK' ]: # return sources # sources = [ si[0] for si in sources[ 'Value' ] ] sites = CSHelpers.getSites() if not sites[ 'OK' ]: return sites sites = sites[ 'Value' ] if not sites: return S_ERROR( 'Sites is empty' ) fromD = datetime.utcnow() - timedelta( hours = hours ) toD = datetime.utcnow() runJobs = self.rClient.getReport( 'WMSHistory', 'NumberOfJobs', fromD, toD, {}, 'Site') if not runJobs[ 'OK' ]: return runJobs runJobs = runJobs[ 'Value' ] if not 'data' in runJobs: return S_ERROR( 'Missing data key' ) if not 'granularity' in runJobs: return S_ERROR( 'Missing granularity key' ) singlePlots = {} for site, value in runJobs[ 'data' ].items(): if site in sites: plot = {} plot[ 'data' ] = { site: value } plot[ 'granularity' ] = runJobs[ 'granularity' ] singlePlots[ site ] = plot return S_OK( singlePlots ) ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def doCommand( self ): """ Returns failed jobs using the DIRAC accounting system for every site for the last self.args[0] hours :params: :attr:`sites`: list of sites (when not given, take every site) :returns: """ if 'hours' not in self.args: return S_ERROR( 'Number of hours not specified' ) hours = self.args[ 'hours' ] sites = None if 'sites' in self.args: sites = self.args[ 'sites' ] if sites is None: #FIXME: pointing to the CSHelper instead # sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} ) # if not sources[ 'OK' ]: # return sources # sources = [ si[0] for si in sources[ 'Value' ] ] sites = CSHelpers.getSites() if not sites[ 'OK' ]: return sites sites = sites[ 'Value' ] if not sites: return S_ERROR( 'Sites is empty' ) fromD = datetime.utcnow() - timedelta( hours = hours ) toD = datetime.utcnow() failedPilots = self.rClient.getReport( 'Pilot', 'NumberOfPilots', fromD, toD, { 'GridStatus' : [ 'Aborted' ], 'Site' : sites }, 'Site' ) if not failedPilots[ 'OK' ]: return failedPilots failedPilots = failedPilots[ 'Value' ] if not 'data' in failedPilots: return S_ERROR( 'Missing data key' ) if not 'granularity' in failedPilots: return S_ERROR( 'Missing granularity key' ) singlePlots = {} for site, value in failedPilots[ 'data' ].items(): if site in sites: plot = {} plot[ 'data' ] = { site: value } plot[ 'granularity' ] = failedPilots[ 'granularity' ] singlePlots[ site ] = plot return S_OK( singlePlots )
def export_getSites(self): """ Returns list of all sites considered by RSS :return: S_OK( [ sites ] ) | S_ERROR """ gLogger.info('getSites') return CSHelpers.getSites()
def doCommand(self): """ Returns running and runned jobs, querying the WMSHistory for the last self.args[0] hours :params: :attr:`sites`: list of sites (when not given, take every sites) :returns: """ if 'hours' not in self.args: return S_ERROR('Number of hours not specified') hours = self.args['hours'] sites = None if 'sites' in self.args: sites = self.args['sites'] if sites is None: #FIXME: pointing to the CSHelper instead # sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} ) # if not sources[ 'OK' ]: # return sources # sources = [ si[0] for si in sources[ 'Value' ] ] sites = CSHelpers.getSites() if not sites['OK']: return sites sites = sites['Value'] if not sites: return S_ERROR('Sites is empty') fromD = datetime.utcnow() - timedelta(hours=hours) toD = datetime.utcnow() runJobs = self.rClient.getReport('WMSHistory', 'NumberOfJobs', fromD, toD, {}, 'Site') if not runJobs['OK']: return runJobs runJobs = runJobs['Value'] if not 'data' in runJobs: return S_ERROR('Missing data key') if not 'granularity' in runJobs: return S_ERROR('Missing granularity key') singlePlots = {} for site, value in runJobs['data'].items(): if site in sites: plot = {} plot['data'] = {site: value} plot['granularity'] = runJobs['granularity'] singlePlots[site] = plot return S_OK(singlePlots)
def doCommand(self): """ Returns simple jobs efficiency :attr:`args`: - args[0]: string: should be a ValidElement - args[1]: string should be the name of the ValidElement returns: { 'Result': 'Good'|'Fair'|'Poor'|'Idle'|'Bad' } """ if not 'siteName' in self.args: return self.returnERROR(S_ERROR('siteName is missing')) siteName = self.args['siteName'] # If siteName is None, we take all sites if siteName is None: siteName = CSHelpers.getSites() if not siteName['OK']: return self.returnERROR(siteName) siteName = siteName['Value'] results = self.wmsAdmin.getSiteSummaryWeb({'Site': siteName}, [], 0, 500) if not results['OK']: return self.returnERROR(results) results = results['Value'] if not 'ParameterNames' in results: return self.returnERROR(S_ERROR('Malformed result dictionary')) params = results['ParameterNames'] if not 'Records' in results: return self.returnERROR(S_ERROR('Malformed result dictionary')) records = results['Records'] jobResults = [] for record in records: jobDict = dict(zip(params, record)) try: jobDict['Efficiency'] = float(jobDict['Efficiency']) except KeyError, e: return self.returnERROR(S_ERROR(e)) except ValueError, e: return self.returnERROR(S_ERROR(e))
def doCommand( self ): """ Returns simple jobs efficiency :attr:`args`: - args[0]: string: should be a ValidElement - args[1]: string should be the name of the ValidElement returns: { 'Result': 'Good'|'Fair'|'Poor'|'Idle'|'Bad' } """ if not 'siteName' in self.args: return self.returnERROR( S_ERROR( 'siteName is missing' ) ) siteName = self.args[ 'siteName' ] # If siteName is None, we take all sites if siteName is None: siteName = CSHelpers.getSites() if not siteName[ 'OK' ]: return self.returnERROR( siteName ) siteName = siteName[ 'Value' ] results = self.wmsAdmin.getSiteSummaryWeb( { 'Site' : siteName }, [], 0, 500 ) if not results[ 'OK' ]: return self.returnERROR( results ) results = results[ 'Value' ] if not 'ParameterNames' in results: return self.returnERROR( S_ERROR( 'Malformed result dictionary' ) ) params = results[ 'ParameterNames' ] if not 'Records' in results: return self.returnERROR( S_ERROR( 'Malformed result dictionary' ) ) records = results[ 'Records' ] jobResults = [] for record in records: jobDict = dict( zip( params , record )) try: jobDict[ 'Efficiency' ] = float( jobDict[ 'Efficiency' ] ) except KeyError, e: return self.returnERROR( S_ERROR( e ) ) except ValueError, e: return self.returnERROR( S_ERROR( e ) )
def _syncSites(self): ''' Sync sites: compares CS with DB and does the necessary modifications. ''' gLogger.info('-- Synchronizing sites --') # sites in CS res = CSHelpers.getSites() if not res['OK']: return res sitesCS = res['Value'] gLogger.verbose('%s sites found in CS' % len(sitesCS)) # sites in RSS result = self.rStatus.selectStatusElement('Site', 'Status', meta={'columns': ['Name']}) if not result['OK']: return result sitesDB = [siteDB[0] for siteDB in result['Value']] # Sites that are in DB but not (anymore) in CS toBeDeleted = list(set(sitesDB).difference(set(sitesCS))) gLogger.verbose('%s sites to be deleted' % len(toBeDeleted)) # Delete sites for siteName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Site', siteName) gLogger.verbose('Deleting site %s' % siteName) if not deleteQuery['OK']: return deleteQuery # Sites that are in CS but not (anymore) in DB toBeAdded = list(set(sitesCS).difference(set(sitesDB))) gLogger.verbose('%s site entries to be added' % len(toBeAdded)) for site in toBeAdded: query = self.rStatus.addIfNotThereStatusElement('Site', 'Status', name=site, statusType='all', status=self.defaultStatus, elementType='Site', tokenOwner=self.tokenOwner, reason='Synchronized') if not query['OK']: return query return S_OK()
def doMaster(self): """ Master method. Gets all sites and calls doNew method. """ siteNames = CSHelpers.getSites() if not siteNames['OK']: return siteNames siteNames = siteNames['Value'] jobsResults = self.doNew(siteNames) if not jobsResults['OK']: self.metrics['failed'].append(jobsResults['Message']) return S_OK(self.metrics)
def doMaster( self ): """ Master method. Gets all sites and calls doNew method. """ siteNames = CSHelpers.getSites() if not siteNames[ 'OK' ]: return siteNames siteNames = siteNames[ 'Value' ] jobsResults = self.doNew( siteNames ) if not jobsResults[ 'OK' ]: self.metrics[ 'failed' ].append( jobsResults[ 'Message' ] ) return S_OK( self.metrics )
def doMaster(self): ''' Master method, which looks little bit spaguetti code, sorry ! - It gets all Sites. - It gets all StorageElements As there is no bulk query, it compares with what we have on the database. It queries a portion of them. ''' sites = CSHelpers.getSites() if not sites['OK']: return sites sites = sites['Value'] ses = CSHelpers.getStorageElements() if not ses['OK']: return ses ses = ses['Value'] elementNames = sites + ses # sourceQuery = self.rmClient.selectTransferCache( meta = { 'columns' : [ 'SourceName' ] } ) # if not sourceQuery[ 'OK' ]: # return sourceQuery # sourceQuery = [ element[0] for element in sourceQuery[ 'Value' ] ] # # sourceElementsToQuery = list( set( elementNames ).difference( set( sourceQuery ) ) ) gLogger.info('Processing %s' % ', '.join(elementNames)) for metric in ['Quality', 'FailedTransfers']: for direction in ['Source', 'Destination']: # 2 hours of window result = self.doNew((2, elementNames, direction, metric)) if not result['OK']: self.metrics['failed'].append(result) return S_OK(self.metrics) ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def doMaster( self ): ''' Master method, which looks little bit spaguetti code, sorry ! - It gets all Sites. - It gets all StorageElements As there is no bulk query, it compares with what we have on the database. It queries a portion of them. ''' sites = CSHelpers.getSites() if not sites[ 'OK' ]: return sites sites = sites[ 'Value' ] ses = CSHelpers.getStorageElements() if not ses[ 'OK' ]: return ses ses = ses[ 'Value' ] elementNames = sites + ses # sourceQuery = self.rmClient.selectTransferCache( meta = { 'columns' : [ 'SourceName' ] } ) # if not sourceQuery[ 'OK' ]: # return sourceQuery # sourceQuery = [ element[0] for element in sourceQuery[ 'Value' ] ] # # sourceElementsToQuery = list( set( elementNames ).difference( set( sourceQuery ) ) ) gLogger.info( 'Processing %s' % ', '.join( elementNames ) ) for metric in [ 'Quality', 'FailedTransfers' ]: for direction in [ 'Source', 'Destination' ]: # 2 hours of window result = self.doNew( ( 2, elementNames, direction, metric ) ) if not result[ 'OK' ]: self.metrics[ 'failed' ].append( result ) return S_OK( self.metrics ) ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def doMaster(self): siteNames = CSHelpers.getSites() if not siteNames['OK']: return siteNames siteNames = siteNames['Value'] ces = CSHelpers.getComputingElements() if not ces['OK']: return ces ces = ces['Value'] pilotResults = self.doNew(('Site', siteNames)) if not pilotResults['OK']: self.metrics['failed'].append(pilotResults['Message']) pilotResults = self.doNew(('Resource', ces)) if not pilotResults['OK']: self.metrics['failed'].append(pilotResults['Message']) return S_OK(self.metrics)
def doMaster( self ): siteNames = CSHelpers.getSites() if not siteNames[ 'OK' ]: return siteNames siteNames = siteNames[ 'Value' ] ces = CSHelpers.getComputingElements() if not ces[ 'OK' ]: return ces ces = ces[ 'Value' ] pilotResults = self.doNew( ( 'Site', siteNames ) ) if not pilotResults[ 'OK' ]: self.metrics[ 'failed' ].append( pilotResults[ 'Message' ] ) pilotResults = self.doNew( ( 'Resource', ces ) ) if not pilotResults[ 'OK' ]: self.metrics[ 'failed' ].append( pilotResults[ 'Message' ] ) return S_OK( self.metrics )
def export_getSitesResources(self, siteNames): """ Returns dictionary with SEs and CEs for the given site(s). If siteNames is None, all sites are taken into account. :return: S_OK( { site1 : { ces : [ ces ], 'ses' : [ ses ] },... } ) | S_ERROR """ gLogger.info('getSitesResources') if siteNames is None: siteNames = CSHelpers.getSites() if not siteNames['OK']: return siteNames siteNames = siteNames['Value'] if isinstance(siteNames, basestring): siteNames = [siteNames] sitesRes = {} for siteName in siteNames: res = {} res['ces'] = CSHelpers.getSiteComputingElements(siteName) # Convert StorageElements to host names ses = CSHelpers.getSiteStorageElements(siteName) sesHosts = CSHelpers.getStorageElementsHosts(ses) if not sesHosts['OK']: return sesHosts # Remove duplicates res['ses'] = list(set(sesHosts['Value'])) sitesRes[siteName] = res return S_OK(sitesRes)
def doCommand(self): """ # Returns simple pilots efficiency # # :attr:`args`: # - args[0]: string - should be a ValidElement # # - args[1]: string - should be the name of the ValidElement # # returns: # { # 'Result': 'Good'|'Fair'|'Poor'|'Idle'|'Bad' # } """ if not 'element' in self.args: return self.returnERROR(S_ERROR('element is missing')) element = self.args['element'] if not 'siteName' in self.args: return self.returnERROR(S_ERROR('siteName is missing')) siteName = self.args['siteName'] # If siteName is None, we take all sites if siteName is None: siteName = CSHelpers.getSites() if not siteName['OK']: return self.returnERROR(siteName) siteName = siteName['Value'] if element == 'Site': results = self.wmsAdmin.getPilotSummaryWeb({'GridSite': siteName}, [], 0, 300) elif element == 'Resource': results = self.wmsAdmin.getPilotSummaryWeb( {'ExpandSite': siteName}, [], 0, 300) else: return self.returnERROR(S_ERROR('%s is a wrong element' % element)) if not results['OK']: return self.returnERROR(results) results = results['Value'] if not 'ParameterNames' in results: return self.returnERROR(S_ERROR('Malformed result dictionary')) params = results['ParameterNames'] if not 'Records' in results: return self.returnERROR(S_ERROR('Malformed result dictionary')) records = results['Records'] pilotResults = [] for record in records: pilotDict = dict(zip(params, record)) try: pilotDict['PilotsPerJob'] = float(pilotDict['PilotsPerJob']) pilotDict['PilotsJobEff'] = float(pilotDict['PilotsJobEff']) except KeyError, e: return self.returnERROR(S_ERROR(e)) except ValueError, e: return self.returnERROR(S_ERROR(e))
def doCommand( self ): """ # Returns simple pilots efficiency # # :attr:`args`: # - args[0]: string - should be a ValidElement # # - args[1]: string - should be the name of the ValidElement # # returns: # { # 'Result': 'Good'|'Fair'|'Poor'|'Idle'|'Bad' # } """ if not 'element' in self.args: return self.returnERROR( S_ERROR( 'element is missing' ) ) element = self.args[ 'element' ] if not 'siteName' in self.args: return self.returnERROR( S_ERROR( 'siteName is missing' ) ) siteName = self.args[ 'siteName' ] # If siteName is None, we take all sites if siteName is None: siteName = CSHelpers.getSites() if not siteName[ 'OK' ]: return self.returnERROR( siteName ) siteName = siteName[ 'Value' ] if element == 'Site': results = self.wmsAdmin.getPilotSummaryWeb( { 'GridSite' : siteName }, [], 0, 300 ) elif element == 'Resource': results = self.wmsAdmin.getPilotSummaryWeb( { 'ExpandSite' : siteName }, [], 0, 300 ) else: return self.returnERROR( S_ERROR( '%s is a wrong element' % element ) ) if not results[ 'OK' ]: return self.returnERROR( results ) results = results[ 'Value' ] if not 'ParameterNames' in results: return self.returnERROR( S_ERROR( 'Malformed result dictionary' ) ) params = results[ 'ParameterNames' ] if not 'Records' in results: return self.returnERROR( S_ERROR( 'Malformed result dictionary' ) ) records = results[ 'Records' ] pilotResults = [] for record in records: pilotDict = dict( zip( params , record )) try: pilotDict[ 'PilotsPerJob' ] = float( pilotDict[ 'PilotsPerJob' ] ) pilotDict[ 'PilotsJobEff' ] = float( pilotDict[ 'PilotsJobEff' ] ) except KeyError, e: return self.returnERROR( S_ERROR( e ) ) except ValueError, e: return self.returnERROR( S_ERROR( e ) )