Exemplo n.º 1
0
 def __init__(self, **args):
     StartPolicyInterface.__init__(self, **args)
     self.args.setdefault('SliceType', 'NumberOfFiles')
     self.args.setdefault('SliceSize', 1)
     self.lumiType = "NumberOfLumis"
     self.sites = []
     self.siteDB = SiteDB()
Exemplo n.º 2
0
    def setUp(self):
        """
        _setUp_

        Setup couchdb and the test environment
        """
        super(ResubmitBlockTest, self).setUp()

        self.group = 'unknown'
        self.user = '******'

        # Set external test helpers
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")

        # Define test environment
        self.couchUrl = os.environ["COUCHURL"]
        self.acdcDBName = 'resubmitblock_t'
        self.validLocations = ['T2_US_Nebraska', 'T1_US_FNAL_Disk', 'T1_UK_RAL_Disk']
        self.siteWhitelist = ['T2_XX_SiteA']
        siteDB = SiteDB()
        #Convert phedex node name to a valid processing site name
        self.PSNs = siteDB.PNNstoPSNs(self.validLocations)
        self.workflowName = 'dballest_ReReco_workflow'
        couchServer = CouchServer(dburl=self.couchUrl)
        self.acdcDB = couchServer.connectDatabase(self.acdcDBName, create=False)
        user = makeUser(self.group, '*****@*****.**', self.couchUrl, self.acdcDBName)
        user.create()

        return
Exemplo n.º 3
0
    def __init__(self, **args):
        StartPolicyInterface.__init__(self, **args)
        self.args.setdefault('SliceType', 'NumberOfFiles')
        self.args.setdefault('SliceSize', 1)
        self.args.setdefault('SplittingAlgo', 'LumiBased')
        self.lumiType = "NumberOfLumis"

        # Define how to handle the different splitting algorithms
        self.algoMapping = {
            'Harvest': self.singleChunk,
            'ParentlessMergeBySize': self.singleChunk,
            'MinFileBased': self.singleChunk,
            'LumiBased': self.singleChunk,
            'EventAwareLumiBased': self.singleChunk,
            'EventBased': self.singleChunk
        }
        self.unsupportedAlgos = ['WMBSMergeBySize', 'SiblingProcessingBased']
        self.defaultAlgo = self.fixedSizeChunk
        self.sites = []
        if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv(
                "WMCORE_USE_CRIC", False):
            self.cric = CRIC()
        else:
            self.cric = None
            self.siteDB = SiteDB()
Exemplo n.º 4
0
 def __init__(self, **args):
     StartPolicyInterface.__init__(self, **args)
     self.args.setdefault('SliceType', 'NumberOfRuns')
     self.args.setdefault('SliceSize', 1)
     self.lumiType = "NumberOfLumis"
     self.sites = []
     if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv(
             "WMCORE_USE_CRIC", False):
         self.cric = CRIC()
     else:
         self.cric = None
         self.siteDB = SiteDB()
Exemplo n.º 5
0
    def __init__(self, **args):
        StartPolicyInterface.__init__(self, **args)
        self.args.setdefault('SliceType', 'NumberOfFiles')
        self.args.setdefault('SliceSize', 1)
        self.lumiType = "NumberOfLumis"

        # Initialize a list of sites where the data is
        self.sites = []

        # Initialize modifiers of the policy
        self.blockBlackListModifier = []

        self.siteDB = SiteDB()
Exemplo n.º 6
0
    def __init__(self, url, **contact):

        # instantiate dbs api object
        try:
            self.dbs = DbsApi(url, **contact)
        except dbsClientException as ex:
            msg = "Error in DBSReader with DbsApi\n"
            msg += "%s\n" % formatEx3(ex)
            raise DBSReaderError(msg)

        # connection to PhEDEx (Use default endpoint url)
        self.phedex = PhEDEx(responseType="json")
        self.siteDB = SiteDB()
Exemplo n.º 7
0
def cmsSiteNames():
    """Get all cms sites"""
    global __cmsSiteNames
    if __cmsSiteNames:
        return __cmsSiteNames
    global __sitedb
    if not __sitedb:
        from WMCore.Services.SiteDB.SiteDB import SiteDBJSON as SiteDB
        __sitedb = SiteDB()
    try:
        __cmsSiteNames = __sitedb.getAllCMSNames()
    except:
        pass
    return __cmsSiteNames
Exemplo n.º 8
0
def sitesFromStorageEelements(ses):
    """Return Sites given Storage Elements"""
    global __sitedb
    if not __sitedb:
        from WMCore.Services.SiteDB.SiteDB import SiteDBJSON as SiteDB
        __sitedb = SiteDB()
    result = set()
    for se in ses:
        try:
            site = __sitedb.seToCMSName(se)
        except:
            print "Unable to get site name for %s" % se
        else:
            result.add(site)
    return list(result)
Exemplo n.º 9
0
    def __init__(self, **args):
        StartPolicyInterface.__init__(self, **args)
        self.args.setdefault('SliceType', 'NumberOfFiles')
        self.args.setdefault('SliceSize', 1)
        self.lumiType = "NumberOfLumis"

        # Initialize a list of sites where the data is
        self.sites = []

        # Initialize modifiers of the policy
        self.blockBlackListModifier = []
        if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv(
                "WMCORE_USE_CRIC", False):
            self.cric = CRIC()
        else:
            self.cric = None
            self.siteDB = SiteDB()
Exemplo n.º 10
0
def cmsSiteNames():
    """Get all cms sites"""
    global __cmsSiteNames
    if __cmsSiteNames:
        return __cmsSiteNames
    logging.info("cmsSiteNames Using CRIC Service: %s", __USE_CRIC)
    global __sitedb
    if not __sitedb:
        if __USE_CRIC:
            from WMCore.Services.CRIC.CRIC import CRIC
            __sitedb = CRIC()
        else:
            from WMCore.Services.SiteDB.SiteDB import SiteDBJSON as SiteDB
            __sitedb = SiteDB()
    try:
        if __USE_CRIC:
            __cmsSiteNames = __sitedb.getAllPSNs()
        else:
            __cmsSiteNames = __sitedb.getAllCMSNames()
    except Exception:
        pass
    return __cmsSiteNames
Exemplo n.º 11
0
class WorkQueue(WorkQueueBase):
    """
    _WorkQueue_

    WorkQueue object - interface to WorkQueue functionality.
    """
    def __init__(self, logger=None, dbi=None, **params):

        WorkQueueBase.__init__(self, logger, dbi)
        self.parent_queue = None
        self.params = params

        # config argument (within params) shall be reference to
        # Configuration instance (will later be checked for presence of "Alert")
        self.config = params.get("Config", None)
        self.params.setdefault('CouchUrl', os.environ.get('COUCHURL'))
        if not self.params.get('CouchUrl'):
            raise RuntimeError, 'CouchUrl config value mandatory'
        self.params.setdefault('DbName', 'workqueue')
        self.params.setdefault('InboxDbName', self.params['DbName'] + '_inbox')
        self.params.setdefault('ParentQueueCouchUrl',
                               None)  # We get work from here

        self.backend = WorkQueueBackend(self.params['CouchUrl'],
                                        self.params['DbName'],
                                        self.params['InboxDbName'],
                                        self.params['ParentQueueCouchUrl'],
                                        self.params.get('QueueURL'),
                                        logger=self.logger)
        if self.params.get('ParentQueueCouchUrl'):
            try:
                self.parent_queue = WorkQueueBackend(
                    self.params['ParentQueueCouchUrl'].rsplit('/', 1)[0],
                    self.params['ParentQueueCouchUrl'].rsplit('/', 1)[1])
            except IndexError, ex:
                # Probable cause: Someone didn't put the global WorkQueue name in
                # the ParentCouchUrl
                msg = "Parsing failure for ParentQueueCouchUrl - probably missing dbname in input\n"
                msg += "Exception: %s\n" % str(ex)
                msg += str("ParentQueueCouchUrl: %s\n" %
                           self.params['ParentQueueCouchUrl'])
                self.logger.error(msg)
                raise WorkQueueError(msg)
            self.params['ParentQueueCouchUrl'] = self.parent_queue.queueUrl

        self.params.setdefault(
            "GlobalDBS",
            "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet")
        self.params.setdefault('QueueDepth',
                               0.5)  # when less than this locally
        self.params.setdefault('LocationRefreshInterval', 600)
        self.params.setdefault('FullLocationRefreshInterval', 7200)
        self.params.setdefault('TrackLocationOrSubscription', 'subscription')
        self.params.setdefault('ReleaseIncompleteBlocks', False)
        self.params.setdefault('ReleaseRequireSubscribed', True)
        self.params.setdefault('PhEDExEndpoint', None)
        self.params.setdefault('PopulateFilesets', True)
        self.params.setdefault('LocalQueueFlag', True)
        self.params.setdefault('QueueRetryTime', 86400)
        self.params.setdefault('stuckElementAlertTime', 86400)
        self.params.setdefault('reqmgrCompleteGraceTime', 604800)
        self.params.setdefault('cancelGraceTime', 604800)

        self.params.setdefault('JobDumpConfig', None)
        self.params.setdefault('BossAirConfig', None)

        self.params[
            'QueueURL'] = self.backend.queueUrl  # url this queue is visible on
        # backend took previous QueueURL and sanitized it
        self.params.setdefault('WMBSUrl',
                               None)  # this will only be set on local Queue
        if self.params.get('WMBSUrl'):
            self.params['WMBSUrl'] = Lexicon.sanitizeURL(
                self.params['WMBSUrl'])['url']
        self.params.setdefault('Teams', [])
        self.params.setdefault('DrainMode', False)
        if self.params.get('CacheDir'):
            try:
                os.makedirs(self.params['CacheDir'])
            except OSError:
                pass
        elif self.params.get('PopulateFilesets'):
            raise RuntimeError, 'CacheDir mandatory for local queue'

        self.params.setdefault('SplittingMapping', {})
        self.params['SplittingMapping'].setdefault('DatasetBlock', {
            'name': 'Block',
            'args': {}
        })
        self.params['SplittingMapping'].setdefault('MonteCarlo', {
            'name': 'MonteCarlo',
            'args': {}
        })
        self.params['SplittingMapping'].setdefault('Dataset', {
            'name': 'Dataset',
            'args': {}
        })
        self.params['SplittingMapping'].setdefault('Block', {
            'name': 'Block',
            'args': {}
        })
        self.params['SplittingMapping'].setdefault('ResubmitBlock', {
            'name': 'ResubmitBlock',
            'args': {}
        })

        self.params.setdefault('EndPolicySettings', {})

        assert (self.params['TrackLocationOrSubscription']
                in ('subscription', 'location'))
        # Can only release blocks on location
        if self.params['TrackLocationOrSubscription'] == 'location':
            if self.params['SplittingMapping']['DatasetBlock'][
                    'name'] != 'Block':
                raise RuntimeError, 'Only blocks can be released on location'

        if self.params.get('PhEDEx'):
            self.phedexService = self.params['PhEDEx']
        else:
            phedexArgs = {}
            if self.params.get('PhEDExEndpoint'):
                phedexArgs['endpoint'] = self.params['PhEDExEndpoint']
            self.phedexService = PhEDEx(phedexArgs)

        if self.params.get('SiteDB'):
            self.SiteDB = self.params['SiteDB']
        else:
            self.SiteDB = SiteDB()

        if type(self.params['Teams']) in types.StringTypes:
            self.params['Teams'] = [x.strip() for x in \
                                    self.params['Teams'].split(',')]

        self.dataLocationMapper = WorkQueueDataLocationMapper(
            self.logger,
            self.backend,
            phedex=self.phedexService,
            sitedb=self.SiteDB,
            locationFrom=self.params['TrackLocationOrSubscription'],
            incompleteBlocks=self.params['ReleaseIncompleteBlocks'],
            requireBlocksSubscribed=not self.params['ReleaseIncompleteBlocks'],
            fullRefreshInterval=self.params['FullLocationRefreshInterval'],
            updateIntervalCoarseness=self.params['LocationRefreshInterval'])

        # initialize alerts sending client (self.sendAlert() method)
        # usage: self.sendAlert(levelNum, msg = msg) ; level - integer 1 .. 10
        #    1 - 4 - lower levels ; 5 - 10 higher levels
        preAlert, self.alertSender = \
            alertAPI.setUpAlertsMessaging(self, compName = "WorkQueueManager")
        self.sendAlert = alertAPI.getSendAlert(sender=self.alertSender,
                                               preAlert=preAlert)

        self.logger.debug("WorkQueue created successfully")