def __init__(self, **args):
StartPolicyInterface.__init__(self, **args)
self.args.setdefault('SliceType', 'NumberOfFiles')
self.args.setdefault('SliceSize', 1)
self.lumiType = "NumberOfLumis"
self.sites = []
self.siteDB = SiteDB()
def setUp(self):
"""
_setUp_
Setup couchdb and the test environment
"""
super(ResubmitBlockTest, self).setUp()
self.group = 'unknown'
self.user = '******'
# Set external test helpers
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")
# Define test environment
self.couchUrl = os.environ["COUCHURL"]
self.acdcDBName = 'resubmitblock_t'
self.validLocations = ['T2_US_Nebraska', 'T1_US_FNAL_Disk', 'T1_UK_RAL_Disk']
self.siteWhitelist = ['T2_XX_SiteA']
siteDB = SiteDB()
#Convert phedex node name to a valid processing site name
self.PSNs = siteDB.PNNstoPSNs(self.validLocations)
self.workflowName = 'dballest_ReReco_workflow'
couchServer = CouchServer(dburl=self.couchUrl)
self.acdcDB = couchServer.connectDatabase(self.acdcDBName, create=False)
user = makeUser(self.group, '*****@*****.**', self.couchUrl, self.acdcDBName)
user.create()
return
def __init__(self, **args):
StartPolicyInterface.__init__(self, **args)
self.args.setdefault('SliceType', 'NumberOfFiles')
self.args.setdefault('SliceSize', 1)
self.args.setdefault('SplittingAlgo', 'LumiBased')
self.lumiType = "NumberOfLumis"
# Define how to handle the different splitting algorithms
self.algoMapping = {
'Harvest': self.singleChunk,
'ParentlessMergeBySize': self.singleChunk,
'MinFileBased': self.singleChunk,
'LumiBased': self.singleChunk,
'EventAwareLumiBased': self.singleChunk,
'EventBased': self.singleChunk
}
self.unsupportedAlgos = ['WMBSMergeBySize', 'SiblingProcessingBased']
self.defaultAlgo = self.fixedSizeChunk
self.sites = []
if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv(
"WMCORE_USE_CRIC", False):
self.cric = CRIC()
else:
self.cric = None
self.siteDB = SiteDB()
def __init__(self, **args):
StartPolicyInterface.__init__(self, **args)
self.args.setdefault('SliceType', 'NumberOfRuns')
self.args.setdefault('SliceSize', 1)
self.lumiType = "NumberOfLumis"
self.sites = []
if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv(
"WMCORE_USE_CRIC", False):
self.cric = CRIC()
else:
self.cric = None
self.siteDB = SiteDB()
def __init__(self, **args):
StartPolicyInterface.__init__(self, **args)
self.args.setdefault('SliceType', 'NumberOfFiles')
self.args.setdefault('SliceSize', 1)
self.lumiType = "NumberOfLumis"
# Initialize a list of sites where the data is
self.sites = []
# Initialize modifiers of the policy
self.blockBlackListModifier = []
self.siteDB = SiteDB()
def __init__(self, url, **contact):
# instantiate dbs api object
try:
self.dbs = DbsApi(url, **contact)
except dbsClientException as ex:
msg = "Error in DBSReader with DbsApi\n"
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
# connection to PhEDEx (Use default endpoint url)
self.phedex = PhEDEx(responseType="json")
self.siteDB = SiteDB()
def cmsSiteNames():
"""Get all cms sites"""
global __cmsSiteNames
if __cmsSiteNames:
return __cmsSiteNames
global __sitedb
if not __sitedb:
from WMCore.Services.SiteDB.SiteDB import SiteDBJSON as SiteDB
__sitedb = SiteDB()
try:
__cmsSiteNames = __sitedb.getAllCMSNames()
except:
pass
return __cmsSiteNames
def sitesFromStorageEelements(ses):
"""Return Sites given Storage Elements"""
global __sitedb
if not __sitedb:
from WMCore.Services.SiteDB.SiteDB import SiteDBJSON as SiteDB
__sitedb = SiteDB()
result = set()
for se in ses:
try:
site = __sitedb.seToCMSName(se)
except:
print "Unable to get site name for %s" % se
else:
result.add(site)
return list(result)
def __init__(self, **args):
StartPolicyInterface.__init__(self, **args)
self.args.setdefault('SliceType', 'NumberOfFiles')
self.args.setdefault('SliceSize', 1)
self.lumiType = "NumberOfLumis"
# Initialize a list of sites where the data is
self.sites = []
# Initialize modifiers of the policy
self.blockBlackListModifier = []
if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv(
"WMCORE_USE_CRIC", False):
self.cric = CRIC()
else:
self.cric = None
self.siteDB = SiteDB()
def cmsSiteNames():
"""Get all cms sites"""
global __cmsSiteNames
if __cmsSiteNames:
return __cmsSiteNames
logging.info("cmsSiteNames Using CRIC Service: %s", __USE_CRIC)
global __sitedb
if not __sitedb:
if __USE_CRIC:
from WMCore.Services.CRIC.CRIC import CRIC
__sitedb = CRIC()
else:
from WMCore.Services.SiteDB.SiteDB import SiteDBJSON as SiteDB
__sitedb = SiteDB()
try:
if __USE_CRIC:
__cmsSiteNames = __sitedb.getAllPSNs()
else:
__cmsSiteNames = __sitedb.getAllCMSNames()
except Exception:
pass
return __cmsSiteNames
class WorkQueue(WorkQueueBase):
"""
_WorkQueue_
WorkQueue object - interface to WorkQueue functionality.
"""
def __init__(self, logger=None, dbi=None, **params):
WorkQueueBase.__init__(self, logger, dbi)
self.parent_queue = None
self.params = params
# config argument (within params) shall be reference to
# Configuration instance (will later be checked for presence of "Alert")
self.config = params.get("Config", None)
self.params.setdefault('CouchUrl', os.environ.get('COUCHURL'))
if not self.params.get('CouchUrl'):
raise RuntimeError, 'CouchUrl config value mandatory'
self.params.setdefault('DbName', 'workqueue')
self.params.setdefault('InboxDbName', self.params['DbName'] + '_inbox')
self.params.setdefault('ParentQueueCouchUrl',
None) # We get work from here
self.backend = WorkQueueBackend(self.params['CouchUrl'],
self.params['DbName'],
self.params['InboxDbName'],
self.params['ParentQueueCouchUrl'],
self.params.get('QueueURL'),
logger=self.logger)
if self.params.get('ParentQueueCouchUrl'):
try:
self.parent_queue = WorkQueueBackend(
self.params['ParentQueueCouchUrl'].rsplit('/', 1)[0],
self.params['ParentQueueCouchUrl'].rsplit('/', 1)[1])
except IndexError, ex:
# Probable cause: Someone didn't put the global WorkQueue name in
# the ParentCouchUrl
msg = "Parsing failure for ParentQueueCouchUrl - probably missing dbname in input\n"
msg += "Exception: %s\n" % str(ex)
msg += str("ParentQueueCouchUrl: %s\n" %
self.params['ParentQueueCouchUrl'])
self.logger.error(msg)
raise WorkQueueError(msg)
self.params['ParentQueueCouchUrl'] = self.parent_queue.queueUrl
self.params.setdefault(
"GlobalDBS",
"http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet")
self.params.setdefault('QueueDepth',
0.5) # when less than this locally
self.params.setdefault('LocationRefreshInterval', 600)
self.params.setdefault('FullLocationRefreshInterval', 7200)
self.params.setdefault('TrackLocationOrSubscription', 'subscription')
self.params.setdefault('ReleaseIncompleteBlocks', False)
self.params.setdefault('ReleaseRequireSubscribed', True)
self.params.setdefault('PhEDExEndpoint', None)
self.params.setdefault('PopulateFilesets', True)
self.params.setdefault('LocalQueueFlag', True)
self.params.setdefault('QueueRetryTime', 86400)
self.params.setdefault('stuckElementAlertTime', 86400)
self.params.setdefault('reqmgrCompleteGraceTime', 604800)
self.params.setdefault('cancelGraceTime', 604800)
self.params.setdefault('JobDumpConfig', None)
self.params.setdefault('BossAirConfig', None)
self.params[
'QueueURL'] = self.backend.queueUrl # url this queue is visible on
# backend took previous QueueURL and sanitized it
self.params.setdefault('WMBSUrl',
None) # this will only be set on local Queue
if self.params.get('WMBSUrl'):
self.params['WMBSUrl'] = Lexicon.sanitizeURL(
self.params['WMBSUrl'])['url']
self.params.setdefault('Teams', [])
self.params.setdefault('DrainMode', False)
if self.params.get('CacheDir'):
try:
os.makedirs(self.params['CacheDir'])
except OSError:
pass
elif self.params.get('PopulateFilesets'):
raise RuntimeError, 'CacheDir mandatory for local queue'
self.params.setdefault('SplittingMapping', {})
self.params['SplittingMapping'].setdefault('DatasetBlock', {
'name': 'Block',
'args': {}
})
self.params['SplittingMapping'].setdefault('MonteCarlo', {
'name': 'MonteCarlo',
'args': {}
})
self.params['SplittingMapping'].setdefault('Dataset', {
'name': 'Dataset',
'args': {}
})
self.params['SplittingMapping'].setdefault('Block', {
'name': 'Block',
'args': {}
})
self.params['SplittingMapping'].setdefault('ResubmitBlock', {
'name': 'ResubmitBlock',
'args': {}
})
self.params.setdefault('EndPolicySettings', {})
assert (self.params['TrackLocationOrSubscription']
in ('subscription', 'location'))
# Can only release blocks on location
if self.params['TrackLocationOrSubscription'] == 'location':
if self.params['SplittingMapping']['DatasetBlock'][
'name'] != 'Block':
raise RuntimeError, 'Only blocks can be released on location'
if self.params.get('PhEDEx'):
self.phedexService = self.params['PhEDEx']
else:
phedexArgs = {}
if self.params.get('PhEDExEndpoint'):
phedexArgs['endpoint'] = self.params['PhEDExEndpoint']
self.phedexService = PhEDEx(phedexArgs)
if self.params.get('SiteDB'):
self.SiteDB = self.params['SiteDB']
else:
self.SiteDB = SiteDB()
if type(self.params['Teams']) in types.StringTypes:
self.params['Teams'] = [x.strip() for x in \
self.params['Teams'].split(',')]
self.dataLocationMapper = WorkQueueDataLocationMapper(
self.logger,
self.backend,
phedex=self.phedexService,
sitedb=self.SiteDB,
locationFrom=self.params['TrackLocationOrSubscription'],
incompleteBlocks=self.params['ReleaseIncompleteBlocks'],
requireBlocksSubscribed=not self.params['ReleaseIncompleteBlocks'],
fullRefreshInterval=self.params['FullLocationRefreshInterval'],
updateIntervalCoarseness=self.params['LocationRefreshInterval'])
# initialize alerts sending client (self.sendAlert() method)
# usage: self.sendAlert(levelNum, msg = msg) ; level - integer 1 .. 10
# 1 - 4 - lower levels ; 5 - 10 higher levels
preAlert, self.alertSender = \
alertAPI.setUpAlertsMessaging(self, compName = "WorkQueueManager")
self.sendAlert = alertAPI.getSendAlert(sender=self.alertSender,
preAlert=preAlert)
self.logger.debug("WorkQueue created successfully")