def __init__(self, logger = None, dbi = None, **params):
WorkQueueBase.__init__(self, logger, dbi)
self.parent_queue = None
self.params = params
# config argument (within params) shall be reference to
# Configuration instance (will later be checked for presence of "Alert")
self.config = params.get("Config", None)
self.params.setdefault('CouchUrl', os.environ.get('COUCHURL'))
if not self.params.get('CouchUrl'):
raise RuntimeError, 'CouchUrl config value mandatory'
self.params.setdefault('DbName', 'workqueue')
self.params.setdefault('InboxDbName', self.params['DbName'] + '_inbox')
self.params.setdefault('ParentQueueCouchUrl', None) # We get work from here
self.backend = WorkQueueBackend(self.params['CouchUrl'], self.params['DbName'],
self.params['InboxDbName'],
self.params['ParentQueueCouchUrl'], self.params.get('QueueURL'),
logger = self.logger)
if self.params.get('ParentQueueCouchUrl'):
try:
self.parent_queue = WorkQueueBackend(self.params['ParentQueueCouchUrl'].rsplit('/', 1)[0],
self.params['ParentQueueCouchUrl'].rsplit('/', 1)[1])
except IndexError, ex:
# Probable cause: Someone didn't put the global WorkQueue name in
# the ParentCouchUrl
msg = "Parsing failure for ParentQueueCouchUrl - probably missing dbname in input\n"
msg += "Exception: %s\n" % str(ex)
msg += str("ParentQueueCouchUrl: %s\n" % self.params['ParentQueueCouchUrl'])
self.logger.error(msg)
raise WorkQueueError(msg)
self.params['ParentQueueCouchUrl'] = self.parent_queue.queueUrl
def __init__(self, logger=None, dbi=None, **params):
WorkQueueBase.__init__(self, logger, dbi)
self.parent_queue = None
self.params = params
# config argument (within params) shall be reference to
# Configuration instance (will later be checked for presence of "Alert")
self.config = params.get("Config", None)
self.params.setdefault('CouchUrl', os.environ.get('COUCHURL'))
if not self.params.get('CouchUrl'):
raise RuntimeError, 'CouchUrl config value mandatory'
self.params.setdefault('DbName', 'workqueue')
self.params.setdefault('InboxDbName', self.params['DbName'] + '_inbox')
self.params.setdefault('ParentQueueCouchUrl',
None) # We get work from here
self.backend = WorkQueueBackend(self.params['CouchUrl'],
self.params['DbName'],
self.params['InboxDbName'],
self.params['ParentQueueCouchUrl'],
self.params.get('QueueURL'),
logger=self.logger)
if self.params.get('ParentQueueCouchUrl'):
try:
self.parent_queue = WorkQueueBackend(
self.params['ParentQueueCouchUrl'].rsplit('/', 1)[0],
self.params['ParentQueueCouchUrl'].rsplit('/', 1)[1])
except IndexError, ex:
# Probable cause: Someone didn't put the global WorkQueue name in
# the ParentCouchUrl
msg = "Parsing failure for ParentQueueCouchUrl - probably missing dbname in input\n"
msg += "Exception: %s\n" % str(ex)
msg += str("ParentQueueCouchUrl: %s\n" %
self.params['ParentQueueCouchUrl'])
self.logger.error(msg)
raise WorkQueueError(msg)
self.params['ParentQueueCouchUrl'] = self.parent_queue.queueUrl
def __init__(self, logger = None, dbi = None, **params):
WorkQueueBase.__init__(self, logger, dbi)
self.parent_queue = None
self.params = params
# config argument (within params) shall be reference to
# Configuration instance (will later be checked for presence of "Alert")
self.config = params.get("Config", None)
self.params.setdefault('CouchUrl', os.environ.get('COUCHURL'))
if not self.params.get('CouchUrl'):
raise RuntimeError, 'CouchUrl config value mandatory'
self.params.setdefault('DbName', 'workqueue')
self.params.setdefault('InboxDbName', self.params['DbName'] + '_inbox')
self.params.setdefault('ParentQueueCouchUrl', None) # We get work from here
self.backend = WorkQueueBackend(self.params['CouchUrl'], self.params['DbName'],
self.params['InboxDbName'],
self.params['ParentQueueCouchUrl'], self.params.get('QueueURL'),
logger = self.logger)
if self.params.get('ParentQueueCouchUrl'):
self.parent_queue = WorkQueueBackend(self.params['ParentQueueCouchUrl'].rsplit('/', 1)[0],
self.params['ParentQueueCouchUrl'].rsplit('/', 1)[1])
self.params.setdefault("GlobalDBS",
"http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet")
self.params.setdefault('QueueDepth', 2) # when less than this locally
self.params.setdefault('LocationRefreshInterval', 600)
self.params.setdefault('FullLocationRefreshInterval', 7200)
self.params.setdefault('TrackLocationOrSubscription', 'subscription')
self.params.setdefault('ReleaseIncompleteBlocks', False)
self.params.setdefault('ReleaseRequireSubscribed', True)
self.params.setdefault('PhEDExEndpoint', None)
self.params.setdefault('PopulateFilesets', True)
self.params.setdefault('LocalQueueFlag', True)
self.params.setdefault('JobDumpConfig', None)
self.params.setdefault('BossAirConfig', None)
self.params['QueueURL'] = self.backend.queueUrl # url this queue is visible on
# backend took previous QueueURL and sanitized it
self.params.setdefault('WMBSUrl', None) # this will only be set on local Queue
self.params.setdefault('Teams', [''])
self.params.setdefault('DrainMode', False)
if self.params.get('CacheDir'):
try:
os.makedirs(self.params['CacheDir'])
except OSError:
pass
elif self.params.get('PopulateFilesets'):
raise RuntimeError, 'CacheDir mandatory for local queue'
self.params.setdefault('SplittingMapping', {})
self.params['SplittingMapping'].setdefault('DatasetBlock',
{'name': 'Block',
'args': {}}
)
self.params['SplittingMapping'].setdefault('MonteCarlo',
{'name': 'MonteCarlo',
'args':{}}
)
self.params['SplittingMapping'].setdefault('Dataset',
{'name': 'Dataset',
'args': {}}
)
self.params['SplittingMapping'].setdefault('Block',
{'name': 'Block',
'args': {}}
)
self.params['SplittingMapping'].setdefault('ResubmitBlock',
{'name': 'ResubmitBlock',
'args': {}}
)
self.params.setdefault('EndPolicySettings', {})
assert(self.params['TrackLocationOrSubscription'] in ('subscription',
'location'))
# Can only release blocks on location
if self.params['TrackLocationOrSubscription'] == 'location':
if self.params['SplittingMapping']['DatasetBlock']['name'] != 'Block':
raise RuntimeError, 'Only blocks can be released on location'
if self.params.get('PhEDEx'):
self.phedexService = self.params['PhEDEx']
else:
phedexArgs = {}
if self.params.get('PhEDExEndpoint'):
phedexArgs['endpoint'] = self.params['PhEDExEndpoint']
self.phedexService = PhEDEx(phedexArgs)
if self.params.get('SiteDB'):
self.SiteDB = self.params['SiteDB']
else:
self.SiteDB = SiteDB()
if type(self.params['Teams']) in types.StringTypes:
self.params['Teams'] = [x.strip() for x in \
self.params['Teams'].split(',')]
self.dataLocationMapper = WorkQueueDataLocationMapper(self.logger, self.backend,
phedex = self.phedexService,
sitedb = self.SiteDB,
locationFrom = self.params['TrackLocationOrSubscription'],
incompleteBlocks = self.params['ReleaseIncompleteBlocks'],
requireBlocksSubscribed = not self.params['ReleaseIncompleteBlocks'],
fullRefreshInterval = self.params['FullLocationRefreshInterval'],
updateIntervalCoarseness = self.params['LocationRefreshInterval'])
# initialize alerts sending client (self.sendAlert() method)
# usage: self.sendAlert(levelNum, msg = msg) ; level - integer 1 .. 10
# 1 - 4 - lower levels ; 5 - 10 higher levels
preAlert, self.alertSender = \
alertAPI.setUpAlertsMessaging(self, compName = "WorkQueueManager")
self.sendAlert = alertAPI.getSendAlert(sender = self.alertSender,
preAlert = preAlert)
self.logger.debug("WorkQueue created successfully")
