class FactoryMock(object):
def __init__(self, fcl, am):
self.fcl = fcl
self.mcl = Config()
self.mcl.add_section('MockMonitor')
self.mcl.set('MockMonitor','monitorURL','')
self.threadsregistry = ThreadsRegistry()
self.authmanager = am
def getMockFactoryConfig():
cp = Config()
cp.add_section('Factory')
cp.set('Factory', 'batchstatus.condor.maxage', '60' )
cp.set('Factory', 'batchstatus.condor.sleep', '30' )
cp.set('Factory', 'factoryUser', 'autopyfactory' )
cp.set('Factory', 'factoryId', 'dummy-mock' )
return cp
class AgisCEQueue(object):
"""
Represents a single CE queue within a Panda queue description.
"""
def __init__(self, parent, cedict):
self.log = logging.getLogger('autopyfactory.config')
self.parent = parent
self.panda_queue_name = parent.panda_queue_name
self.ce_name = cedict['ce_name'] # AGLT2-CE-gate04.aglt2.org
self.ce_endpoint = cedict['ce_endpoint'] # gate04.aglt2.org:2119
self.ce_host = self.ce_endpoint.split(":")[0]
self.ce_state = cedict['ce_state'].lower() # 'active'
self.ce_status = cedict['ce_status'].lower() #
self.ce_queue_status = cedict['ce_queue_status'].lower()
self.ce_flavour = cedict['ce_flavour'].lower() # GLOBUS
self.ce_version = cedict['ce_version'].lower() # GT5
self.ce_queue_name = cedict['ce_queue_name'] # default
self.ce_jobmanager = cedict['ce_jobmanager'].lower() # condor
self.ce_queue_maxcputime = cedict['ce_queue_maxcputime'] # in seconds
self.ce_queue_maxwctime = cedict['ce_queue_maxwctime'] # in seconds
self.apf_scale_factor = 1.0
# Empty/default attributes:
self.gridresource = None
self.submitplugin = None
self.submitpluginstr = None
self.gramversion = None
self.gramqueue = None
self.creamenv = None
self.creamattr = ''
self.condorattr = None
self.maxmemory = None
self.maxtime = None
if self.ce_flavour in ['osg-ce','globus']:
self._initglobus()
elif self.ce_flavour == 'htcondor-ce':
self._initcondorce()
elif self.ce_flavour == 'cream-ce':
self._initcream()
elif self.ce_flavour == 'arc-ce':
self._initarc()
elif self.ce_flavour == 'lcg-ce':
self.log.debug("Ignoring old CE type 'LCG-CE'")
else:
self.log.warning("CEQueue %s has unknown ce_flavour: %s" % (self.ce_name, self.ce_flavour))
def _initcondorce(self):
self.gridresource = self.ce_endpoint.split(':')[0]
self.submitplugin = 'CondorOSGCE'
self.submitpluginstr = 'condorosgce'
def _initglobus(self):
self.gridresource = '%s/jobmanager-%s' % (self.ce_endpoint, self.ce_jobmanager)
if self.ce_version == 'gt2':
self.submitplugin = 'CondorGT2'
self.submitpluginstr = 'condorgt2'
elif self.ce_version == 'gt5':
self.submitplugin = 'CondorGT5'
self.submitpluginstr = 'condorgt5'
self.gramversion = 'gram5'
self.gramqueue = self.ce_queue_name
def _initcream(self):
self.gridresource = '%s/ce-cream/services/CREAM2 %s %s' % (self.ce_endpoint,
self.ce_jobmanager,
self.ce_queue_name)
self.submitplugin = 'CondorCREAM'
self.submitpluginstr = 'condorcream'
if self.parent.pilot_version not in ['current']:
self.creamenv = 'RUCIO_ACCOUNT=pilot PILOT_HTTP_SOURCES=%s' % self.parent.pilot_version
else:
self.creamenv = 'RUCIO_ACCOUNT=pilot'
# glue 1.3 uses minutes and this / operator uses floor value
# https://wiki.italiangrid.it/twiki/bin/view/CREAM/UserGuideEMI2#Forward_of_requirements_to_the_b
self.maxtime = self.parent.maxtime / 60
self.maxmemory = self.parent.maxmemory
self.cputime = self.parent.corecount * self.maxtime
# maxrss and maxtime are expected to be set in AGIS for all queues
if self.parent.corecount:
self.creamattr = 'CpuNumber=%d;WholeNodes=false;SMPGranularity=%d;' % (self.parent.corecount,
self.parent.corecount)
if self.parent.corecount:
cputime = self.parent.corecount * self.maxtime
else:
cputime = self.maxtime
self.creamattr += 'CERequirements = "other.GlueCEPolicyMaxCPUTime == %d ' % cputime
self.creamattr += '&& other.GlueCEPolicyMaxWallClockTime == %d ' % self.maxtime
self.creamattr += '&& other.GlueHostMainMemoryRAMSize == %d' % self.parent.maxrss
if self.parent.maxswap:
maxvirtual = self.parent.maxrss + self.parent.maxswap
self.creamattr += ' && other.GlueHostMainMemoryVirtualSize == %d";' % maxvirtual
else:
self.creamattr += '";'
def _initarc(self):
# ignore :port part
self.gridresource = self.ce_endpoint.split(':')[0]
self.submitplugin = 'CondorNordugrid'
self.submitpluginstr = 'condornordugrid'
self.maxmemory = self.parent.maxmemory
#### self.maxtime = self.ce_queue_maxwctime
self.maxtime = self.parent.maxtime
self.nordugridrsl = '(jobname = arc_pilot)'
self.rsladd = '(runtimeenvironment = APPS/HEP/ATLAS-SITE-LCG)(runtimeenvironment = ENV/PROXY)'
self.rsladd += '(count = %d)' % self.parent.corecount
self.rsladd += '(countpernode = %d)' % self.parent.corecount
if self.parent.maxrss:
percore = self.parent.maxrss/self.parent.corecount
self.rsladd += '(memory = %d)' % percore
else:
percore = self.parent.maxmemory/self.parent.corecount
self.rsladd += '(memory = %d)' % percore
if self.maxtime:
self.rsladd += '(walltime = %d)' % self.maxtime
if self.maxtime:
self.rsladd += '(cputime = %d)' % (self.maxtime * self.parent.corecount)
def getAPFConfigString(self):
"""
Returns string of valid APF configuration for this queue-ce entry.
Calculates scale factor based on how many other CEs serve this PQ
"""
cp = self.getAPFConfig()
sio = StringIO()
s = cp.write(sio)
return sio.getvalue()
def getAPFConfig(self):
"""
Returns ConfigParser object representing config
"""
self.cp = Config()
sect = '%s-%s' % ( self.parent.panda_resource, self.ce_host )
sect = str(sect)
self.cp.add_section(sect)
# Unconditional config
self.cp.set( sect, 'enabled', 'True')
self.cp.set( sect, 'batchqueue', self.panda_queue_name)
self.cp.set( sect, 'wmsqueue', self.parent.panda_resource )
self.cp.set( sect, 'batchsubmitplugin', self.submitplugin )
self.cp.set( sect, 'batchsubmit.%s.gridresource' % self.submitpluginstr , self.gridresource )
#if self.parent.type == 'analysis':
# self.cp.set( sect, 'executable.arguments' , '%(executable.defaultarguments)s -u user' )
#else:
# self.cp.set( sect, 'executable.arguments' , '%(executable.defaultarguments)s -u managed' )
try:
self.apf_scale_factor = ((( 1.0 / float(self.parent.parent.numfactories) ) / len(self.parent.ce_queues) ) / float(self.parent.parent.jobsperpilot) )
except ZeroDivisionError:
self.log.error("Division by zero. Something wrong with scale factory calc.")
self.apf_scale_factor = 1.0
self.cp.set( sect, 'sched.scale.factor', str(self.apf_scale_factor) )
#HTCondor CE
if self.ce_flavour == 'htcondor-ce':
pr = 'periodic_remove = (JobStatus == 2 && (CurrentTime - EnteredCurrentStatus) > 604800)'
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes', pr )
if self.parent.maxrss is not None:
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes.+maxMemory', str(self.parent.maxrss) )
else:
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes.+maxMemory', str(self.parent.maxmemory) )
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes.+xcount', str(self.parent.corecount) )
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes.+voactivity', '"%s"' % self.parent.type )
if self.ce_queue_name not in ['default','']:
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes.+remote_queue', '"%s"' % self.ce_queue_name)
if self.ce_queue_maxwctime:
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes.+maxWallTime', '%s' % self.ce_queue_maxwctime)
# Globus
if self.ce_flavour in ['osg-ce','globus']:
self.cp.set( sect, 'globusrsl.%s.queue' % self.gramversion, self.gramqueue )
# Cream-specific JDL
if self.ce_flavour == 'cream-ce':
self.cp.set( sect, 'batchsubmit.condorcream.environ', self.creamenv )
if self.creamattr is not None:
self.cp.set(sect, 'creamattr', self.creamattr)
self.cp.set(sect, 'batchsubmit.condorcream.condor_attributes' , '%(req)s,%(hold)s,%(remove)s,cream_attributes = %(creamattr)s,notification=Never' )
else:
self.cp.set(sect, 'batchsubmit.condorcream.condor_attributes' , '%(req)s,%(hold)s,%(remove)s,notification=Never' )
# Arc-CE
if self.ce_flavour == 'arc-ce':
pr = 'periodic_remove = (JobStatus == 2 && (CurrentTime - EnteredCurrentStatus) > 604800)'
pr = '%(req)s,%(hold)s,%(remove)s,notification=Never'
self.cp.set( sect, 'batchsubmit.condornordugrid.condor_attributes', pr )
self.cp.set( sect, 'batchsubmit.condornordugrid.condor_attributes.+remote_queue', self.ce_queue_name )
self.cp.set( sect, 'batchsubmit.condornordugrid.nordugridrsl', self.nordugridrsl )
self.cp.set( sect, 'nordugridrsl.nordugridrsladd', self.rsladd )
self.cp.set( sect, 'nordugridrsl.queue', self.ce_queue_name )
self.cp.set( sect, 'nordugridrsl.addenv.RUCIO_ACCOUNT', 'pilot' )
return self.cp
def __str__(self):
s = "AgisCEQueue: "
s += "PQ=%s " % self.panda_queue_name
s += "wmsqueue=%s " % self.parent.panda_resource
s += "submitplugin=%s " % self.submitplugin
s += "host=%s " % self.ce_host
s += "endpoint=%s " %self.ce_endpoint
s += "gridresource=%s " % self.gridresource
s += "maxtime=%s " % self.parent.maxtime
return s
log.setLevel(logging.DEBUG) # Override with command line switches
if info:
log.setLevel(logging.INFO) # Override with command line switches
log.debug("Logging initialized.")
fconfig=Config()
if fconfig_file is not None:
fconfig_file = os.path.expanduser(fconfig_file)
got_config = fconfig.read(fconfig_file)
log.debug("Read config file %s, return value: %s" % (fconfig_file, got_config))
else:
# Create valid config...
fconfig.add_section('Factory')
# Set unconditional defaults
fconfig.set('Factory', 'config.queues.agis.baseurl', 'http://atlas-agis-api.cern.ch/request/pandaqueue/query/list/?json&preset=schedconf.all' )
fconfig.set('Factory', 'config.queues.agis.sleep', '3600' )
fconfig.set('Factory', 'config.queues.agis.jobsperpilot', '1.5' )
fconfig.set('Factory', 'config.queues.agis.numfactories', '4')
"""
config.queues.agis.vos = atlas
config.queues.agis.clouds = US
config.queues.agis.activities = analysis,production
config.queues.agis.defaultsfiles = /etc/autopyfactory/agisdefaults.conf
"""
# Override defaults with command line values, if given
if vo is not None:
fconfig.set('Factory', 'config.queues.agis.vos', vo)
if cloud is not None:
log.setLevel(logging.INFO) # Override with command line switches
log.debug("Logging initialized.")
fconfig = Config()
if fconfig_file is not None:
fconfig_file = os.path.expanduser(fconfig_file)
got_config = fconfig.read(fconfig_file)
log.debug("Read config file %s, return value: %s" %
(fconfig_file, got_config))
else:
# Create valid config...
fconfig.add_section('Factory')
# Set unconditional defaults
fconfig.set(
'Factory', 'config.queues.agis.baseurl',
'http://atlas-agis-api.cern.ch/request/pandaqueue/query/list/?json&preset=schedconf.all'
)
fconfig.set('Factory', 'config.queues.agis.sleep', '3600')
fconfig.set('Factory', 'config.queues.agis.jobsperpilot', '1.5')
fconfig.set('Factory', 'config.queues.agis.numfactories', '4')
"""
config.queues.agis.vos = atlas
config.queues.agis.clouds = US
config.queues.agis.activities = analysis,production
config.queues.agis.defaultsfiles = /etc/autopyfactory/agisdefaults.conf
"""
# Override defaults with command line values, if given
if vo is not None:
fconfig.set('Factory', 'config.queues.agis.vos', vo)
if cloud is not None:
def getMockQueueConfig():
cp = Config()
cp.add_section('mock')
cp.set('mock','batchstatusplugin', 'Condor')
cp.set('mock','batchstatus.condor.queryargs', 'None')
return cp
def getMockQueueConfig():
cp = Config()
cp.add_section('mock')
cp.set('mock', 'batchstatusplugin', 'Condor')
cp.set('mock', 'batchstatus.condor.queryargs', 'None')
return cp
class AgisCEQueue(object):
"""
Represents a single CE queue within a Panda queue description.
"""
def __init__(self, parent, cedict):
self.log = logging.getLogger('autopyfactory.config')
self.parent = parent
self.panda_queue_name = parent.panda_queue_name
self.ce_name = cedict['ce_name'] # AGLT2-CE-gate04.aglt2.org
self.ce_endpoint = cedict['ce_endpoint'] # gate04.aglt2.org:2119
self.ce_host = self.ce_endpoint.split(":")[0]
self.ce_state = cedict['ce_state'].lower() # 'active'
self.ce_status = cedict['ce_status'].lower() #
self.ce_queue_status = cedict['ce_queue_status'].lower()
self.ce_flavour = cedict['ce_flavour'].lower() # GLOBUS
self.ce_version = cedict['ce_version'].lower() # GT5
self.ce_queue_name = cedict['ce_queue_name'] # default
self.ce_jobmanager = cedict['ce_jobmanager'].lower() # condor
self.ce_queue_maxcputime = cedict['ce_queue_maxcputime'] # in seconds
self.ce_queue_maxwctime = cedict['ce_queue_maxwctime'] # in seconds
self.apf_scale_factor = 1.0
# Empty/default attributes:
self.gridresource = None
self.submitplugin = None
self.submitpluginstr = None
self.gramversion = None
self.gramqueue = None
self.creamenv = None
self.creamattr = ''
self.condorattr = None
self.maxmemory = None
self.maxtime = None
if self.ce_flavour in ['osg-ce', 'globus']:
self._initglobus()
elif self.ce_flavour == 'htcondor-ce':
self._initcondorce()
elif self.ce_flavour == 'cream-ce':
self._initcream()
elif self.ce_flavour == 'arc-ce':
self._initarc()
elif self.ce_flavour == 'lcg-ce':
self.log.debug("Ignoring old CE type 'LCG-CE'")
else:
self.log.warning("CEQueue %s has unknown ce_flavour: %s" %
(self.ce_name, self.ce_flavour))
def _initcondorce(self):
self.gridresource = self.ce_endpoint.split(':')[0]
self.submitplugin = 'CondorOSGCE'
self.submitpluginstr = 'condorosgce'
def _initglobus(self):
self.gridresource = '%s/jobmanager-%s' % (self.ce_endpoint,
self.ce_jobmanager)
if self.ce_version == 'gt2':
self.submitplugin = 'CondorGT2'
self.submitpluginstr = 'condorgt2'
elif self.ce_version == 'gt5':
self.submitplugin = 'CondorGT5'
self.submitpluginstr = 'condorgt5'
self.gramversion = 'gram5'
self.gramqueue = self.ce_queue_name
def _initcream(self):
self.gridresource = '%s/ce-cream/services/CREAM2 %s %s' % (
self.ce_endpoint, self.ce_jobmanager, self.ce_queue_name)
self.submitplugin = 'CondorCREAM'
self.submitpluginstr = 'condorcream'
if self.parent.pilot_version not in ['current']:
self.creamenv = 'RUCIO_ACCOUNT=pilot PILOT_HTTP_SOURCES=%s' % self.parent.pilot_version
else:
self.creamenv = 'RUCIO_ACCOUNT=pilot'
# glue 1.3 uses minutes and this / operator uses floor value
# https://wiki.italiangrid.it/twiki/bin/view/CREAM/UserGuideEMI2#Forward_of_requirements_to_the_b
self.maxtime = self.parent.maxtime / 60
self.maxmemory = self.parent.maxmemory
self.cputime = self.parent.corecount * self.maxtime
# maxrss and maxtime are expected to be set in AGIS for all queues
if self.parent.corecount:
self.creamattr = 'CpuNumber=%d;WholeNodes=false;SMPGranularity=%d;' % (
self.parent.corecount, self.parent.corecount)
if self.parent.corecount:
cputime = self.parent.corecount * self.maxtime
else:
cputime = self.maxtime
self.creamattr += 'CERequirements = "other.GlueCEPolicyMaxCPUTime == %d ' % cputime
self.creamattr += '&& other.GlueCEPolicyMaxWallClockTime == %d ' % self.maxtime
self.creamattr += '&& other.GlueHostMainMemoryRAMSize == %d' % self.parent.maxrss
if self.parent.maxswap:
maxvirtual = self.parent.maxrss + self.parent.maxswap
self.creamattr += ' && other.GlueHostMainMemoryVirtualSize == %d";' % maxvirtual
else:
self.creamattr += '";'
def _initarc(self):
# ignore :port part
self.gridresource = self.ce_endpoint.split(':')[0]
self.submitplugin = 'CondorNordugrid'
self.submitpluginstr = 'condornordugrid'
self.maxmemory = self.parent.maxmemory
#### self.maxtime = self.ce_queue_maxwctime
self.maxtime = self.parent.maxtime
self.nordugridrsl = '(jobname = arc_pilot)'
self.rsladd = '(runtimeenvironment = APPS/HEP/ATLAS-SITE-LCG)(runtimeenvironment = ENV/PROXY)'
self.rsladd += '(count = %d)' % self.parent.corecount
self.rsladd += '(countpernode = %d)' % self.parent.corecount
if self.parent.maxrss:
percore = self.parent.maxrss / self.parent.corecount
self.rsladd += '(memory = %d)' % percore
else:
percore = self.parent.maxmemory / self.parent.corecount
self.rsladd += '(memory = %d)' % percore
if self.maxtime:
self.rsladd += '(walltime = %d)' % self.maxtime
if self.maxtime:
self.rsladd += '(cputime = %d)' % (self.maxtime *
self.parent.corecount)
def getAPFConfigString(self):
"""
Returns string of valid APF configuration for this queue-ce entry.
Calculates scale factor based on how many other CEs serve this PQ
"""
cp = self.getAPFConfig()
sio = StringIO()
s = cp.write(sio)
return sio.getvalue()
def getAPFConfig(self):
"""
Returns ConfigParser object representing config
"""
self.cp = Config()
sect = '%s-%s' % (self.parent.panda_resource, self.ce_host)
sect = str(sect)
self.cp.add_section(sect)
# Unconditional config
self.cp.set(sect, 'enabled', 'True')
self.cp.set(sect, 'batchqueue', self.panda_queue_name)
self.cp.set(sect, 'wmsqueue', self.parent.panda_resource)
self.cp.set(sect, 'batchsubmitplugin', self.submitplugin)
self.cp.set(sect, 'batchsubmit.%s.gridresource' % self.submitpluginstr,
self.gridresource)
#if self.parent.type == 'analysis':
# self.cp.set( sect, 'executable.arguments' , '%(executable.defaultarguments)s -u user' )
#else:
# self.cp.set( sect, 'executable.arguments' , '%(executable.defaultarguments)s -u managed' )
try:
self.apf_scale_factor = (
((1.0 / float(self.parent.parent.numfactories)) /
len(self.parent.ce_queues)) /
float(self.parent.parent.jobsperpilot))
except ZeroDivisionError:
self.log.error(
"Division by zero. Something wrong with scale factory calc.")
self.apf_scale_factor = 1.0
self.cp.set(sect, 'sched.scale.factor', str(self.apf_scale_factor))
#HTCondor CE
if self.ce_flavour == 'htcondor-ce':
if self.parent.maxrss is not None:
self.cp.set(
sect,
'batchsubmit.condorosgce.condor_attributes.+maxMemory',
str(self.parent.maxrss))
else:
self.cp.set(
sect,
'batchsubmit.condorosgce.condor_attributes.+maxMemory',
str(self.parent.maxmemory))
self.cp.set(sect,
'batchsubmit.condorosgce.condor_attributes.+xcount',
str(self.parent.corecount))
self.cp.set(
sect, 'batchsubmit.condorosgce.condor_attributes.+voactivity',
'"%s"' % self.parent.type)
if self.ce_queue_name not in ['default', '']:
self.cp.set(
sect,
'batchsubmit.condorosgce.condor_attributes.+remote_queue',
'"%s"' % self.ce_queue_name)
if self.ce_queue_maxwctime:
self.cp.set(
sect,
'batchsubmit.condorosgce.condor_attributes.+maxWallTime',
'%s' % self.ce_queue_maxwctime)
# Globus
if self.ce_flavour in ['osg-ce', 'globus']:
self.cp.set(sect, 'globusrsl.%s.queue' % self.gramversion,
self.gramqueue)
# Cream-specific JDL
if self.ce_flavour == 'cream-ce':
self.cp.set(sect, 'batchsubmit.condorcream.environ', self.creamenv)
if self.creamattr is not None:
self.cp.set(sect, 'creamattr', self.creamattr)
self.cp.set(
sect, 'batchsubmit.condorcream.condor_attributes',
'%(req)s,%(hold)s,%(remove)s,cream_attributes = %(creamattr)s,notification=Never'
)
else:
self.cp.set(sect, 'batchsubmit.condorcream.condor_attributes',
'%(req)s,%(hold)s,%(remove)s,notification=Never')
# Arc-CE
if self.ce_flavour == 'arc-ce':
pr = 'periodic_remove = (JobStatus == 2 && (CurrentTime - EnteredCurrentStatus) > 604800)'
pr = '%(req)s,%(hold)s,%(remove)s,notification=Never'
self.cp.set(sect, 'batchsubmit.condornordugrid.condor_attributes',
pr)
self.cp.set(
sect,
'batchsubmit.condornordugrid.condor_attributes.+remote_queue',
self.ce_queue_name)
self.cp.set(sect, 'batchsubmit.condornordugrid.nordugridrsl',
self.nordugridrsl)
self.cp.set(sect, 'nordugridrsl.nordugridrsladd', self.rsladd)
self.cp.set(sect, 'nordugridrsl.queue', self.ce_queue_name)
self.cp.set(sect, 'nordugridrsl.addenv.RUCIO_ACCOUNT', 'pilot')
return self.cp
def __str__(self):
s = "AgisCEQueue: "
s += "PQ=%s " % self.panda_queue_name
s += "wmsqueue=%s " % self.parent.panda_resource
s += "submitplugin=%s " % self.submitplugin
s += "host=%s " % self.ce_host
s += "endpoint=%s " % self.ce_endpoint
s += "gridresource=%s " % self.gridresource
s += "maxtime=%s " % self.parent.maxtime
return s