def __init__(self, fcl, am):
self.fcl = fcl
self.mcl = Config()
self.mcl.add_section('MockMonitor')
self.mcl.set('MockMonitor', 'monitorURL', '')
self.threadsregistry = ThreadsRegistry()
self.authmanager = am
class FactoryMock(object):
def __init__(self, fcl, am):
self.fcl = fcl
self.mcl = Config()
self.mcl.add_section('MockMonitor')
self.mcl.set('MockMonitor','monitorURL','')
self.threadsregistry = ThreadsRegistry()
self.authmanager = am
def getConfig(self):
"""
get updated configuration from the Factory Config plugins
"""
newqcl = Config()
for config_plugin in self.factory.config_plugins:
tmpqcl = config_plugin.getConfig()
newqcl.merge(tmpqcl)
return newqcl
def getAuthConfig(self):
"""
get updated configuration from the Factory Config/Auth plugins
"""
self.log.debug('starting')
newconfig = Config()
for config_plugin in self.factory.auth_config_plugins:
tmpconfig = config_plugin.getConfig()
newconfig.merge(tmpconfig)
self.log.debug('leaving with newconf = %s with %s sections.' % ( newconfig, len(newconfig.sections())))
return newconfig
def getConfigWMSQueue(self, wmsqueue):
"""
get the config sections only for a given wmsqueue
"""
conf = self.getConfig()
out = Config()
for section_name in conf.sections():
section = conf.getSection(section_name)
if section.get(section_name, 'wmsqueue') == wmsqueue:
out.merge(section)
return out
def getConfigWMSQueue(self, wmsqueue):
"""
get the config sections only for a given wmsqueue
"""
conf = self.getConfig()
out = Config()
for section_name in conf.sections():
section = conf.getSection(section_name)
if section.get(section_name, 'wmsqueue') == wmsqueue:
out.merge(section)
return out
def __init__(self, factory=None):
self.log = logging.getLogger('autopyfactory')
self.log.info("Creating new authmanager...")
self.aconfig = Config()
self.handlers = []
self.factory = factory
if factory:
self.sleep = int(
self.factory.fcl.get('Factory', 'authmanager.sleep'))
else:
self.sleep = 5
def __init__(self, factory):
_thread.__init__(self)
self.log = logging.getLogger('autopyfactory.confighandler')
self.factory = factory
self.reconfig = factory.fcl.generic_get('Factory', 'config.reconfig', 'getboolean', default_value=False)
self.authmanagerenabled = factory.fcl.generic_get('Factory', 'authmanager.enabled', 'getboolean', default_value=False)
self.interval = None
if self.reconfig:
self.interval = factory.fcl.generic_get('Factory','config.reconfig.interval', 'getint', default_value=3600)
self.qcl = Config()
self.acl = Config()
class AuthManager(object):
"""
Manager to maintain multiple credential Handlers, one for each target account.
For some handlers, if they need to perform periodic checks, they will be run
as threads. Others, which only hold information, will just be objects.
"""
def __init__(self, factory=None):
self.log = logging.getLogger('autopyfactory')
self.log.info("Creating new authmanager...")
self.aconfig = Config()
self.handlers = []
self.factory = factory
if factory:
self.sleep = int(
self.factory.fcl.get('Factory', 'authmanager.sleep'))
else:
self.sleep = 5
def reconfig(self, newconfig):
hdiff = ConfigsDiff(self.aconfig, newconfig)
self.aconfig = newconfig
self._addhandlers(hdiff.added())
def _addhandlers(self, newsections):
for sect in newsections:
try:
pclass = self.aconfig.get(sect, 'plugin')
except Exception, e:
self.log.warn("No plugin attribute for section %s" % sect)
if pclass == 'X509':
self.log.debug("Creating X509 handler for %s" % sect)
authpluginname = self.aconfig.get(sect, 'plugin')
x509h = pluginmanager.getplugin(
['autopyfactory', 'plugins', 'authmanager', 'auth'],
authpluginname, self, self.aconfig, sect)
self.handlers.append(x509h)
elif pclass == 'SSH':
self.log.debug("Creating SSH handler for %s" % sect)
authpluginname = self.aconfig.get(sect, 'plugin')
sshh = pluginmanager.getplugin(
['autopyfactory', 'plugins', 'authmanager', 'auth'],
authpluginname, self, self.aconfig, sect)
self.handlers.append(sshh)
else:
self.log.warn("Unrecognized auth plugin %s" % pclass)
def _createConfig(self):
'''
Creates and sets self.currentconfig and self.stringconfig as configured by files.
'''
td = datetime.datetime.now() - self.lastconfigupdate
totalseconds = td.seconds + (td.days * 24 * 3600)
if totalseconds > self.sleep:
self.log.debug("Configs older than %s seconds. Re-creating..." %
self.sleep)
self._filter()
cp = Config()
sc = '' # New string config.
for i in range(len(self.activities)):
tmpcp = Config()
vo = self.vos[i]
cloud = self.clouds[i]
activity = self.activities[i]
default = self.defaultsfiles[i]
pilotmgr = self.pilotmanager[i]
if default is not None:
tmpfile = open(default)
tmpcp.readfp(tmpfile)
tmpfile.seek(0) # to read the file over again
for line in tmpfile.readlines():
sc += line
for q in self.currentinfo:
if q.vo_name == vo and\
q.cloud == cloud and\
q.type == activity and\
q.pilot_manager == pilotmgr :
for cq in q.ce_queues:
try:
qc = cq.getAPFConfig()
tmpcp.merge(qc)
# add content of Config object to the string representation
sc += "\n"
sc += qc.getContent()
except Exception as e:
self.log.error('Captured exception %s' % e)
cp.merge(tmpcp)
self.currentconfig = cp
self.stringconfig = sc
else:
self.log.debug("Configs up to date.")
def __init__(self, config, authman=None):
#self.log = logging.getLogger('%s' % config.sections()[0])
self.qcl = config
self.apfqname = config.sections()[0]
#if len(self.log.parent.handlers) < 1:
# logStream = logging.StreamHandler()
# FORMAT='%(asctime)s (UTC) [ %(levelname)s ] %(name)s %(filename)s:%(lineno)d %(funcName)s(): %(message)s'
# formatter = logging.Formatter(FORMAT)
# formatter.converter = time.gmtime # to convert timestamps to UTC
# logStream.setFormatter(formatter)
# self.log.addHandler(logStream)
# self.log.setLevel(logging.DEBUG)
# Mock objects
logging.debug("Creating config for factory mock.")
fcl = Config()
fconf = self.qcl.get(self.apfqname, 'factoryconf')
okread = fcl.read(fconf)
logging.debug("Successfully read %s " % okread)
####from autopyfactory.threadsmanagement import ThreadsRegistry
####class FactoryMock(object):
####
#### def __init__(self, fcl, am):
#### self.fcl = fcl
#### self.mcl = Config()
#### self.mcl.add_section('MockMonitor')
#### self.mcl.set('MockMonitor','monitorURL','')
#### self.threadsregistry = ThreadsRegistry()
#### self.authmanager = am
####
####self.factory = FactoryMock(fcl, authman)
from autopyfactory.factory import Factory
self.factory = Factory.getFactoryMock(fcl, authman)
logging.debug('APFQueue init: initial configuration:\n%s' %
self.qcl.getSection(self.apfqname).getContent())
try:
self._plugins()
except Exception, ex:
logging.error('APFQueue: Exception getting plugins')
raise ex
def __init__(self, **kwarg):
self.log = core_utils.make_logger(baseLogger)
self.config = Config()
factoryconffile = os.path.expanduser('~/harvester/etc/autopyfactory/autopyfactory.conf')
self.log.debug('Reading config: %s' % factoryconffile)
okread = self.config.read(factoryconffile)
self.log.debug('Successfully read %s' % okread)
# Setup Authmanager
authconfigfile = os.path.expanduser(self.config.get('Factory','authConf'))
ac = Config()
self.log.debug('Reading config: %s' % authconfigfile)
okread = ac.read(authconfigfile)
self.log.debug('Successfully read %s' % okread)
if APFGridSubmitter.authman is None :
APFGridSubmitter.authman = AuthManager()
APFGridSubmitter.authman.reconfig(ac)
self.authman = APFGridSubmitter.authman
APFGridSubmitter.authman.activate()
# Setup factory mock object.
from autopyfactory.factory import Factory
if APFGridSubmitter.factorymock is None:
APFGridSubmitter.factorymock = Factory.getFactoryMock(fcl=self.config, am=self.authman)
# Setup AGIS
if APFGridSubmitter.agis is None:
APFGridSubmitter.agisobj = Agis(None, self.config, None)
self.agisobj = APFGridSubmitter.agisobj
self.log.debug("AGIS object: %s" % self.agisobj)
# Setup logserver
self.log.debug("Handling LogServer...")
if self.config.generic_get('Factory', 'logserver.enabled', 'getboolean'):
self.log.info("LogServer enabled. Initializing...")
self.logserver = LogServer(self.config)
self.log.info('LogServer initialized. Starting...')
self.logserver.start()
self.log.debug('LogServer thread started.')
else:
self.log.info('LogServer disabled. Not running.')
self.log.debug('APFGridSubmitter initialized.')
def __init__(self, factory=None):
self.log = logging.getLogger('autopyfactory')
self.log.info("Creating new authmanager...")
self.aconfig = Config()
self.handlers = []
self.factory = factory
if factory:
self.sleep = int(self.factory.fcl.get('Factory', 'authmanager.sleep'))
else:
self.sleep = 5
def __init__(self, fcl):
"""
fcl is a FactoryConfigLoader object.
"""
self.version = __version__
self.log = logging.getLogger('autopyfactory')
self.log.info('AutoPyFactory version %s' % self.version)
self.fcl = fcl
# to decide if the main loop needs to stop or not
self.shutdown = False
# threads registry
self.threadsregistry = ThreadsRegistry()
# APF Queues Manager
self.apfqueuesmanager = APFQueuesManager(self)
self._authmanager()
self._queues_monitor_conf()
self._mappings()
# Handle Log Serving
self._initLogserver()
# Collect other factory attibutes
self.adminemail = self.fcl.get('Factory', 'factoryAdminEmail')
self.factoryid = self.fcl.get('Factory', 'factoryId')
self.smtpserver = self.fcl.get('Factory', 'factorySMTPServer')
miners = self.fcl.generic_get('Factory',
'factoryMinEmailRepeatSeconds',
get_function='getint',
default_value=3600)
self.minemailrepeat = int(miners)
# Dictionary of emails sent, indexed by subject + message, with value
# being a Date object of when it was sent.
self.emaildict = {}
self.hostname = socket.gethostname()
self.username = pwd.getpwuid(os.getuid()).pw_name
# to shutdown the factory when there are no active queues
self.abort_no_queues = self.fcl.generic_get('Factory',
'abort_no_queues',
get_function='getboolean',
default_value=False)
# the the queues config loader object, to be filled by a Config plugin
self.qcl = Config()
self._plugins()
# Log some info...
self.log.debug('Factory shell PATH: %s' % os.getenv('PATH'))
self.log.info("Factory: Object initialized.")
def __init__(self, config, authman=None):
self.qcl = config
self.apfqname = config.sections()[0]
# Mock objects
logging.debug("Creating config for factory mock.")
fcl = Config()
fconf = self.qcl.get(self.apfqname, 'factoryconf')
okread = fcl.read(fconf)
logging.debug("Successfully read %s " % okread)
from autopyfactory.factory import Factory
self.factory = Factory.getFactoryMock(fcl, authman)
logging.debug('APFQueue init: initial configuration:\n%s' %self.qcl.getSection(self.apfqname).getContent())
try:
self._plugins()
except Exception as ex:
logging.error('APFQueue: Exception getting plugins' )
raise ex
logging.debug('APFQueue: Object initialized.')
def _createConfig(self):
'''
Creates and sets self.currentconfig and self.stringconfig as configured by files.
'''
td = datetime.datetime.now() - self.lastconfigupdate
totalseconds = td.seconds + ( td.days * 24 * 3600)
if totalseconds > self.sleep:
self.log.debug("Configs older than %s seconds. Re-creating..." % self.sleep)
self._filter()
cp = Config()
sc = '' # New string config.
for i in range(len(self.activities)):
tmpcp = Config()
vo = self.vos[i]
cloud = self.clouds[i]
activity = self.activities[i]
default = self.defaultsfiles[i]
pilotmgr = self.pilotmanager[i]
if default is not None:
tmpfile = open(default)
tmpcp.readfp(tmpfile)
tmpfile.seek(0) # to read the file over again
for line in tmpfile.readlines():
sc += line
for q in self.currentinfo:
if q.vo_name == vo and\
q.cloud == cloud and\
q.type == activity and\
q.pilot_manager == pilotmgr :
for cq in q.ce_queues:
try:
qc = cq.getAPFConfig()
tmpcp.merge(qc)
# add content of Config object to the string representation
sc += "\n"
sc += qc.getContent()
except Exception as e:
self.log.error('Captured exception %s' % e)
cp.merge(tmpcp)
self.currentconfig = cp
self.stringconfig = sc
else:
self.log.debug("Configs up to date.")
def getMockQueueConfig():
cp = Config()
cp.add_section('mock')
cp.set('mock','batchstatusplugin', 'Condor')
cp.set('mock','batchstatus.condor.queryargs', 'None')
return cp
def getMockFactoryConfig():
cp = Config()
cp.add_section('Factory')
cp.set('Factory', 'batchstatus.condor.maxage', '60' )
cp.set('Factory', 'batchstatus.condor.sleep', '30' )
cp.set('Factory', 'factoryUser', 'autopyfactory' )
cp.set('Factory', 'factoryId', 'dummy-mock' )
return cp
class APFGridSubmitter(object):
__metaclass__ = APFGridSubmitterSingleton
factorymock = None
authman = None
agis = None
logserver = None
cleanlogs = None
def __init__(self, **kwarg):
self.log = core_utils.make_logger(baseLogger)
self.config = Config()
factoryconffile = os.path.expanduser('~/harvester/etc/autopyfactory/autopyfactory.conf')
self.log.debug('Reading config: %s' % factoryconffile)
okread = self.config.read(factoryconffile)
self.log.debug('Successfully read %s' % okread)
# Setup Authmanager
authconfigfile = os.path.expanduser(self.config.get('Factory','authConf'))
ac = Config()
self.log.debug('Reading config: %s' % authconfigfile)
okread = ac.read(authconfigfile)
self.log.debug('Successfully read %s' % okread)
if APFGridSubmitter.authman is None :
APFGridSubmitter.authman = AuthManager()
APFGridSubmitter.authman.reconfig(ac)
self.authman = APFGridSubmitter.authman
APFGridSubmitter.authman.activate()
# Setup factory mock object.
from autopyfactory.factory import Factory
if APFGridSubmitter.factorymock is None:
APFGridSubmitter.factorymock = Factory.getFactoryMock(fcl=self.config, am=self.authman)
# Setup AGIS
if APFGridSubmitter.agis is None:
APFGridSubmitter.agisobj = Agis(None, self.config, None)
self.agisobj = APFGridSubmitter.agisobj
self.log.debug("AGIS object: %s" % self.agisobj)
# Setup logserver
self.log.debug("Handling LogServer...")
if self.config.generic_get('Factory', 'logserver.enabled', 'getboolean'):
self.log.info("LogServer enabled. Initializing...")
self.logserver = LogServer(self.config)
self.log.info('LogServer initialized. Starting...')
self.logserver.start()
self.log.debug('LogServer thread started.')
else:
self.log.info('LogServer disabled. Not running.')
self.log.debug('APFGridSubmitter initialized.')
def _print_config(self, config):
s=""
for section in config.sections():
s+= "[%s]\n" % section
for opt in config.options(section):
s+="%s = %s\n" % (opt, config.get(section, opt))
return s
def submit_workers(self, workspec_list):
'''
Submit workers to a scheduling system like batch systems and computing elements.
This method takes a list of WorkSpecs as input argument, and returns a list of tuples.
Each tuple is composed of a return code and a dialog message.
Nth tuple in the returned list corresponds to submission status and dialog message for Nth worker
in the given WorkSpec list.
A unique identifier is set to WorkSpec.batchID when submission is successful,
so that they can be identified in the scheduling system.
:param workspec_list: a list of work specs instances
:return: A list of tuples. Each tuple is composed of submission status (True for success, False otherwise)
and dialog message
:rtype: [(bool, string),]
For UPQ
PQ name, e.g. HARVESTER_BNL_APF_TEST/MCORE
SCORE, SCORE_HIMEM, MCORE, and MCORE_HIMEM
Workspec AGIS PQ/CEQ attribute.
WorkSpec.minRamCount PQ.memory
WorkSpec.nCore PQ.corecount
WorkSpec.maxDiskCount PQ.maxwdir
WorkSpec.maxWalltime PQ.maxtime
'''
self.log.debug('start nWorkers={0}'.format(len(workspec_list)))
self.log.debug("Update AGIS info...")
qc = self.agisobj.getConfig()
#self.log.debug('Agis config output %s' % self._print_config(qc))
#self.log.debug('Agis config defaults= %s' % qc.defaults() )
retlist = []
# wsmap is indexed by computing site:
# { <computingsite> : [ws1, ws2, ws3 ],
# <computingsite2> : [ws4, ws5]
# }
wsmap = {}
jobmap = {}
try:
self.log.debug("Handling workspec_list with %d items..." % len(workspec_list))
for workSpec in workspec_list:
self.log.debug("Worker(workerId=%s queueName=%s computingSite=%s nCore=%s status=%s " % (workSpec.workerID,
workSpec.queueName,
workSpec.computingSite,
workSpec.nCore,
workSpec.status) )
try:
wslist = wsmap[workSpec.computingSite]
wslist.append(workSpec)
except KeyError:
wsmap[workSpec.computingSite] = [workSpec]
self.log.debug("wsmap = %s" % wsmap)
for pq in wsmap.keys():
found = False
apfqsections = []
for s in qc.sections():
qcq = qc.get(s, 'wmsqueue').strip()
self.log.debug('Checking %s' % qcq)
if qcq == pq:
found = True
apfqsections.append(s)
self.log.debug("Found a queues config for %s" % pq )
self.log.info("Found %d sections for PQ %s" % (len(apfqsections), pq))
if found:
# make apfq and submit
self.log.debug("One or more Agis configs found for PQ. Choosing one...")
section = random.choice(apfqsections)
pqc = qc.getSection(section)
ac = os.path.expanduser('~/harvester/etc/autopyfactory/autopyfactory.conf')
pqc.set(section, 'factoryconf', ac)
self.log.debug("Section config= %s" % pqc)
self.log.debug("Making APF queue for PQ %s with label %s"% (pq, section))
apfq = SubmitAPFQueue( pqc, self.authman )
self.log.debug("Successfully made APFQueue")
joblist = []
for ws in wsmap[pq]:
jobentry = { "+workerid" : ws.workerID }
joblist.append(jobentry)
self.log.debug("joblist made= %s. Submitting..." % joblist)
jobinfo = apfq.submitlist(joblist)
self.log.debug("Got jobinfo %s" % jobinfo)
wslist = wsmap[pq]
self.log.debug("wslist for pq %s is length %s" % (pq, len(wslist)))
for i in range(0, len(wslist)):
self.log.debug("Setting ws.batchID to %s" % jobinfo[i].jobid )
wslist[i].batchID = jobinfo[i].jobid
wslist[i].set_status(WorkSpec.ST_submitted)
retlist.append((True, ''))
else:
self.log.info('No AGIS config found for PQ %s skipping.' % pq)
except Exception, e:
self.log.error(traceback.format_exc(None))
self.log.debug("return list=%s " % retlist)
return retlist
def getAPFConfig(self):
"""
Returns ConfigParser object representing config
"""
self.cp = Config()
sect = '%s-%s' % ( self.parent.panda_resource, self.ce_host )
sect = str(sect)
self.cp.add_section(sect)
# Unconditional config
self.cp.set( sect, 'enabled', 'True')
self.cp.set( sect, 'batchqueue', self.panda_queue_name)
self.cp.set( sect, 'wmsqueue', self.parent.panda_resource )
self.cp.set( sect, 'batchsubmitplugin', self.submitplugin )
self.cp.set( sect, 'batchsubmit.%s.gridresource' % self.submitpluginstr , self.gridresource )
#if self.parent.type == 'analysis':
# self.cp.set( sect, 'executable.arguments' , '%(executable.defaultarguments)s -u user' )
#else:
# self.cp.set( sect, 'executable.arguments' , '%(executable.defaultarguments)s -u managed' )
try:
self.apf_scale_factor = ((( 1.0 / float(self.parent.parent.numfactories) ) / len(self.parent.ce_queues) ) / float(self.parent.parent.jobsperpilot) )
except ZeroDivisionError:
self.log.error("Division by zero. Something wrong with scale factory calc.")
self.apf_scale_factor = 1.0
self.cp.set( sect, 'sched.scale.factor', str(self.apf_scale_factor) )
#HTCondor CE
if self.ce_flavour == 'htcondor-ce':
pr = 'periodic_remove = (JobStatus == 2 && (CurrentTime - EnteredCurrentStatus) > 604800)'
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes', pr )
if self.parent.maxrss is not None:
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes.+maxMemory', str(self.parent.maxrss) )
else:
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes.+maxMemory', str(self.parent.maxmemory) )
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes.+xcount', str(self.parent.corecount) )
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes.+voactivity', '"%s"' % self.parent.type )
if self.ce_queue_name not in ['default','']:
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes.+remote_queue', '"%s"' % self.ce_queue_name)
if self.ce_queue_maxwctime:
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes.+maxWallTime', '%s' % self.ce_queue_maxwctime)
# Globus
if self.ce_flavour in ['osg-ce','globus']:
self.cp.set( sect, 'globusrsl.%s.queue' % self.gramversion, self.gramqueue )
# Cream-specific JDL
if self.ce_flavour == 'cream-ce':
self.cp.set( sect, 'batchsubmit.condorcream.environ', self.creamenv )
if self.creamattr is not None:
self.cp.set(sect, 'creamattr', self.creamattr)
self.cp.set(sect, 'batchsubmit.condorcream.condor_attributes' , '%(req)s,%(hold)s,%(remove)s,cream_attributes = %(creamattr)s,notification=Never' )
else:
self.cp.set(sect, 'batchsubmit.condorcream.condor_attributes' , '%(req)s,%(hold)s,%(remove)s,notification=Never' )
# Arc-CE
if self.ce_flavour == 'arc-ce':
pr = 'periodic_remove = (JobStatus == 2 && (CurrentTime - EnteredCurrentStatus) > 604800)'
pr = '%(req)s,%(hold)s,%(remove)s,notification=Never'
self.cp.set( sect, 'batchsubmit.condornordugrid.condor_attributes', pr )
self.cp.set( sect, 'batchsubmit.condornordugrid.condor_attributes.+remote_queue', self.ce_queue_name )
self.cp.set( sect, 'batchsubmit.condornordugrid.nordugridrsl', self.nordugridrsl )
self.cp.set( sect, 'nordugridrsl.nordugridrsladd', self.rsladd )
self.cp.set( sect, 'nordugridrsl.queue', self.ce_queue_name )
self.cp.set( sect, 'nordugridrsl.addenv.RUCIO_ACCOUNT', 'pilot' )
return self.cp
class AgisCEQueue(object):
"""
Represents a single CE queue within a Panda queue description.
"""
def __init__(self, parent, cedict):
self.log = logging.getLogger('autopyfactory.config')
self.parent = parent
self.panda_queue_name = parent.panda_queue_name
self.ce_name = cedict['ce_name'] # AGLT2-CE-gate04.aglt2.org
self.ce_endpoint = cedict['ce_endpoint'] # gate04.aglt2.org:2119
self.ce_host = self.ce_endpoint.split(":")[0]
self.ce_state = cedict['ce_state'].lower() # 'active'
self.ce_status = cedict['ce_status'].lower() #
self.ce_queue_status = cedict['ce_queue_status'].lower()
self.ce_flavour = cedict['ce_flavour'].lower() # GLOBUS
self.ce_version = cedict['ce_version'].lower() # GT5
self.ce_queue_name = cedict['ce_queue_name'] # default
self.ce_jobmanager = cedict['ce_jobmanager'].lower() # condor
self.ce_queue_maxcputime = cedict['ce_queue_maxcputime'] # in seconds
self.ce_queue_maxwctime = cedict['ce_queue_maxwctime'] # in seconds
self.apf_scale_factor = 1.0
# Empty/default attributes:
self.gridresource = None
self.submitplugin = None
self.submitpluginstr = None
self.gramversion = None
self.gramqueue = None
self.creamenv = None
self.creamattr = ''
self.condorattr = None
self.maxmemory = None
self.maxtime = None
if self.ce_flavour in ['osg-ce','globus']:
self._initglobus()
elif self.ce_flavour == 'htcondor-ce':
self._initcondorce()
elif self.ce_flavour == 'cream-ce':
self._initcream()
elif self.ce_flavour == 'arc-ce':
self._initarc()
elif self.ce_flavour == 'lcg-ce':
self.log.debug("Ignoring old CE type 'LCG-CE'")
else:
self.log.warning("CEQueue %s has unknown ce_flavour: %s" % (self.ce_name, self.ce_flavour))
def _initcondorce(self):
self.gridresource = self.ce_endpoint.split(':')[0]
self.submitplugin = 'CondorOSGCE'
self.submitpluginstr = 'condorosgce'
def _initglobus(self):
self.gridresource = '%s/jobmanager-%s' % (self.ce_endpoint, self.ce_jobmanager)
if self.ce_version == 'gt2':
self.submitplugin = 'CondorGT2'
self.submitpluginstr = 'condorgt2'
elif self.ce_version == 'gt5':
self.submitplugin = 'CondorGT5'
self.submitpluginstr = 'condorgt5'
self.gramversion = 'gram5'
self.gramqueue = self.ce_queue_name
def _initcream(self):
self.gridresource = '%s/ce-cream/services/CREAM2 %s %s' % (self.ce_endpoint,
self.ce_jobmanager,
self.ce_queue_name)
self.submitplugin = 'CondorCREAM'
self.submitpluginstr = 'condorcream'
if self.parent.pilot_version not in ['current']:
self.creamenv = 'RUCIO_ACCOUNT=pilot PILOT_HTTP_SOURCES=%s' % self.parent.pilot_version
else:
self.creamenv = 'RUCIO_ACCOUNT=pilot'
# glue 1.3 uses minutes and this / operator uses floor value
# https://wiki.italiangrid.it/twiki/bin/view/CREAM/UserGuideEMI2#Forward_of_requirements_to_the_b
self.maxtime = self.parent.maxtime / 60
self.maxmemory = self.parent.maxmemory
self.cputime = self.parent.corecount * self.maxtime
# maxrss and maxtime are expected to be set in AGIS for all queues
if self.parent.corecount:
self.creamattr = 'CpuNumber=%d;WholeNodes=false;SMPGranularity=%d;' % (self.parent.corecount,
self.parent.corecount)
if self.parent.corecount:
cputime = self.parent.corecount * self.maxtime
else:
cputime = self.maxtime
self.creamattr += 'CERequirements = "other.GlueCEPolicyMaxCPUTime == %d ' % cputime
self.creamattr += '&& other.GlueCEPolicyMaxWallClockTime == %d ' % self.maxtime
self.creamattr += '&& other.GlueHostMainMemoryRAMSize == %d' % self.parent.maxrss
if self.parent.maxswap:
maxvirtual = self.parent.maxrss + self.parent.maxswap
self.creamattr += ' && other.GlueHostMainMemoryVirtualSize == %d";' % maxvirtual
else:
self.creamattr += '";'
def _initarc(self):
# ignore :port part
self.gridresource = self.ce_endpoint.split(':')[0]
self.submitplugin = 'CondorNordugrid'
self.submitpluginstr = 'condornordugrid'
self.maxmemory = self.parent.maxmemory
#### self.maxtime = self.ce_queue_maxwctime
self.maxtime = self.parent.maxtime
self.nordugridrsl = '(jobname = arc_pilot)'
self.rsladd = '(runtimeenvironment = APPS/HEP/ATLAS-SITE-LCG)(runtimeenvironment = ENV/PROXY)'
self.rsladd += '(count = %d)' % self.parent.corecount
self.rsladd += '(countpernode = %d)' % self.parent.corecount
if self.parent.maxrss:
percore = self.parent.maxrss/self.parent.corecount
self.rsladd += '(memory = %d)' % percore
else:
percore = self.parent.maxmemory/self.parent.corecount
self.rsladd += '(memory = %d)' % percore
if self.maxtime:
self.rsladd += '(walltime = %d)' % self.maxtime
if self.maxtime:
self.rsladd += '(cputime = %d)' % (self.maxtime * self.parent.corecount)
def getAPFConfigString(self):
"""
Returns string of valid APF configuration for this queue-ce entry.
Calculates scale factor based on how many other CEs serve this PQ
"""
cp = self.getAPFConfig()
sio = StringIO()
s = cp.write(sio)
return sio.getvalue()
def getAPFConfig(self):
"""
Returns ConfigParser object representing config
"""
self.cp = Config()
sect = '%s-%s' % ( self.parent.panda_resource, self.ce_host )
sect = str(sect)
self.cp.add_section(sect)
# Unconditional config
self.cp.set( sect, 'enabled', 'True')
self.cp.set( sect, 'batchqueue', self.panda_queue_name)
self.cp.set( sect, 'wmsqueue', self.parent.panda_resource )
self.cp.set( sect, 'batchsubmitplugin', self.submitplugin )
self.cp.set( sect, 'batchsubmit.%s.gridresource' % self.submitpluginstr , self.gridresource )
#if self.parent.type == 'analysis':
# self.cp.set( sect, 'executable.arguments' , '%(executable.defaultarguments)s -u user' )
#else:
# self.cp.set( sect, 'executable.arguments' , '%(executable.defaultarguments)s -u managed' )
try:
self.apf_scale_factor = ((( 1.0 / float(self.parent.parent.numfactories) ) / len(self.parent.ce_queues) ) / float(self.parent.parent.jobsperpilot) )
except ZeroDivisionError:
self.log.error("Division by zero. Something wrong with scale factory calc.")
self.apf_scale_factor = 1.0
self.cp.set( sect, 'sched.scale.factor', str(self.apf_scale_factor) )
#HTCondor CE
if self.ce_flavour == 'htcondor-ce':
pr = 'periodic_remove = (JobStatus == 2 && (CurrentTime - EnteredCurrentStatus) > 604800)'
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes', pr )
if self.parent.maxrss is not None:
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes.+maxMemory', str(self.parent.maxrss) )
else:
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes.+maxMemory', str(self.parent.maxmemory) )
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes.+xcount', str(self.parent.corecount) )
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes.+voactivity', '"%s"' % self.parent.type )
if self.ce_queue_name not in ['default','']:
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes.+remote_queue', '"%s"' % self.ce_queue_name)
if self.ce_queue_maxwctime:
self.cp.set( sect, 'batchsubmit.condorosgce.condor_attributes.+maxWallTime', '%s' % self.ce_queue_maxwctime)
# Globus
if self.ce_flavour in ['osg-ce','globus']:
self.cp.set( sect, 'globusrsl.%s.queue' % self.gramversion, self.gramqueue )
# Cream-specific JDL
if self.ce_flavour == 'cream-ce':
self.cp.set( sect, 'batchsubmit.condorcream.environ', self.creamenv )
if self.creamattr is not None:
self.cp.set(sect, 'creamattr', self.creamattr)
self.cp.set(sect, 'batchsubmit.condorcream.condor_attributes' , '%(req)s,%(hold)s,%(remove)s,cream_attributes = %(creamattr)s,notification=Never' )
else:
self.cp.set(sect, 'batchsubmit.condorcream.condor_attributes' , '%(req)s,%(hold)s,%(remove)s,notification=Never' )
# Arc-CE
if self.ce_flavour == 'arc-ce':
pr = 'periodic_remove = (JobStatus == 2 && (CurrentTime - EnteredCurrentStatus) > 604800)'
pr = '%(req)s,%(hold)s,%(remove)s,notification=Never'
self.cp.set( sect, 'batchsubmit.condornordugrid.condor_attributes', pr )
self.cp.set( sect, 'batchsubmit.condornordugrid.condor_attributes.+remote_queue', self.ce_queue_name )
self.cp.set( sect, 'batchsubmit.condornordugrid.nordugridrsl', self.nordugridrsl )
self.cp.set( sect, 'nordugridrsl.nordugridrsladd', self.rsladd )
self.cp.set( sect, 'nordugridrsl.queue', self.ce_queue_name )
self.cp.set( sect, 'nordugridrsl.addenv.RUCIO_ACCOUNT', 'pilot' )
return self.cp
def __str__(self):
s = "AgisCEQueue: "
s += "PQ=%s " % self.panda_queue_name
s += "wmsqueue=%s " % self.parent.panda_resource
s += "submitplugin=%s " % self.submitplugin
s += "host=%s " % self.ce_host
s += "endpoint=%s " %self.ce_endpoint
s += "gridresource=%s " % self.gridresource
s += "maxtime=%s " % self.parent.maxtime
return s
class AuthManager(object):
"""
Manager to maintain multiple credential Handlers, one for each target account.
For some handlers, if they need to perform periodic checks, they will be run
as threads. Others, which only hold information, will just be objects.
"""
def __init__(self, factory=None):
self.log = logging.getLogger('autopyfactory')
self.log.info("Creating new authmanager...")
self.aconfig = Config()
self.handlers = []
self.factory = factory
if factory:
self.sleep = int(self.factory.fcl.get('Factory', 'authmanager.sleep'))
else:
self.sleep = 5
def reconfig(self, newconfig):
hdiff = ConfigsDiff(self.aconfig, newconfig)
self.aconfig = newconfig
self._addhandlers(hdiff.added())
def _addhandlers(self, newsections):
for sect in newsections:
try:
pclass = self.aconfig.get(sect, 'plugin')
except Exception as e:
self.log.warn("No plugin attribute for section %s" % sect)
if pclass == 'X509':
self.log.debug("Creating X509 handler for %s" % sect )
authpluginname = self.aconfig.get(sect, 'plugin')
x509h = pluginmanager.getplugin(['autopyfactory', 'plugins', 'authmanager', 'auth'], authpluginname, self, self.aconfig, sect)
self.handlers.append(x509h)
elif pclass == 'SSH':
self.log.debug("Creating SSH handler for %s" % sect )
authpluginname = self.aconfig.get(sect, 'plugin')
sshh = pluginmanager.getplugin(['autopyfactory', 'plugins', 'authmanager', 'auth'], authpluginname, self, self.aconfig, sect)
self.handlers.append(sshh)
else:
self.log.warn("Unrecognized auth plugin %s" % pclass )
def activate(self):
"""
start all Handlers, if needed
"""
for ah in self.handlers:
if isinstance(ah, threading.Thread) :
self.log.debug("Handler [%s] is a thread. Starting..." % ah.name)
ah.start()
else:
self.log.debug("Handler [%s] is not a thread. No action." % ah.name)
def listNames(self):
"""
Returns list of valid names of Handlers in this Manager.
"""
names = []
for h in self.handlers:
names.append(h.name)
return names
#
# API for X509Handler
#
def getProxyPath(self, profilelist):
"""
Check all the handlers for matching profile name(s).
profiles argument is a list
"""
pp = None
for profile in profilelist:
self.log.debug("Getting proxy path for profile %s" % profile)
ph = None
for h in self.handlers:
self.log.debug("Finding handler. Checking %s" % h.name)
if h.name == profile:
ph = h
break
if ph:
self.log.debug("Found handler %s. Getting proxypath..." % ph.name)
pp = ph.getProxyPath()
self.log.debug("Proxypath is %s" % pp)
if pp:
break
if not pp:
subject = "Proxy problem on %s" % self.factory.factoryid
messagestring = "Unable to get valid proxy from configured profiles: %s" % profilelist
self.factory.sendAdminEmail(subject, messagestring)
raise InvalidAuthFailure("Problem getting proxy for profile %s" % profilelist)
return pp
#
# API for SSHKeyHandler
#
def getSSHKeyPair(self, profile):
"""
Returns tuple (public, private, pass) key/phrase string from profile.
"""
pass
def getSSHKeyPairPaths(self, profile):
"""
Returns tuple (public, private, pass) key/passfile paths to files from profile.
"""
h = self._getHandler(profile)
pub = h.getSSHPubKeyFilePath()
priv = h.getSSHPrivKeyFilePath()
pasf = h.getSSHPassFilePath()
self.log.info('Got file paths for pub, priv, pass for SSH profile %s' % profile)
return (pub,priv,pasf)
def _getHandler(self, profile):
"""
"""
handler = None
for h in self.handlers:
self.log.debug("Finding handler. Checking %s" % h.name)
if h.name == profile:
self.log.debug("Found handler for %s" % h.name)
handler = h
if handler is None:
raise InvalidAuthFailure('No handler for %s ' % profile)
return handler
def getMockFactoryConfig():
cp = Config()
cp.add_section('Factory')
cp.set('Factory', 'batchstatus.condor.maxage', '60')
cp.set('Factory', 'batchstatus.condor.sleep', '30')
cp.set('Factory', 'factoryUser', 'autopyfactory')
cp.set('Factory', 'factoryId', 'dummy-mock')
return cp
def getMockQueueConfig():
cp = Config()
cp.add_section('mock')
cp.set('mock', 'batchstatusplugin', 'Condor')
cp.set('mock', 'batchstatus.condor.queryargs', 'None')
return cp
def getAPFConfig(self):
"""
Returns ConfigParser object representing config
"""
self.cp = Config()
sect = '%s-%s' % (self.parent.panda_resource, self.ce_host)
sect = str(sect)
self.cp.add_section(sect)
# Unconditional config
self.cp.set(sect, 'enabled', 'True')
self.cp.set(sect, 'batchqueue', self.panda_queue_name)
self.cp.set(sect, 'wmsqueue', self.parent.panda_resource)
self.cp.set(sect, 'batchsubmitplugin', self.submitplugin)
self.cp.set(sect, 'batchsubmit.%s.gridresource' % self.submitpluginstr,
self.gridresource)
#if self.parent.type == 'analysis':
# self.cp.set( sect, 'executable.arguments' , '%(executable.defaultarguments)s -u user' )
#else:
# self.cp.set( sect, 'executable.arguments' , '%(executable.defaultarguments)s -u managed' )
try:
self.apf_scale_factor = (
((1.0 / float(self.parent.parent.numfactories)) /
len(self.parent.ce_queues)) /
float(self.parent.parent.jobsperpilot))
except ZeroDivisionError:
self.log.error(
"Division by zero. Something wrong with scale factory calc.")
self.apf_scale_factor = 1.0
self.cp.set(sect, 'sched.scale.factor', str(self.apf_scale_factor))
#HTCondor CE
if self.ce_flavour == 'htcondor-ce':
pr = 'periodic_remove = (JobStatus == 2 && (CurrentTime - EnteredCurrentStatus) > 604800)'
self.cp.set(sect, 'batchsubmit.condorosgce.condor_attributes', pr)
if self.parent.maxrss is not None:
self.cp.set(
sect,
'batchsubmit.condorosgce.condor_attributes.+maxMemory',
str(self.parent.maxrss))
else:
self.cp.set(
sect,
'batchsubmit.condorosgce.condor_attributes.+maxMemory',
str(self.parent.maxmemory))
self.cp.set(sect,
'batchsubmit.condorosgce.condor_attributes.+xcount',
str(self.parent.corecount))
self.cp.set(
sect, 'batchsubmit.condorosgce.condor_attributes.+voactivity',
'"%s"' % self.parent.type)
self.cp.set(
sect,
'batchsubmit.condorosgce.condor_attributes.+remote_queue',
'"%s"' % self.ce_queue_name)
if self.ce_queue_maxwctime:
self.cp.set(
sect,
'batchsubmit.condorosgce.condor_attributes.+maxWallTime',
'%s' % self.ce_queue_maxwctime)
# Globus
if self.ce_flavour in ['osg-ce', 'globus']:
self.cp.set(sect, 'globusrsl.%s.queue' % self.gramversion,
self.gramqueue)
# Cream-specific JDL
if self.ce_flavour == 'cream-ce':
self.cp.set(sect, 'batchsubmit.condorcream.environ', self.creamenv)
if self.creamattr is not None:
self.cp.set(sect, 'creamattr', self.creamattr)
self.cp.set(
sect, 'batchsubmit.condorcream.condor_attributes',
'%(req)s,%(hold)s,%(remove)s,cream_attributes = %(creamattr)s,notification=Never'
)
else:
self.cp.set(sect, 'batchsubmit.condorcream.condor_attributes',
'%(req)s,%(hold)s,%(remove)s,notification=Never')
# Arc-CE
if self.ce_flavour == 'arc-ce':
pr = 'periodic_remove = (JobStatus == 2 && (CurrentTime - EnteredCurrentStatus) > 604800)'
pr = '%(req)s,%(hold)s,%(remove)s,notification=Never'
self.cp.set(sect, 'batchsubmit.condornordugrid.condor_attributes',
pr)
self.cp.set(
sect,
'batchsubmit.condornordugrid.condor_attributes.+remote_queue',
self.ce_queue_name)
self.cp.set(sect, 'batchsubmit.condornordugrid.nordugridrsl',
self.nordugridrsl)
self.cp.set(sect, 'nordugridrsl.nordugridrsladd', self.rsladd)
self.cp.set(sect, 'nordugridrsl.queue', self.ce_queue_name)
self.cp.set(sect, 'nordugridrsl.addenv.RUCIO_ACCOUNT', 'pilot')
return self.cp
class AgisCEQueue(object):
"""
Represents a single CE queue within a Panda queue description.
"""
def __init__(self, parent, cedict):
self.log = logging.getLogger('autopyfactory.config')
self.parent = parent
self.panda_queue_name = parent.panda_queue_name
self.ce_name = cedict['ce_name'] # AGLT2-CE-gate04.aglt2.org
self.ce_endpoint = cedict['ce_endpoint'] # gate04.aglt2.org:2119
self.ce_host = self.ce_endpoint.split(":")[0]
self.ce_state = cedict['ce_state'].lower() # 'active'
self.ce_status = cedict['ce_status'].lower() #
self.ce_queue_status = cedict['ce_queue_status'].lower()
self.ce_flavour = cedict['ce_flavour'].lower() # GLOBUS
self.ce_version = cedict['ce_version'].lower() # GT5
self.ce_queue_name = cedict['ce_queue_name'] # default
self.ce_jobmanager = cedict['ce_jobmanager'].lower() # condor
self.ce_queue_maxcputime = cedict['ce_queue_maxcputime'] # in seconds
self.ce_queue_maxwctime = cedict['ce_queue_maxwctime'] # in seconds
self.apf_scale_factor = 1.0
# Empty/default attributes:
self.gridresource = None
self.submitplugin = None
self.submitpluginstr = None
self.gramversion = None
self.gramqueue = None
self.creamenv = None
self.creamattr = ''
self.condorattr = None
self.maxmemory = None
self.maxtime = None
if self.ce_flavour in ['osg-ce', 'globus']:
self._initglobus()
elif self.ce_flavour == 'htcondor-ce':
self._initcondorce()
elif self.ce_flavour == 'cream-ce':
self._initcream()
elif self.ce_flavour == 'arc-ce':
self._initarc()
elif self.ce_flavour == 'lcg-ce':
self.log.debug("Ignoring old CE type 'LCG-CE'")
else:
self.log.warning("CEQueue %s has unknown ce_flavour: %s" %
(self.ce_name, self.ce_flavour))
def _initcondorce(self):
self.gridresource = self.ce_endpoint.split(':')[0]
self.submitplugin = 'CondorOSGCE'
self.submitpluginstr = 'condorosgce'
def _initglobus(self):
self.gridresource = '%s/jobmanager-%s' % (self.ce_endpoint,
self.ce_jobmanager)
if self.ce_version == 'gt2':
self.submitplugin = 'CondorGT2'
self.submitpluginstr = 'condorgt2'
elif self.ce_version == 'gt5':
self.submitplugin = 'CondorGT5'
self.submitpluginstr = 'condorgt5'
self.gramversion = 'gram5'
self.gramqueue = self.ce_queue_name
def _initcream(self):
self.gridresource = '%s/ce-cream/services/CREAM2 %s %s' % (
self.ce_endpoint, self.ce_jobmanager, self.ce_queue_name)
self.submitplugin = 'CondorCREAM'
self.submitpluginstr = 'condorcream'
if self.parent.pilot_version not in ['current']:
self.creamenv = 'RUCIO_ACCOUNT=pilot PILOT_HTTP_SOURCES=%s' % self.parent.pilot_version
else:
self.creamenv = 'RUCIO_ACCOUNT=pilot'
# glue 1.3 uses minutes and this / operator uses floor value
# https://wiki.italiangrid.it/twiki/bin/view/CREAM/UserGuideEMI2#Forward_of_requirements_to_the_b
self.maxtime = self.parent.maxtime / 60
self.maxmemory = self.parent.maxmemory
self.cputime = self.parent.corecount * self.maxtime
# maxrss and maxtime are expected to be set in AGIS for all queues
if self.parent.corecount:
self.creamattr = 'CpuNumber=%d;WholeNodes=false;SMPGranularity=%d;' % (
self.parent.corecount, self.parent.corecount)
if self.parent.corecount:
cputime = self.parent.corecount * self.maxtime
else:
cputime = self.maxtime
self.creamattr += 'CERequirements = "other.GlueCEPolicyMaxCPUTime == %d ' % cputime
self.creamattr += '&& other.GlueCEPolicyMaxWallClockTime == %d ' % self.maxtime
self.creamattr += '&& other.GlueHostMainMemoryRAMSize == %d' % self.parent.maxrss
if self.parent.maxswap:
maxvirtual = self.parent.maxrss + self.parent.maxswap
self.creamattr += ' && other.GlueHostMainMemoryVirtualSize == %d";' % maxvirtual
else:
self.creamattr += '";'
def _initarc(self):
# ignore :port part
self.gridresource = self.ce_endpoint.split(':')[0]
self.submitplugin = 'CondorNordugrid'
self.submitpluginstr = 'condornordugrid'
self.maxmemory = self.parent.maxmemory
#### self.maxtime = self.ce_queue_maxwctime
self.maxtime = self.parent.maxtime
self.nordugridrsl = '(jobname = arc_pilot)'
self.rsladd = '(runtimeenvironment = APPS/HEP/ATLAS-SITE-LCG)(runtimeenvironment = ENV/PROXY)'
self.rsladd += '(count = %d)' % self.parent.corecount
self.rsladd += '(countpernode = %d)' % self.parent.corecount
if self.parent.maxrss:
percore = self.parent.maxrss / self.parent.corecount
self.rsladd += '(memory = %d)' % percore
else:
percore = self.parent.maxmemory / self.parent.corecount
self.rsladd += '(memory = %d)' % percore
if self.maxtime:
self.rsladd += '(walltime = %d)' % self.maxtime
if self.maxtime:
self.rsladd += '(cputime = %d)' % (self.maxtime *
self.parent.corecount)
def getAPFConfigString(self):
"""
Returns string of valid APF configuration for this queue-ce entry.
Calculates scale factor based on how many other CEs serve this PQ
"""
cp = self.getAPFConfig()
sio = StringIO()
s = cp.write(sio)
return sio.getvalue()
def getAPFConfig(self):
"""
Returns ConfigParser object representing config
"""
self.cp = Config()
sect = '%s-%s' % (self.parent.panda_resource, self.ce_host)
sect = str(sect)
self.cp.add_section(sect)
# Unconditional config
self.cp.set(sect, 'enabled', 'True')
self.cp.set(sect, 'batchqueue', self.panda_queue_name)
self.cp.set(sect, 'wmsqueue', self.parent.panda_resource)
self.cp.set(sect, 'batchsubmitplugin', self.submitplugin)
self.cp.set(sect, 'batchsubmit.%s.gridresource' % self.submitpluginstr,
self.gridresource)
#if self.parent.type == 'analysis':
# self.cp.set( sect, 'executable.arguments' , '%(executable.defaultarguments)s -u user' )
#else:
# self.cp.set( sect, 'executable.arguments' , '%(executable.defaultarguments)s -u managed' )
try:
self.apf_scale_factor = (
((1.0 / float(self.parent.parent.numfactories)) /
len(self.parent.ce_queues)) /
float(self.parent.parent.jobsperpilot))
except ZeroDivisionError:
self.log.error(
"Division by zero. Something wrong with scale factory calc.")
self.apf_scale_factor = 1.0
self.cp.set(sect, 'sched.scale.factor', str(self.apf_scale_factor))
#HTCondor CE
if self.ce_flavour == 'htcondor-ce':
pr = 'periodic_remove = (JobStatus == 2 && (CurrentTime - EnteredCurrentStatus) > 604800)'
self.cp.set(sect, 'batchsubmit.condorosgce.condor_attributes', pr)
if self.parent.maxrss is not None:
self.cp.set(
sect,
'batchsubmit.condorosgce.condor_attributes.+maxMemory',
str(self.parent.maxrss))
else:
self.cp.set(
sect,
'batchsubmit.condorosgce.condor_attributes.+maxMemory',
str(self.parent.maxmemory))
self.cp.set(sect,
'batchsubmit.condorosgce.condor_attributes.+xcount',
str(self.parent.corecount))
self.cp.set(
sect, 'batchsubmit.condorosgce.condor_attributes.+voactivity',
'"%s"' % self.parent.type)
self.cp.set(
sect,
'batchsubmit.condorosgce.condor_attributes.+remote_queue',
'"%s"' % self.ce_queue_name)
if self.ce_queue_maxwctime:
self.cp.set(
sect,
'batchsubmit.condorosgce.condor_attributes.+maxWallTime',
'%s' % self.ce_queue_maxwctime)
# Globus
if self.ce_flavour in ['osg-ce', 'globus']:
self.cp.set(sect, 'globusrsl.%s.queue' % self.gramversion,
self.gramqueue)
# Cream-specific JDL
if self.ce_flavour == 'cream-ce':
self.cp.set(sect, 'batchsubmit.condorcream.environ', self.creamenv)
if self.creamattr is not None:
self.cp.set(sect, 'creamattr', self.creamattr)
self.cp.set(
sect, 'batchsubmit.condorcream.condor_attributes',
'%(req)s,%(hold)s,%(remove)s,cream_attributes = %(creamattr)s,notification=Never'
)
else:
self.cp.set(sect, 'batchsubmit.condorcream.condor_attributes',
'%(req)s,%(hold)s,%(remove)s,notification=Never')
# Arc-CE
if self.ce_flavour == 'arc-ce':
pr = 'periodic_remove = (JobStatus == 2 && (CurrentTime - EnteredCurrentStatus) > 604800)'
pr = '%(req)s,%(hold)s,%(remove)s,notification=Never'
self.cp.set(sect, 'batchsubmit.condornordugrid.condor_attributes',
pr)
self.cp.set(
sect,
'batchsubmit.condornordugrid.condor_attributes.+remote_queue',
self.ce_queue_name)
self.cp.set(sect, 'batchsubmit.condornordugrid.nordugridrsl',
self.nordugridrsl)
self.cp.set(sect, 'nordugridrsl.nordugridrsladd', self.rsladd)
self.cp.set(sect, 'nordugridrsl.queue', self.ce_queue_name)
self.cp.set(sect, 'nordugridrsl.addenv.RUCIO_ACCOUNT', 'pilot')
return self.cp
def __str__(self):
s = "AgisCEQueue: "
s += "PQ=%s " % self.panda_queue_name
s += "wmsqueue=%s " % self.parent.panda_resource
s += "submitplugin=%s " % self.submitplugin
s += "host=%s " % self.ce_host
s += "endpoint=%s " % self.ce_endpoint
s += "gridresource=%s " % self.gridresource
s += "maxtime=%s " % self.parent.maxtime
return s
formatstr = FORMAT26
log = logging.getLogger()
hdlr = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter(formatstr)
hdlr.setFormatter(formatter)
log.addHandler(hdlr)
log.setLevel(logging.WARNING)
if debug:
log.setLevel(logging.DEBUG) # Override with command line switches
if info:
log.setLevel(logging.INFO) # Override with command line switches
log.debug("Logging initialized.")
fconfig = Config()
if fconfig_file is not None:
fconfig_file = os.path.expanduser(fconfig_file)
got_config = fconfig.read(fconfig_file)
log.debug("Read config file %s, return value: %s" %
(fconfig_file, got_config))
else:
# Create valid config...
fconfig.add_section('Factory')
# Set unconditional defaults
fconfig.set(
'Factory', 'config.queues.agis.baseurl',
'http://atlas-agis-api.cern.ch/request/pandaqueue/query/list/?json&preset=schedconf.all'
)
fconfig.set('Factory', 'config.queues.agis.sleep', '3600')
class AuthManager(object):
"""
Manager to maintain multiple credential Handlers, one for each target account.
For some handlers, if they need to perform periodic checks, they will be run
as threads. Others, which only hold information, will just be objects.
"""
def __init__(self, factory=None):
self.log = logging.getLogger('autopyfactory')
self.log.info("Creating new authmanager...")
self.aconfig = Config()
self.handlers = []
self.factory = factory
if factory:
self.sleep = int(
self.factory.fcl.get('Factory', 'authmanager.sleep'))
else:
self.sleep = 5
def reconfig(self, newconfig):
self.log.debug("Performing reconfig() for auth handlers...")
hdiff = ConfigsDiff(self.aconfig, newconfig)
self.aconfig = newconfig
self.log.debug("Deleting removed handlers...")
self._delhandlers(hdiff.removed())
self.log.debug("Deleting modified handlers...")
self._delhandlers(hdiff.modified())
self.log.debug("Re-creating modified handlers...")
self._addhandlers(hdiff.modified())
self.log.debug("Adding new handlers...")
self._addhandlers(hdiff.added())
self.log.info("Completed reconfig() on auth handlers.")
def _addhandlers(self, newsections):
for sect in newsections:
try:
pclass = self.aconfig.get(sect, 'plugin')
except Exception as e:
self.log.warn("No plugin attribute for section %s" % sect)
if pclass == 'X509':
self.log.debug("Creating X509 handler for %s" % sect)
authpluginname = self.aconfig.get(sect, 'plugin')
x509h = pluginmanager.getplugin(
['autopyfactory', 'plugins', 'authmanager', 'auth'],
authpluginname, self, self.aconfig, sect)
self.handlers.append(x509h)
elif pclass == 'SSH' or 'GSISSH':
self.log.debug("Creating SSH handler for %s" % sect)
authpluginname = self.aconfig.get(sect, 'plugin')
sshh = pluginmanager.getplugin(
['autopyfactory', 'plugins', 'authmanager', 'auth'],
authpluginname, self, self.aconfig, sect)
self.handlers.append(sshh)
else:
self.log.warn("Unrecognized auth plugin %s" % pclass)
def _delhandlers(self, delsections):
for sect in delsections:
try:
handler = self._gethandlerbyname(sect)
if handler is not None:
if isinstance(handler, threading.Thread):
try:
handler.join()
except Exception:
self.log.warning(
'attempt to join() thread for handler %s failed, handler is not active'
% handler.name)
# In any case, remove the handler object...
self.handlers.pop(handler)
except Exception:
self.log.warning(
'attempt to join() thread for handler %s failed, handler is not active'
% handler.name)
def _gethandlerbyname(self, name):
h = None
for h in self.handlers:
if h.name == name:
self.log.debug("Found handler %s" % name)
return h
return h
def activate(self):
"""
start all Handlers, if needed
"""
for ah in self.handlers:
if isinstance(ah, threading.Thread):
self.log.debug("Handler [%s] is a thread. Starting..." %
ah.name)
ah.start()
else:
self.log.debug("Handler [%s] is not a thread. No action." %
ah.name)
def listNames(self):
"""
Returns list of valid names of Handlers in this Manager.
"""
names = []
for h in self.handlers:
names.append(h.name)
return names
#
# API for X509Handler
#
def getProxyPath(self, profilelist):
"""
Check all the handlers for matching profile name(s).
profiles argument is a list
"""
pp = None
for profile in profilelist:
self.log.debug("Getting proxy path for profile %s" % profile)
ph = None
for h in self.handlers:
self.log.debug("Finding handler. Checking %s" % h.name)
if h.name == profile:
ph = h
break
if ph:
self.log.debug("Found handler %s. Getting proxypath..." %
ph.name)
pp = ph.getProxyPath()
self.log.debug("Proxypath is %s" % pp)
if pp:
break
if not pp:
subject = "Proxy problem on %s" % self.factory.factoryid
messagestring = "Unable to get valid proxy from configured profiles: %s" % profilelist
self.factory.sendAdminEmail(subject, messagestring)
raise InvalidAuthFailure("Problem getting proxy for profile %s" %
profilelist)
return pp
#
# API for SSHKeyHandler
#
def getSSHKeyPair(self, profile):
"""
Returns tuple (public, private, pass) key/phrase string from profile.
"""
pass
def getSSHKeyPairPaths(self, profile):
"""
Returns tuple (public, private, pass) key/passfile paths to files from profile.
"""
h = self._getHandler(profile)
pub = h.getSSHPubKeyFilePath()
priv = h.getSSHPrivKeyFilePath()
pasf = h.getSSHPassFilePath()
self.log.info('Got file paths for pub, priv, pass for SSH profile %s' %
profile)
return (pub, priv, pasf)
#
# API for GSISSHKeyHandler
#
def getGSISSHKey(self, profile):
"""
Returns GSI proxy string from profile
"""
pass
def getGSISSHKeyPaths(self, profile):
"""
Returns GSI proxy (private key) from path as a string
"""
h = self._getHandler(profile)
priv = h.getGSISSHPrivKeyFilePath()
self.log.info('Got file paths for proxy for GSISSH profile %s' %
profile)
return priv
def _getHandler(self, profile):
"""
"""
handler = None
for h in self.handlers:
self.log.debug("Finding handler. Checking %s" % h.name)
if h.name == profile:
self.log.debug("Found handler for %s" % h.name)
handler = h
if handler is None:
raise InvalidAuthFailure('No handler for %s ' % profile)
return handler
formatstr = FORMAT26
log = logging.getLogger()
hdlr = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter(formatstr)
hdlr.setFormatter(formatter)
log.addHandler(hdlr)
log.setLevel(logging.WARNING)
if debug:
log.setLevel(logging.DEBUG) # Override with command line switches
if info:
log.setLevel(logging.INFO) # Override with command line switches
log.debug("Logging initialized.")
fconfig=Config()
if fconfig_file is not None:
fconfig_file = os.path.expanduser(fconfig_file)
got_config = fconfig.read(fconfig_file)
log.debug("Read config file %s, return value: %s" % (fconfig_file, got_config))
else:
# Create valid config...
fconfig.add_section('Factory')
# Set unconditional defaults
fconfig.set('Factory', 'config.queues.agis.baseurl', 'http://atlas-agis-api.cern.ch/request/pandaqueue/query/list/?json&preset=schedconf.all' )
fconfig.set('Factory', 'config.queues.agis.sleep', '3600' )
fconfig.set('Factory', 'config.queues.agis.jobsperpilot', '1.5' )
fconfig.set('Factory', 'config.queues.agis.numfactories', '4')
"""
def getMockMappingsConfig():
mappingsconf = '''[NATIVECONDORBATCHSTATUS]
0 = unexpanded
1 = idle
2 = running
3 = removed
4 = completed
5 = held
6 = submission_err
[CONDORBATCHSTATUS-JOBSTATUS2INFO]
0 = pending
1 = pending
2 = running
3 = done
4 = done
5 = pending
6 = running
[CONDOREC2BATCHSTATUS-JOBSTATUS2INFO]
0 = pending
1 = pending
2 = running
3 = done
4 = done
5 = pending
6 = running
[PANDAWMSSTATUS-JOBSSTATISTICSPERSITE2INFO]
pending = notready
defined = notready
assigned = notready
waiting = notready
throttled = notready
activated = ready
starting = running
sent = running
running = running
holding = running
transferring = running
finished = done
failed = failed
cancelled = failed
[CONDORWMSSTATUS-JOBSTATUS2INFO]
0 = ready
1 = ready
2 = running
3 = done
4 = done
5 = failed
6 = running'''
buf = StringIO.StringIO(mappingsconf)
cp = Config()
cp.readfp(buf)
return cp
def getMockMappingsConfig():
mappingsconf = '''[NATIVECONDORBATCHSTATUS]
0 = unexpanded
1 = idle
2 = running
3 = removed
4 = completed
5 = held
6 = submission_err
[CONDORBATCHSTATUS-JOBSTATUS2INFO]
0 = pending
1 = pending
2 = running
3 = done
4 = done
5 = pending
6 = running
[CONDOREC2BATCHSTATUS-JOBSTATUS2INFO]
0 = pending
1 = pending
2 = running
3 = done
4 = done
5 = pending
6 = running
[PANDAWMSSTATUS-JOBSSTATISTICSPERSITE2INFO]
pending = notready
defined = notready
assigned = notready
waiting = notready
throttled = notready
activated = ready
starting = running
sent = running
running = running
holding = running
transferring = running
finished = done
failed = failed
cancelled = failed
[CONDORWMSSTATUS-JOBSTATUS2INFO]
0 = ready
1 = ready
2 = running
3 = done
4 = done
5 = failed
6 = running'''
buf = StringIO.StringIO(mappingsconf)
cp = Config()
cp.readfp(buf)
return cp