def __init__(self, ceflavour=['ARC-CE']):
# Get agent name from /path/to/aCTAgent.py
self.name = os.path.basename(sys.argv[0])[:-3]
# logger
self.logger = aCTLogger.aCTLogger(self.name)
self.log = self.logger()
self.criticallogger = aCTLogger.aCTLogger('aCTCritical', arclog=False)
self.criticallog = self.criticallogger()
# config
self.conf = aCTConfig.aCTConfigAPP()
self.arcconf = aCTConfig.aCTConfigARC()
self.tmpdir = str(self.arcconf.get(['tmp', 'dir']))
# database
self.dbarc = aCTDBArc.aCTDBArc(self.log)
self.dbcondor = aCTDBCondor.aCTDBCondor(self.log)
self.dbpanda = aCTDBPanda.aCTDBPanda(self.log)
# APFMon
self.apfmon = aCTAPFMon.aCTAPFMon(self.conf)
# CRIC info
self.flavour = ceflavour
self.cricparser = aCTCRICParser.aCTCRICParser(self.log)
self.sites = {}
self.osmap = {}
self.sitesselect = ''
# start time for periodic restart
self.starttime = time.time()
self.log.info("Started %s", self.name)
def reconnectDB(self):
'''
Reconnect DB
'''
try:
del self.dbarc
del self.dbcondor
except AttributeError: # Already deleted
pass
self.dbarc = aCTDBArc.aCTDBArc(self.log)
self.dbcondor = aCTDBCondor.aCTDBCondor(self.log)
def bootstrap_db():
'''Set up the ARC and Condor DB tables'''
logger = aCTLogger('aCTBootstrap')
log = logger()
dbarc = aCTDBArc(log)
dbcondor = aCTDBCondor(log)
print('Setting up ARC tables...')
if not dbarc.createTables():
print('Error creating arc tables, see aCTBootstrap.log for details')
print('Setting up Condor tables...')
if not dbcondor.createTables():
print('Error creating condor tables, see aCTBootstrap.log for details')
def __init__(self, log, conf, appconf):
# logger
self.log = log
self.actlocation = conf.get(["actlocation", "dir"])
self.logdir = conf.get(["logger", "logdir"])
# DB connection
self.dbarc = aCTDBArc.aCTDBArc(self.log)
self.dbcondor = aCTDBCondor.aCTDBCondor(self.log)
# list of processes to run per cluster
self.arcprocesses = [
'act/arc/aCTStatus', 'act/arc/aCTFetcher', 'act/arc/aCTCleaner'
]
self.condorprocesses = [
'act/condor/aCTStatus', 'act/condor/aCTFetcher',
'act/condor/aCTCleaner'
]
# submitter process
self.arcsubmitter = 'act/arc/aCTSubmitter'
self.condorsubmitter = 'act/condor/aCTSubmitter'
# dictionary of processes:aCTProcessHandler of which to run a single instance
self.processes_single = {'act/common/aCTProxyHandler': None}
apps = appconf.getList(["modules", "app"])
for app in apps:
try:
ap = importlib.import_module(app).app_processes
self.processes_single.update(
{f'{app.replace(".", "/")}/{p}': None
for p in ap})
except ModuleNotFoundError as e:
self.log.critical(f'No such module {app}')
raise e
except AttributeError:
self.log.info(f'No app-specific processes found in {app}')
else:
self.log.info(f'Loaded {", ".join(ap)} processes from {app}')
# dictionary of cluster to list of aCTProcessHandlers
self.running = {}
# dictionary of cluster to Submitter processes handlers, there should
# be one per unique cluster in clusterlist
self.submitters = {}
# Start single instance processes
for process in self.processes_single:
proc = self.aCTProcessHandler(process,
self.logdir,
actlocation=self.actlocation)
proc.start()
self.processes_single[process] = proc
def bootstrap_db():
'''Set up the DB tables'''
# TODO: setup only what is needed based on config and app
logger = aCTLogger('aCTBootstrap')
log = logger()
dbarc = aCTDBArc(log)
dbclient = ClientDB(log)
dbcondor = aCTDBCondor(log)
dbpanda = aCTDBPanda(log)
if not dbarc.createTables():
print('Error creating arc tables, see aCTBootstrap.log for details')
if not dbclient.createTables():
print('Error creating client tables, see aCTBootstrap.log for details')
if not dbcondor.createTables():
print('Error creating condor tables, see aCTBootstrap.log for details')
if not dbpanda.createTables():
print('Error creating panda tables, see aCTBootstrap.log for details')
def __init__(self):
# Get agent name from /path/to/aCTAgent.py
self.name = os.path.basename(sys.argv[0])[:-3]
self.cluster = ''
clusterhost = ''
if len(sys.argv) == 2:
self.cluster = sys.argv[1]
url = urlparse(self.cluster)
clusterhost = url.netloc.split(':')[0] if url.netloc else url.path
# logger
logname = '%s-%s' % (self.name,
clusterhost) if clusterhost else self.name
self.logger = aCTLogger.aCTLogger(logname, cluster=self.cluster)
self.log = self.logger()
self.criticallogger = aCTLogger.aCTLogger('aCTCritical',
cluster=self.cluster,
arclog=False)
self.criticallog = self.criticallogger()
# config
self.conf = aCTConfig.aCTConfigARC()
self.tmpdir = str(self.conf.get(['tmp', 'dir']))
# database
# TODO: subclasses for arc and condor with respective DBs defined there
self.db = aCTDBArc.aCTDBArc(self.log)
self.dbcondor = aCTDBCondor(self.log)
# ARC Configuration
# Credentials will be set by ARC agents for each job or set of jobs
# but for now set default credential in config to keep ARC happy
cred_type = arc.initializeCredentialsType(
arc.initializeCredentialsType.SkipCredentials)
self.uc = arc.UserConfig(cred_type)
self.uc.ProxyPath(str(self.conf.get(['voms', 'proxypath'])))
self.uc.CACertificatesDirectory(
str(self.conf.get(["voms", "cacertdir"])))
timeout = int(self.conf.get(['atlasgiis', 'timeout']))
self.uc.Timeout(timeout)
# start time for periodic restart
self.starttime = time.time()
self.log.info("Started %s for cluster %s", self.name, self.cluster)
#!/usr/bin/python
import classad
import logging
from act.arc.aCTDBArc import aCTDBArc
from act.condor.aCTDBCondor import aCTDBCondor
from act.common.aCTProxy import aCTProxy
from act.common.aCTLogger import aCTLogger
logger = aCTLogger('acttest', cluster='test')
log = logger()
db = aCTDBArc(log)
dbcondor = aCTDBCondor(log)
xrsl = '''&(executable=/bin/sleep)
(arguments=100)
(stdout=stdout)
(rerun=2)
(gmlog=gmlog)
(inputfiles = ("runpilot2-wrapper.sh" "/cvmfs/atlas.cern.ch/repo/sw/PandaPilotWrapper/latest/runpilot2-wrapper.sh"))
(runtimeenvironment = "ENV/PROXY")
'''
cad = classad.ClassAd('''\n
[\n
UserLog = "/tmp/log/test.$(ClusterId).log"; \n
Out = "/tmp/output/test.$(ClusterId).out"; \n
Err = "/tmp/error/test.$(ClusterId).err"; \n
JobUniverse = 9; \n
X509UserProxy = "/tmp/x509up_u100780"; \n