def __init__(self, ceflavour=['ARC-CE']): # Get agent name from /path/to/aCTAgent.py self.name = os.path.basename(sys.argv[0])[:-3] # logger self.logger = aCTLogger.aCTLogger(self.name) self.log = self.logger() self.criticallogger = aCTLogger.aCTLogger('aCTCritical', arclog=False) self.criticallog = self.criticallogger() # config self.conf = aCTConfig.aCTConfigAPP() self.arcconf = aCTConfig.aCTConfigARC() self.tmpdir = str(self.arcconf.get(['tmp', 'dir'])) # database self.dbarc = aCTDBArc.aCTDBArc(self.log) self.dbcondor = aCTDBCondor.aCTDBCondor(self.log) self.dbpanda = aCTDBPanda.aCTDBPanda(self.log) # APFMon self.apfmon = aCTAPFMon.aCTAPFMon(self.conf) # CRIC info self.flavour = ceflavour self.cricparser = aCTCRICParser.aCTCRICParser(self.log) self.sites = {} self.osmap = {} self.sitesselect = '' # start time for periodic restart self.starttime = time.time() self.log.info("Started %s", self.name)
def reconnectDB(self): ''' Reconnect DB ''' try: del self.dbarc del self.dbcondor except AttributeError: # Already deleted pass self.dbarc = aCTDBArc.aCTDBArc(self.log) self.dbcondor = aCTDBCondor.aCTDBCondor(self.log)
def bootstrap_db(): '''Set up the ARC and Condor DB tables''' logger = aCTLogger('aCTBootstrap') log = logger() dbarc = aCTDBArc(log) dbcondor = aCTDBCondor(log) print('Setting up ARC tables...') if not dbarc.createTables(): print('Error creating arc tables, see aCTBootstrap.log for details') print('Setting up Condor tables...') if not dbcondor.createTables(): print('Error creating condor tables, see aCTBootstrap.log for details')
def __init__(self, log, conf, appconf): # logger self.log = log self.actlocation = conf.get(["actlocation", "dir"]) self.logdir = conf.get(["logger", "logdir"]) # DB connection self.dbarc = aCTDBArc.aCTDBArc(self.log) self.dbcondor = aCTDBCondor.aCTDBCondor(self.log) # list of processes to run per cluster self.arcprocesses = [ 'act/arc/aCTStatus', 'act/arc/aCTFetcher', 'act/arc/aCTCleaner' ] self.condorprocesses = [ 'act/condor/aCTStatus', 'act/condor/aCTFetcher', 'act/condor/aCTCleaner' ] # submitter process self.arcsubmitter = 'act/arc/aCTSubmitter' self.condorsubmitter = 'act/condor/aCTSubmitter' # dictionary of processes:aCTProcessHandler of which to run a single instance self.processes_single = {'act/common/aCTProxyHandler': None} apps = appconf.getList(["modules", "app"]) for app in apps: try: ap = importlib.import_module(app).app_processes self.processes_single.update( {f'{app.replace(".", "/")}/{p}': None for p in ap}) except ModuleNotFoundError as e: self.log.critical(f'No such module {app}') raise e except AttributeError: self.log.info(f'No app-specific processes found in {app}') else: self.log.info(f'Loaded {", ".join(ap)} processes from {app}') # dictionary of cluster to list of aCTProcessHandlers self.running = {} # dictionary of cluster to Submitter processes handlers, there should # be one per unique cluster in clusterlist self.submitters = {} # Start single instance processes for process in self.processes_single: proc = self.aCTProcessHandler(process, self.logdir, actlocation=self.actlocation) proc.start() self.processes_single[process] = proc
def bootstrap_db(): '''Set up the DB tables''' # TODO: setup only what is needed based on config and app logger = aCTLogger('aCTBootstrap') log = logger() dbarc = aCTDBArc(log) dbclient = ClientDB(log) dbcondor = aCTDBCondor(log) dbpanda = aCTDBPanda(log) if not dbarc.createTables(): print('Error creating arc tables, see aCTBootstrap.log for details') if not dbclient.createTables(): print('Error creating client tables, see aCTBootstrap.log for details') if not dbcondor.createTables(): print('Error creating condor tables, see aCTBootstrap.log for details') if not dbpanda.createTables(): print('Error creating panda tables, see aCTBootstrap.log for details')
def __init__(self): # Get agent name from /path/to/aCTAgent.py self.name = os.path.basename(sys.argv[0])[:-3] self.cluster = '' clusterhost = '' if len(sys.argv) == 2: self.cluster = sys.argv[1] url = urlparse(self.cluster) clusterhost = url.netloc.split(':')[0] if url.netloc else url.path # logger logname = '%s-%s' % (self.name, clusterhost) if clusterhost else self.name self.logger = aCTLogger.aCTLogger(logname, cluster=self.cluster) self.log = self.logger() self.criticallogger = aCTLogger.aCTLogger('aCTCritical', cluster=self.cluster, arclog=False) self.criticallog = self.criticallogger() # config self.conf = aCTConfig.aCTConfigARC() self.tmpdir = str(self.conf.get(['tmp', 'dir'])) # database # TODO: subclasses for arc and condor with respective DBs defined there self.db = aCTDBArc.aCTDBArc(self.log) self.dbcondor = aCTDBCondor(self.log) # ARC Configuration # Credentials will be set by ARC agents for each job or set of jobs # but for now set default credential in config to keep ARC happy cred_type = arc.initializeCredentialsType( arc.initializeCredentialsType.SkipCredentials) self.uc = arc.UserConfig(cred_type) self.uc.ProxyPath(str(self.conf.get(['voms', 'proxypath']))) self.uc.CACertificatesDirectory( str(self.conf.get(["voms", "cacertdir"]))) timeout = int(self.conf.get(['atlasgiis', 'timeout'])) self.uc.Timeout(timeout) # start time for periodic restart self.starttime = time.time() self.log.info("Started %s for cluster %s", self.name, self.cluster)
#!/usr/bin/python import classad import logging from act.arc.aCTDBArc import aCTDBArc from act.condor.aCTDBCondor import aCTDBCondor from act.common.aCTProxy import aCTProxy from act.common.aCTLogger import aCTLogger logger = aCTLogger('acttest', cluster='test') log = logger() db = aCTDBArc(log) dbcondor = aCTDBCondor(log) xrsl = '''&(executable=/bin/sleep) (arguments=100) (stdout=stdout) (rerun=2) (gmlog=gmlog) (inputfiles = ("runpilot2-wrapper.sh" "/cvmfs/atlas.cern.ch/repo/sw/PandaPilotWrapper/latest/runpilot2-wrapper.sh")) (runtimeenvironment = "ENV/PROXY") ''' cad = classad.ClassAd('''\n [\n UserLog = "/tmp/log/test.$(ClusterId).log"; \n Out = "/tmp/output/test.$(ClusterId).out"; \n Err = "/tmp/error/test.$(ClusterId).err"; \n JobUniverse = 9; \n X509UserProxy = "/tmp/x509up_u100780"; \n