def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) self.log = core_utils.make_logger(baseLogger, 'aCT sweeper', method_name='__init__') self.actDB = aCTDBPanda(self.log)
def bootstrap(): logger = aCTLogger('aCTBootstrap') log = logger() dbpanda = aCTDBPanda(log) if not dbpanda.createTables(): print('Failed to create Panda tables, see aCTBootstrap.log for details')
def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) self.hostname = socket.getfqdn() # Set up aCT DB connection self.log = core_utils.make_logger(baseLogger, 'aCT submitter', method_name='__init__') self.actDB = aCTDBPanda(self.log) # Credential dictionary role: proxy file self.certs = dict(zip([r.split('=')[1] for r in list(harvester_config.credmanager.voms)], list(harvester_config.credmanager.outCertFile))) # Map of role to aCT proxyid self.proxymap = {} # Get proxy info # TODO: better to send aCT the proxy file and let it handle it for role, proxy in self.certs.items(): cred_type = arc.initializeCredentialsType(arc.initializeCredentialsType.SkipCredentials) uc = arc.UserConfig(cred_type) uc.ProxyPath(str(proxy)) cred = arc.Credential(uc) dn = cred.GetIdentityName() actp = aCTProxy(self.log) attr = '/atlas/Role='+role proxyid = actp.getProxyId(dn, attr) if not proxyid: raise Exception("Proxy with DN {0} and attribute {1} was not found in proxies table".format(dn, attr)) self.proxymap[role] = proxyid
def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) # Set up aCT DB connection self.log = core_utils.make_logger(baseLogger, 'aCT submitter', method_name='__init__') self.conf = aCTConfigARC() self.actDB = aCTDBPanda(self.log, self.conf.get(["db", "file"]))
def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) # Set up aCT DB connection self.log = core_utils.make_logger(baseLogger, 'aCT submitter', method_name='__init__') self.actDB = aCTDBPanda(self.log) # Credential dictionary role: proxy file self.certs = dict(zip([r.split('=')[1] for r in list(harvester_config.credmanager.voms)], list(harvester_config.credmanager.outCertFile))) # Map of role to aCT proxyid self.proxymap = {} # Get proxy info # TODO: better to send aCT the proxy file and let it handle it for role, proxy in self.certs.items(): cred_type = arc.initializeCredentialsType(arc.initializeCredentialsType.SkipCredentials) uc = arc.UserConfig(cred_type) uc.ProxyPath(str(proxy)) cred = arc.Credential(uc) dn = cred.GetIdentityName() self.log.info("Proxy {0} with DN {1} and role {2}".format(proxy, dn, role)) actp = aCTProxy(self.log) attr = '/atlas/Role='+role proxyid = actp.getProxyId(dn, attr) if not proxyid: raise Exception("Proxy with DN {0} and attribute {1} was not found in proxies table".format(dn, attr)) self.proxymap[role] = proxyid
def __init__(self, ceflavour=['ARC-CE']): # Get agent name from /path/to/aCTAgent.py self.name = os.path.basename(sys.argv[0])[:-3] # logger self.logger = aCTLogger.aCTLogger(self.name) self.log = self.logger() self.criticallogger = aCTLogger.aCTLogger('aCTCritical', arclog=False) self.criticallog = self.criticallogger() # config self.conf = aCTConfig.aCTConfigAPP() self.arcconf = aCTConfig.aCTConfigARC() self.tmpdir = str(self.arcconf.get(['tmp', 'dir'])) # database self.dbarc = aCTDBArc.aCTDBArc(self.log) self.dbcondor = aCTDBCondor.aCTDBCondor(self.log) self.dbpanda = aCTDBPanda.aCTDBPanda(self.log) # APFMon self.apfmon = aCTAPFMon.aCTAPFMon(self.conf) # CRIC info self.flavour = ceflavour self.cricparser = aCTCRICParser.aCTCRICParser(self.log) self.sites = {} self.osmap = {} self.sitesselect = '' # start time for periodic restart self.starttime = time.time() self.log.info("Started %s", self.name)
def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) # Set up aCT DB connection self.log = core_utils.make_logger(baseLogger, 'aCT submitter', method_name='__init__') self.conf = aCTConfigARC() self.actDB = aCTDBPanda(self.log, self.conf.get(["db", "file"])) # Get proxy info # TODO: specify DN in conf instead cred_type = arc.initializeCredentialsType( arc.initializeCredentialsType.SkipCredentials) uc = arc.UserConfig(cred_type) uc.ProxyPath(str(self.conf.get(['voms', 'proxypath']))) cred = arc.Credential(uc) dn = cred.GetIdentityName() self.log.info("Running under DN %s" % dn) # Set up proxy map (prod/pilot roles) self.proxymap = {} actp = aCTProxy(self.log) for role in self.conf.getList(['voms', 'roles', 'item']): attr = '/atlas/Role=' + role proxyid = actp.getProxyId(dn, attr) if not proxyid: raise Exception( "Proxy with DN {0} and attribute {1} was not found in proxies table" .format(dn, attr)) self.proxymap[role] = proxyid
def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) # Set up aCT DB connection self.log = core_utils.make_logger(baseLogger, 'aCT submitter', method_name='__init__') self.actDB = aCTDBPanda(self.log)
def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) self.log = core_utils.make_logger(baseLogger, 'aCT sweeper', method_name='__init__') try: self.actDB = aCTDBPanda(self.log) except Exception as e: self.log.error('Could not connect to aCT database: {0}'.format(str(e))) self.actDB = None
def __init__(self): self.conf = aCTConfig.aCTConfigARC() self.logger = aCTLogger.aCTLogger("aCTReport") self.log = self.logger() self.criticallogger = aCTLogger.aCTLogger('aCTCritical', arclog=False) self.criticallog = self.criticallogger() #self.db=aCTDB.aCTDB(None,self.conf.get(["db","file"])) self.db = aCTDBArc.aCTDBArc(self.log, self.conf.get(["db", "file"])) self.pandadb = aCTDBPanda.aCTDBPanda(self.log, self.conf.get(["db", "file"]))
def main(): if len(sys.argv) != 2: print("Usage: python aCTHeartbeatWatchdog.py timelimit") sys.exit(1) timelimit = int(sys.argv[1]) # logger logger = aCTLogger('aCTHeartbeatWatchdog') log = logger() # database dbarc = aCTDBArc(log) dbpanda = aCTDBPanda(log) # Query for running jobs with theartbeat longer than timelimit seconds ago select = "sendhb=1 and " \ "pandastatus in ('sent', 'starting', 'running', 'transferring') and " \ "theartbeat != 0 and " + dbpanda.timeStampLessThan("theartbeat", timelimit) columns = ['pandaid', 'pandastatus', 'proxyid', 'sitename', 'theartbeat'] jobs = dbpanda.getJobs(select, columns) if jobs: print( 'Found %d jobs with outdated heartbeat (older than %d seconds):\n' % (len(jobs), timelimit)) print('\t'.join( ['pandaid', 'site', 'status', 'theartbeat', 'Panda response'])) # Panda server for each proxy pandas = {} for job in jobs: proxyid = job['proxyid'] if proxyid not in pandas: panda = aCTPanda(log, dbarc.getProxyPath(proxyid)) pandas[proxyid] = panda response = pandas[proxyid].updateStatus(job['pandaid'], job['pandastatus']) print('\t'.join([ str(job['pandaid']), job['sitename'], job['pandastatus'], str(job['theartbeat']), str(response) ])) # update heartbeat time in the DB dbpanda.updateJob( job['pandaid'], {'theartbeat': dbpanda.getTimeStamp(time.time() + 1)})
def bootstrap_db(): '''Set up the DB tables''' # TODO: setup only what is needed based on config and app logger = aCTLogger('aCTBootstrap') log = logger() dbarc = aCTDBArc(log) dbclient = ClientDB(log) dbcondor = aCTDBCondor(log) dbpanda = aCTDBPanda(log) if not dbarc.createTables(): print('Error creating arc tables, see aCTBootstrap.log for details') if not dbclient.createTables(): print('Error creating client tables, see aCTBootstrap.log for details') if not dbcondor.createTables(): print('Error creating condor tables, see aCTBootstrap.log for details') if not dbpanda.createTables(): print('Error creating panda tables, see aCTBootstrap.log for details')
def __init__(self, args): self.output = "" self.harvester = args.harvester self.outfile = args.web self.actconfs = args.conffiles or [''] # empty string for default behaviour self.logger=aCTLogger.aCTLogger("aCTReport") self.actlog=self.logger() self.actlog.logger.setLevel(logging.ERROR) self.criticallogger = aCTLogger.aCTLogger('aCTCritical', arclog=False) self.criticallog = self.criticallogger() if self.outfile: self.log('<META HTTP-EQUIV="refresh" CONTENT="60"><pre>') self.log(time.asctime() + '\n') self.db=aCTDBArc.aCTDBArc(self.actlog) self.pandadb=aCTDBPanda.aCTDBPanda(self.actlog)
from datetime import datetime from act.arc.aCTDBArc import aCTDBArc from act.atlas.aCTDBPanda import aCTDBPanda from act.common.aCTLogger import aCTLogger from act.common.aCTConfig import aCTConfigARC try: service_id, webpage_url = sys.argv[1:3] except: print('Usage: kibana.py service_id webpage_url') sys.exit(1) logger = aCTLogger('kibana probe') log = logger() arcdb = aCTDBArc(log) pandadb = aCTDBPanda(log) config = aCTConfigARC() def getARCJobs(): return str(arcdb.getNArcJobs('TRUE')) def getARCSlots(): jobs = arcdb.getArcJobsInfo("state='Running'", ['RequestedSlots']) slots = 0 for j in jobs: slots += j['RequestedSlots'] return str(slots)
def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) # Set up aCT DB connection self.log = core_utils.make_logger(baseLogger, 'aCT submitter', method_name='__init__') self.actDB = aCTDBPanda(self.log)
def report(actconfs): actlogger = aCTLogger('aCTReport') logger = actlogger() rep = {} rtot = {} log = '' states = [ "sent", "starting", "running", "slots", "tovalidate", "toresubmit", "toclean", "finished", "done", "failed", "donefailed", "tobekilled", "cancelled", "donecancelled" ] for conf in actconfs: if conf: os.environ['ACTCONFIGARC'] = conf db = aCTDBPanda(logger) c = db.db.conn.cursor() c.execute("select sitename, actpandastatus, corecount from pandajobs") rows = c.fetchall() for r in rows: site, state = (str(r[0]), str(r[1])) if r[2] is None: corecount = 1 else: corecount = int(r[2]) try: rep[site][state] += 1 if state == "running": rep[site]["slots"] += corecount except: try: rep[site][state] = 1 if state == "running": try: rep[site]["slots"] += corecount except: rep[site]["slots"] = corecount except: rep[site] = {} rep[site][state] = 1 if state == "running": rep[site]["slots"] = corecount try: rtot[state] += 1 if state == "running": rtot["slots"] += corecount except: rtot[state] = 1 if state == "running": rtot["slots"] = corecount log += f"All Panda jobs: {sum([v for k,v in rtot.items() if k != 'slots'])}\n" log += f"{'':29} {' '.join([f'{s:>9}' for s in states])}\n" for k in sorted(rep.keys()): log += f"{k:>28.28}:" for s in states: try: log += f'{rep[k][s]:>10}' except KeyError: log += f'{"-":>10}' log += '\n' log += f'{"Totals":>28}:' for s in states: try: log += f'{rtot[s]:>10}' except: log += f'{"-":>10}' log += '\n\n' if len(actconfs) == 1: log += HarvesterReport() return log
def __init__(self, **kwarg): BaseMessenger.__init__(self, **kwarg) # Set up aCT DB connection self.log = core_utils.make_logger(baseLogger, 'aCT messenger', method_name='__init__') self.actDB = aCTDBPanda(self.log)
def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) self.log = core_utils.make_logger(baseLogger, 'aCT sweeper', method_name='__init__') self.actDB = aCTDBPanda(self.log)