def __init__(self, **kwarg):
self.logBaseURL = None
PluginBase.__init__(self, **kwarg)
self.k8s_client = k8s_Client(namespace=self.k8s_namespace, config_file=self.k8s_config_file)
# number of processes
try:
self.nProcesses
except AttributeError:
self.nProcesses = 1
else:
if (not self.nProcesses) or (self.nProcesses < 1):
self.nProcesses = 1
# x509 proxy
try:
self.x509UserProxy
except AttributeError:
if os.getenv('X509_USER_PROXY'):
self.x509UserProxy = os.getenv('X509_USER_PROXY')
# CPU adjust ratio
try:
self.cpuAdjustRatio
except AttributeError:
self.cpuAdjustRatio = 100
# Memory adjust ratio
try:
self.memoryAdjustRatio
except AttributeError:
self.memoryAdjustRatio = 100
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
try:
self.nProcesses
except AttributeError:
self.nProcesses = 4
try:
self.cancelUnknown
except AttributeError:
self.cancelUnknown = False
else:
self.cancelUnknown = bool(self.cancelUnknown)
try:
self.heldTimeout
except AttributeError:
self.heldTimeout = 3600
try:
self.cacheEnable = harvester_config.monitor.pluginCacheEnable
except AttributeError:
self.cacheEnable = False
try:
self.cacheRefreshInterval = harvester_config.monitor.pluginCacheRefreshInterval
except AttributeError:
self.cacheRefreshInterval = harvester_config.monitor.checkInterval
try:
self.useCondorHistory
except AttributeError:
self.useCondorHistory = True
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# make logger
mainLog = self.make_logger(_logger, method_name='__init__')
# attributes
try:
self.inFile
except AttributeError:
self.inFile = self.inCertFile
# parse inFile setup conficuration
try:
with open(self.inFile) as f:
self.setupMap = json.load(f)
except Exception as e:
mainLog.error('Error with inFile/inCertFile . {0}: {1}'.format(
e.__class__.__name__, e))
self.setupMap = {}
raise
# validate setupMap
try:
self.k8s_namespace = self.setupMap['k8s_namespace']
self.k8s_config_file = self.setupMap['k8s_config_file']
self.proxy_file_list = self.setupMap['proxy_files']
self.secret_name = self.setupMap.get('secret_name', 'proxy-secret')
except KeyError as e:
mainLog.error('Missing setup in inFile/inCertFile . {0}: {1}'.format(
e.__class__.__name__, e))
raise
# k8s client
self.k8s_client = k8s_Client(namespace=self.k8s_namespace, config_file=self.k8s_config_file)
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
try:
self.nProcesses
except AttributeError:
self.nProcesses = 4
try:
self.cancelUnknown
except AttributeError:
self.cancelUnknown = False
else:
self.cancelUnknown = bool(self.cancelUnknown)
try:
self.heldTimeout
except AttributeError:
self.heldTimeout = 3600
try:
self.cacheEnable = harvester_config.monitor.pluginCacheEnable
except AttributeError:
self.cacheEnable = False
try:
self.cacheRefreshInterval = harvester_config.monitor.pluginCacheRefreshInterval
except AttributeError:
self.cacheRefreshInterval = harvester_config.monitor.checkInterval
try:
self.useCondorHistory
except AttributeError:
self.useCondorHistory = True
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
self.log = core_utils.make_logger(baseLogger,
'aCT sweeper',
method_name='__init__')
self.actDB = aCTDBPanda(self.log)
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# create Globus Transfer Client
tmpLog = self.make_logger(_logger, method_name='GoPreparator __init__ ')
try:
self.tc = None
# need to get client_id and refresh_token from PanDA server via harvester cache mechanism
tmpLog.debug('about to call dbInterface.get_cache(globus_secret)')
c_data = self.dbInterface.get_cache('globus_secret')
if (not c_data == None) and c_data.data['StatusCode'] == 0 :
tmpLog.debug('Got the globus_secrets from PanDA')
self.client_id = c_data.data['publicKey'] # client_id
self.refresh_token = c_data.data['privateKey'] # refresh_token
tmpStat, self.tc = globus_utils.create_globus_transfer_client(tmpLog,self.client_id,self.refresh_token)
if not tmpStat:
self.tc = None
errStr = 'failed to create Globus Transfer Client'
tmpLog.error(errStr)
else :
self.client_id = None
self.refresh_token = None
self.tc = None
errStr = 'failed to get Globus Client ID and Refresh Token'
tmpLog.error(errStr)
except:
core_utils.dump_error_message(tmpLog)
tmpLog.debug('__init__ finished')
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
with uLock:
global uID
self.dummy_transfer_id = '{0}_{1}'.format(dummy_transfer_id_base, uID)
uID += 1
uID %= harvester_config.preparator.nThreads
def __init__(self, **kwarg):
self.jobAttributesToUse = [
'nCore', 'minRamCount', 'maxDiskCount', 'maxWalltime'
]
PluginBase.__init__(self, **kwarg)
self.rt_mapper = ResourceTypeMapper()
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
self.hostname = socket.getfqdn()
# Set up aCT DB connection
self.log = core_utils.make_logger(baseLogger, 'aCT submitter', method_name='__init__')
self.actDB = aCTDBPanda(self.log)
# Credential dictionary role: proxy file
self.certs = dict(zip([r.split('=')[1] for r in list(harvester_config.credmanager.voms)],
list(harvester_config.credmanager.outCertFile)))
# Map of role to aCT proxyid
self.proxymap = {}
# Get proxy info
# TODO: better to send aCT the proxy file and let it handle it
for role, proxy in self.certs.items():
cred_type = arc.initializeCredentialsType(arc.initializeCredentialsType.SkipCredentials)
uc = arc.UserConfig(cred_type)
uc.ProxyPath(str(proxy))
cred = arc.Credential(uc)
dn = cred.GetIdentityName()
actp = aCTProxy(self.log)
attr = '/atlas/Role='+role
proxyid = actp.getProxyId(dn, attr)
if not proxyid:
raise Exception("Proxy with DN {0} and attribute {1} was not found in proxies table".format(dn, attr))
self.proxymap[role] = proxyid
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# Credential dictionary role: proxy file
self.certs = dict(zip([r.split('=')[1] for r in list(harvester_config.credmanager.voms)],
list(harvester_config.credmanager.outCertFile)))
self.cred_type = arc.initializeCredentialsType(arc.initializeCredentialsType.SkipCredentials)
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# Set up aCT DB connection
self.log = core_utils.make_logger(baseLogger,
'aCT submitter',
method_name='__init__')
self.conf = aCTConfigARC()
self.actDB = aCTDBPanda(self.log, self.conf.get(["db", "file"]))
# Get proxy info
# TODO: specify DN in conf instead
cred_type = arc.initializeCredentialsType(
arc.initializeCredentialsType.SkipCredentials)
uc = arc.UserConfig(cred_type)
uc.ProxyPath(str(self.conf.get(['voms', 'proxypath'])))
cred = arc.Credential(uc)
dn = cred.GetIdentityName()
self.log.info("Running under DN %s" % dn)
# Set up proxy map (prod/pilot roles)
self.proxymap = {}
actp = aCTProxy(self.log)
for role in self.conf.getList(['voms', 'roles', 'item']):
attr = '/atlas/Role=' + role
proxyid = actp.getProxyId(dn, attr)
if not proxyid:
raise Exception(
"Proxy with DN {0} and attribute {1} was not found in proxies table"
.format(dn, attr))
self.proxymap[role] = proxyid
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# make logger
mainLog = self.make_logger(_logger, method_name='__init__')
# attributes
try:
self.inFile
except AttributeError:
self.inFile = self.inCertFile
# parse inFile setup conficuration
try:
with open(self.inFile) as f:
self.setupMap = json.load(f)
except Exception as e:
mainLog.error('Error with inFile/inCertFile . {0}: {1}'.format(
e.__class__.__name__, e))
self.setupMap = {}
raise
# validate setupMap
try:
self.k8s_namespace = self.setupMap['k8s_namespace']
self.k8s_config_file = self.setupMap['k8s_config_file']
self.proxy_file_list = self.setupMap['proxy_files']
self.secret_name = self.setupMap.get('secret_name', 'proxy-secret')
except KeyError as e:
mainLog.error(
'Missing setup in inFile/inCertFile . {0}: {1}'.format(
e.__class__.__name__, e))
raise
# k8s client
self.k8s_client = k8s_Client(namespace=self.k8s_namespace,
config_file=self.k8s_config_file)
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# create Globus Transfer Client
tmpLog = core_utils.make_logger(_logger,
method_name='GoPreparator __init__ ')
try:
self.tc = None
# need to get client_id and refresh_token from PanDA server via harvester cache mechanism
tmpLog.debug('about to call dbInterface.get_cache(globus_secret)')
c_data = self.dbInterface.get_cache('globus_secret')
if (not c_data == None) and c_data.data['StatusCode'] == 0:
tmpLog.debug('Got the globus_secrets from PanDA')
self.client_id = c_data.data['publicKey'] # client_id
self.refresh_token = c_data.data['privateKey'] # refresh_token
tmpStat, self.tc = globus_utils.create_globus_transfer_client(
tmpLog, self.client_id, self.refresh_token)
if not tmpStat:
self.tc = None
errStr = 'failed to create Globus Transfer Client'
tmpLog.error(errStr)
else:
self.client_id = None
self.refresh_token = None
self.tc = None
errStr = 'failed to get Globus Client ID and Refresh Token'
tmpLog.error(errStr)
except:
core_utils.dump_error_message(tmpLog)
tmpLog.debug('__init__ finished')
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# Set up aCT DB connection
self.log = core_utils.make_logger(baseLogger, 'aCT submitter', method_name='__init__')
self.conf = aCTConfigARC()
self.actDB = aCTDBPanda(self.log, self.conf.get(["db", "file"]))
def __init__(self, **kwarg):
self.logBaseURL = None
PluginBase.__init__(self, **kwarg)
self.k8s_client = k8s_Client(namespace=self.k8s_namespace,
config_file=self.k8s_config_file)
# number of processes
try:
self.nProcesses
except AttributeError:
self.nProcesses = 1
else:
if (not self.nProcesses) or (self.nProcesses < 1):
self.nProcesses = 1
# x509 proxy
try:
self.x509UserProxy
except AttributeError:
if os.getenv('X509_USER_PROXY'):
self.x509UserProxy = os.getenv('X509_USER_PROXY')
# CPU adjust ratio
try:
self.cpuAdjustRatio
except AttributeError:
self.cpuAdjustRatio = 100
# Memory adjust ratio
try:
self.memoryAdjustRatio
except AttributeError:
self.memoryAdjustRatio = 100
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# Set up aCT DB connection
self.log = core_utils.make_logger(baseLogger, 'aCT submitter', method_name='__init__')
self.actDB = aCTDBPanda(self.log)
# Credential dictionary role: proxy file
self.certs = dict(zip([r.split('=')[1] for r in list(harvester_config.credmanager.voms)],
list(harvester_config.credmanager.outCertFile)))
# Map of role to aCT proxyid
self.proxymap = {}
# Get proxy info
# TODO: better to send aCT the proxy file and let it handle it
for role, proxy in self.certs.items():
cred_type = arc.initializeCredentialsType(arc.initializeCredentialsType.SkipCredentials)
uc = arc.UserConfig(cred_type)
uc.ProxyPath(str(proxy))
cred = arc.Credential(uc)
dn = cred.GetIdentityName()
self.log.info("Proxy {0} with DN {1} and role {2}".format(proxy, dn, role))
actp = aCTProxy(self.log)
attr = '/atlas/Role='+role
proxyid = actp.getProxyId(dn, attr)
if not proxyid:
raise Exception("Proxy with DN {0} and attribute {1} was not found in proxies table".format(dn, attr))
self.proxymap[role] = proxyid
def __init__(self, **kwarg):
self.logBaseURL = None
PluginBase.__init__(self, **kwarg)
# number of processes
try:
self.nProcesses
except AttributeError:
self.nProcesses = 1
else:
if (not self.nProcesses) or (self.nProcesses < 1):
self.nProcesses = 1
# executable file
try:
self.executableFile
except AttributeError:
self.executableFile = None
# condor log directory
try:
self.logDir
except AttributeError:
self.logDir = os.getenv('TMPDIR') or '/tmp'
# x509 proxy
try:
self.x509UserProxy
except AttributeError:
self.x509UserProxy = os.getenv('X509_USER_PROXY')
# ATLAS AGIS
try:
self.useAtlasAGIS = bool(self.useAtlasAGIS)
except AttributeError:
self.useAtlasAGIS = False
# ATLAS Grid CE, requiring AGIS
try:
self.useAtlasGridCE = bool(self.useAtlasGridCE)
except AttributeError:
self.useAtlasGridCE = False
finally:
self.useAtlasAGIS = self.useAtlasAGIS or self.useAtlasGridCE
# sdf template directories of CEs
try:
self.CEtemplateDir
except AttributeError:
self.CEtemplateDir = ''
# remote condor schedd and pool name (collector), and spool option
try:
self.condorSchedd
except AttributeError:
self.condorSchedd = None
try:
self.condorPool
except AttributeError:
self.condorPool = None
try:
self.useSpool
except AttributeError:
self.useSpool = True
# record of information of CE statistics
self.ceStatsLock = threading.Lock()
self.ceStats = dict()
def __init__(self, **kwarg):
self.jobAttributesToUse = [
'nCore', 'minRamCount', 'maxDiskCount', 'maxWalltime'
]
self.adjusters = None
PluginBase.__init__(self, **kwarg)
self.init_adjusters_defaults()
self.dyn_resources = None
def __init__(self, **kwarg):
self.uploadLog = False
self.logBaseURL = None
PluginBase.__init__(self, **kwarg)
# template for batch script
tmpFile = open(self.templateFile)
self.template = tmpFile.read()
tmpFile.close()
def __init__(self, **kwarg):
dict.__init__(self)
PluginBase.__init__(self, **kwarg)
dbInterface = DBInterface()
cacher_key = kwarg.get('cacher_key', 'panda_queues.json')
panda_queues_cache = dbInterface.get_cache(cacher_key)
if panda_queues_cache:
self.update(panda_queues_cache.data)
def __init__(self, **kwarg):
self.uploadLog = False
self.logBaseURL = None
PluginBase.__init__(self, **kwarg)
# template for batch script
tmpFile = open(self.templateFile)
self.template = tmpFile.read()
tmpFile.close()
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# Set up aCT DB connection
self.log = core_utils.make_logger(baseLogger,
'aCT submitter',
method_name='__init__')
self.actDB = aCTDBPanda(self.log)
def __init__(self, **kwarg):
self.logBaseURL = None
PluginBase.__init__(self, **kwarg)
self.k8s_client = k8s_Client(namespace=self.k8s_namespace,
config_file=self.k8s_config_file)
# required for parsing jobParams
self.parser = argparse.ArgumentParser()
self.parser.add_argument('-p', dest='executable', type=unquote)
self.parser.add_argument('--containerImage', dest='container_image')
# number of processes
try:
self.nProcesses
except AttributeError:
self.nProcesses = 1
else:
if (not self.nProcesses) or (self.nProcesses < 1):
self.nProcesses = 1
# x509 proxy: obsolete mode
try:
self.x509UserProxy
except AttributeError:
if os.getenv('X509_USER_PROXY'):
self.x509UserProxy = os.getenv('X509_USER_PROXY')
# x509 proxy for analysis jobs in grandly unified queues
try:
self.x509UserProxyAnalysis
except AttributeError:
self.x509UserProxyAnalysis = os.getenv('X509_USER_PROXY_ANAL')
# x509 proxy through k8s secrets: preferred way
try:
self.proxySecretPath
except AttributeError:
if os.getenv('PROXY_SECRET_PATH'):
self.proxySecretPath = os.getenv('PROXY_SECRET_PATH')
# analysis x509 proxy through k8s secrets: on GU queues
try:
self.proxySecretPathAnalysis
except AttributeError:
if os.getenv('PROXY_SECRET_PATH_ANAL'):
self.proxySecretPath = os.getenv('PROXY_SECRET_PATH_ANAL')
# CPU adjust ratio
try:
self.cpuAdjustRatio
except AttributeError:
self.cpuAdjustRatio = 100
# Memory adjust ratio
try:
self.memoryAdjustRatio
except AttributeError:
self.memoryAdjustRatio = 100
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
self.log = core_utils.make_logger(baseLogger, 'aCT sweeper', method_name='__init__')
try:
self.actDB = aCTDBPanda(self.log)
except Exception as e:
self.log.error('Could not connect to aCT database: {0}'.format(str(e)))
self.actDB = None
def __init__(self, **kwargs):
dict.__init__(self)
PluginBase.__init__(self, **kwargs)
self.lock = threading.Lock()
self.dbInterface = DBInterface()
self.cacher_key = kwargs.get('cacher_key', 'panda_queues.json')
self.refresh_period = resolver_config.get('refreshPeriod', 300)
self.last_refresh_ts = 0
self._refresh()
def __init__(self, **kwarg):
self.jobSpecFileFormat = 'json'
PluginBase.__init__(self, **kwarg)
self.schedulerid = harvester_config.master.harvester_id
self.tmpdir = '/tmp' # TODO configurable or common function
# Credential dictionary role: proxy file
self.certs = dict(zip([r.split('=')[1] for r in list(harvester_config.credmanager.voms)],
list(harvester_config.credmanager.outCertFile)))
self.cred_type = arc.initializeCredentialsType(arc.initializeCredentialsType.SkipCredentials)
def __init__(self, **kwarg):
self.nProcesses = 1
self.logBaseURL = None
PluginBase.__init__(self, **kwarg)
# template for batch script
tmpFile = open(self.templateFile)
self.template = tmpFile.read()
tmpFile.close()
# number of processes
if self.nProcesses < 1:
self.nProcesses = None
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# States taken from: https://cloud.google.com/compute/docs/instances/checking-instance-status
self.vm_to_worker_status = {
'RUNNING': WorkSpec.ST_running,
'TERMINATED': WorkSpec.ST_finished,
'STOPPING': WorkSpec.ST_finished,
'PROVISIONING': WorkSpec.ST_submitted,
'STAGING': WorkSpec.ST_submitted
}
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# Credential dictionary role: proxy file
self.certs = dict(
zip([
r.split('=')[1]
for r in list(harvester_config.credmanager.voms)
], list(harvester_config.credmanager.outCertFile)))
self.cred_type = arc.initializeCredentialsType(
arc.initializeCredentialsType.SkipCredentials)
def __init__(self, **kwarg):
'''Set up DB connection and credentials'''
PluginBase.__init__(self, **kwarg)
self.dbproxy = DBProxy()
self.schedulerid = harvester_config.master.harvester_id
# Credential dictionary role: proxy file
self.certs = dict(zip([r.split('=')[1] for r in list(harvester_config.credmanager.voms)],
list(harvester_config.credmanager.outCertFile)))
self.cred_type = arc.initializeCredentialsType(arc.initializeCredentialsType.SkipCredentials)
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
self.queue_config_mapper = QueueConfigMapper()
# States taken from: https://cloud.google.com/compute/docs/instances/checking-instance-status
self.vm_to_worker_status = {
'RUNNING': WorkSpec.ST_running,
'TERMINATED': WorkSpec.ST_running, # the VM is stopped, but has to be fully deleted
'STOPPING': WorkSpec.ST_finished,
'PROVISIONING': WorkSpec.ST_submitted,
'STAGING': WorkSpec.ST_submitted
}
def __init__(self, **kwarg):
self.logBaseURL = None
PluginBase.__init__(self, **kwarg)
self.k8s_client = k8s_Client(namespace=self.k8s_namespace, config_file=self.k8s_config_file)
# update or create the pilot starter executable
self.k8s_client.create_or_patch_configmap_starter()
# required for parsing jobParams
self.parser = argparse.ArgumentParser()
self.parser.add_argument('-p', dest='executable', type=unquote)
self.parser.add_argument('--containerImage', dest='container_image')
# allowed associated parameters from AGIS
self._allowed_agis_attrs = (
'pilot_url',
)
# number of processes
try:
self.nProcesses
except AttributeError:
self.nProcesses = 1
else:
if (not self.nProcesses) or (self.nProcesses < 1):
self.nProcesses = 1
# x509 proxy through k8s secrets: preferred way
try:
self.proxySecretPath
except AttributeError:
if os.getenv('PROXY_SECRET_PATH'):
self.proxySecretPath = os.getenv('PROXY_SECRET_PATH')
# analysis x509 proxy through k8s secrets: on GU queues
try:
self.proxySecretPathAnalysis
except AttributeError:
if os.getenv('PROXY_SECRET_PATH_ANAL'):
self.proxySecretPath = os.getenv('PROXY_SECRET_PATH_ANAL')
# CPU adjust ratio
try:
self.cpuAdjustRatio
except AttributeError:
self.cpuAdjustRatio = 100
# Memory adjust ratio
try:
self.memoryAdjustRatio
except AttributeError:
self.memoryAdjustRatio = 100
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
self.queue_config_mapper = QueueConfigMapper()
# States taken from: https://cloud.google.com/compute/docs/instances/checking-instance-status
self.vm_to_worker_status = {
'RUNNING': WorkSpec.ST_running,
'TERMINATED': WorkSpec.
ST_running, # the VM is stopped, but has to be fully deleted
'STOPPING': WorkSpec.ST_finished,
'PROVISIONING': WorkSpec.ST_submitted,
'STAGING': WorkSpec.ST_submitted
}
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
if hasattr(self, 'database_filename'):
_db_filename = self.database_filename
else:
_db_filename = harvester_config.fifo.database_filename
self.db_path = os.path.abspath(
re.sub('\$\(AGENT\)', self.agentName, _db_filename))
self._connection_cache = {}
with self._get_conn() as conn:
conn.execute(self._exclusive_lock_sql)
conn.execute(self._create_sql)
conn.execute(self._create_index_sql)
conn.commit()
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
try:
sshUserName = self.sshUserName
except Exception:
sshUserName = None
try:
sshPassword = self.sshPassword
except Exception:
sshPassword = None
try:
privateKey = self.privateKey
except Exception:
privateKey = None
try:
passPhrase = self.passPhrase
except Exception:
passPhrase = None
try:
jumpHost = self.jumpHost
except Exception:
jumpHost = None
try:
jumpPort = self.jumpPort
except Exception:
jumpPort = 22
try:
remotePort = self.remotePort
except Exception:
remotePort = 22
sshTunnelPool.make_tunnel_server(self.remoteHost,
remotePort,
self.remoteBindPort,
self.numTunnels,
ssh_username=sshUserName,
ssh_password=sshPassword,
private_key=privateKey,
pass_phrase=passPhrase,
jump_host=jumpHost,
jump_port=jumpPort)
tunnelHost, tunnelPort, tunnelCore = sshTunnelPool.get_tunnel(
self.remoteHost, remotePort)
self.conn = rpyc.connect(tunnelHost,
tunnelPort,
config={
"allow_all_attrs": True,
"allow_setattr": True,
"allow_delattr": True
})
def __init__(self, **kwarg):
'''Set up DB connection and credentials'''
PluginBase.__init__(self, **kwarg)
self.dbproxy = DBProxy()
self.schedulerid = harvester_config.master.harvester_id
# Credential dictionary role: proxy file
self.certs = dict(
zip([
r.split('=')[1]
for r in list(harvester_config.credmanager.voms)
], list(harvester_config.credmanager.outCertFile)))
self.cred_type = arc.initializeCredentialsType(
arc.initializeCredentialsType.SkipCredentials)
def __init__(self, **kwarg):
self.logBaseURL = None
PluginBase.__init__(self, **kwarg)
# template for batch script
tmpFile = open(self.templateFile)
self.template = tmpFile.read()
tmpFile.close()
# number of processes
try:
self.nProcesses
except AttributeError:
self.nProcesses = 1
else:
if (not self.nProcesses) or (self.nProcesses < 1):
self.nProcesses = 1
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
if hasattr(self, 'database_filename'):
_db_filename = self.database_filename
else:
_db_filename = harvester_config.fifo.database_filename
_db_filename = re.sub('\$\(TITLE\)', self.titleName, _db_filename)
_db_filename = re.sub('\$\(AGENT\)', self.titleName, _db_filename)
self.db_path = os.path.abspath(_db_filename)
self._connection_cache = {}
with self._get_conn() as conn:
conn.execute(self._exclusive_lock_sql)
conn.execute(self._create_sql)
conn.execute(self._create_index_sql)
conn.commit()
def __init__(self, **kwarg):
dict.__init__(self)
PluginBase.__init__(self, **kwarg)
dbInterface = DBInterface()
cacher_key = kwarg.get('cacher_key', 'panda_queues.json')
panda_queues_cache = dbInterface.get_cache(cacher_key)
if panda_queues_cache and isinstance(panda_queues_cache.data, dict):
panda_queues_dict = panda_queues_cache.data
for (k, v) in iteritems(panda_queues_dict):
try:
panda_resource = v['panda_resource']
assert k == v['nickname']
except Exception:
pass
else:
self[panda_resource] = v
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
try:
self.nProcesses
except AttributeError:
self.nProcesses = 4
try:
self.cancelUnknown
except AttributeError:
self.cancelUnknown = False
else:
self.cancelUnknown = bool(self.cancelUnknown)
try:
self.heldTimeout
except AttributeError:
self.heldTimeout = 3600
def __init__(self, **kwarg):
tmpLog = core_utils.make_logger(_logger, method_name='__init__')
PluginBase.__init__(self, **kwarg)
self.sshUserName = getattr(self, 'sshUserName', None)
self.sshPassword = getattr(self, 'sshPassword', None)
self.privateKey = getattr(self, 'privateKey', None)
self.passPhrase = getattr(self, 'passPhrase', None)
self.jumpHost = getattr(self, 'jumpHost', None)
self.jumpPort = getattr(self, 'jumpPort', 22)
self.remotePort = getattr(self, 'remotePort', 22)
# is connected only if ssh forwarding works
self.is_connected = False
try:
self._get_connection()
except Exception as e:
tmpLog.error('failed to get connection ; {0}: {1}'.format(e.__class__.__name__, e))
else:
self.is_connected = True
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# make logger
tmpLog = self.make_logger(_logger, 'ThreadID={0}'.format(threading.current_thread().ident),
method_name='GlobusBulkPreparator __init__ {} ')
tmpLog.debug('__init__ start')
self.thread_id = threading.current_thread().ident
self.id = GlobusBulkPreparator.next_id
GlobusBulkPreparator.next_id += 1
with uLock:
global uID
self.dummy_transfer_id = '{0}_{1}'.format(dummy_transfer_id_base, 'XXXX')
uID += 1
uID %= harvester_config.preparator.nThreads
# create Globus Transfer Client
try:
self.tc = None
# need to get client_id and refresh_token from PanDA server via harvester cache mechanism
tmpLog.debug('about to call dbInterface.get_cache(globus_secret)')
c_data = self.dbInterface.get_cache('globus_secret')
if (not c_data == None) and c_data.data['StatusCode'] == 0 :
tmpLog.debug('Got the globus_secrets from PanDA')
self.client_id = c_data.data['publicKey'] # client_id
self.refresh_token = c_data.data['privateKey'] # refresh_token
tmpStat, self.tc = globus_utils.create_globus_transfer_client(tmpLog,self.client_id,self.refresh_token)
if not tmpStat:
self.tc = None
errStr = 'failed to create Globus Transfer Client'
tmpLog.error(errStr)
else :
self.client_id = None
self.refresh_token = None
self.tc = None
errStr = 'failed to get Globus Client ID and Refresh Token'
tmpLog.error(errStr)
except:
core_utils.dump_error_message(tmpLog)
# tmp debugging
tmpLog.debug('self.id = {0}'.format(self.id))
tmpLog.debug('self.dummy_transfer_id = {0}'.format(self.dummy_transfer_id))
# tmp debugging
tmpLog.debug('__init__ finish')
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
_redis_conn_opt_dict = {}
if hasattr(self, 'redisHost'):
_redis_conn_opt_dict['host'] = self.redisHost
elif hasattr(harvester_config.fifo, 'redisHost'):
_redis_conn_opt_dict['host'] = harvester_config.fifo.redisHost
if hasattr(self, 'redisPort'):
_redis_conn_opt_dict['port'] = self.redisPort
elif hasattr(harvester_config.fifo, 'redisPort'):
_redis_conn_opt_dict['port'] = harvester_config.fifo.redisPort
if hasattr(self, 'redisDB'):
_redis_conn_opt_dict['db'] = self.redisDB
elif hasattr(harvester_config.fifo, 'redisDB'):
_redis_conn_opt_dict['db'] = harvester_config.fifo.redisDB
if hasattr(self, 'redisPassword'):
_redis_conn_opt_dict['password'] = self.redisPassword
elif hasattr(harvester_config.fifo, 'redisPassword'):
_redis_conn_opt_dict['password'] = harvester_config.fifo.redisPassword
self.qconn = redis.StrictRedis(**_redis_conn_opt_dict)
self.id_score = '{0}-fifo_id-score'.format(self.titleName)
self.id_item = '{0}-fifo_id-item'.format(self.titleName)
self.id_temp = '{0}-fifo_id-temp'.format(self.titleName)
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
self.pluginFactory = PluginFactory()
self.queue_config_mapper = QueueConfigMapper()
tmpLog = self.make_logger(baseLogger, method_name='__init__')
tmpLog.info("[{0}] SAGA adaptor will be used.".format(self.adaptor))
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
self.queue_config_mapper = QueueConfigMapper()
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
self.tableName = '{title}_FIFO'.format(title=self.titleName)
# DB access attribues
if hasattr(self, 'db_host'):
db_host = self.db_host
else:
try:
db_host = harvester_config.fifo.db_host
except AttributeError:
db_host = '127.0.0.1'
if hasattr(self, 'db_port'):
db_port = self.db_port
else:
try:
db_port = harvester_config.fifo.db_port
except AttributeError:
db_port = 3306
if hasattr(self, 'db_user'):
db_user = self.db_user
else:
db_user = harvester_config.fifo.db_user
if hasattr(self, 'db_password'):
db_password = self.db_password
else:
db_password = harvester_config.fifo.db_password
if hasattr(self, 'db_schema'):
db_schema = self.db_schema
else:
db_schema = harvester_config.fifo.db_schema
# get connection, cursor and error types
try:
import MySQLdb
import MySQLdb.cursors
except ImportError:
try:
import mysql.connector
except ImportError:
raise Exception('No available MySQL DB API installed. Please pip install mysqlclient or mysql-connection-python')
else:
self.con = mysql.connector.connect(user=db_user, passwd=db_password,
db=db_schema, host=db_host, port=db_port)
self.cur = self.con.cursor(buffered=True)
self.OperationalError = mysql.connector.errors.OperationalError
else:
class MyCursor (MySQLdb.cursors.Cursor):
def fetchone(self):
tmpRet = MySQLdb.cursors.Cursor.fetchone(self)
if tmpRet is None:
return None
return tmpRet
def fetchall(self):
tmpRets = MySQLdb.cursors.Cursor.fetchall(self)
return tmpRets
self.con = MySQLdb.connect(user=db_user, passwd=db_password,
db=db_schema, host=db_host, port=db_port,
cursorclass=MyCursor)
self.cur = self.con.cursor()
self.OperationalError = MySQLdb.OperationalError
# create table for fifo
try:
self._make_table()
# self._make_index()
self.commit()
except Exception as _e:
self.rollback()
raise _e
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
tmpLog = self.make_logger(baseLogger, method_name='__init__')
tmpLog.info("Titan utils initiated")
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# Set up aCT DB connection
self.log = core_utils.make_logger(baseLogger, 'aCT submitter', method_name='__init__')
self.actDB = aCTDBPanda(self.log)
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
def __init__(self, **kwarg):
self.logBaseURL = 'http://localhost/test'
PluginBase.__init__(self, **kwarg)
def __init__(self, **kwarg):
# logic type : AND: throttled if all rules are satisfied, OR: throttled if one rule is satisfied
self.logicType = 'OR'
PluginBase.__init__(self, **kwarg)
self.dbProxy = DBProxy()
def __init__(self, **kwarg):
self.logBaseURL = None
PluginBase.__init__(self, **kwarg)
# number of processes
try:
self.nProcesses
except AttributeError:
self.nProcesses = 1
else:
if (not self.nProcesses) or (self.nProcesses < 1):
self.nProcesses = 1
# executable file
try:
self.executableFile
except AttributeError:
self.executableFile = None
# condor log directory
try:
self.logDir
except AttributeError:
self.logDir = os.getenv('TMPDIR') or '/tmp'
# x509 proxy
try:
self.x509UserProxy
except AttributeError:
self.x509UserProxy = os.getenv('X509_USER_PROXY')
# ATLAS AGIS
try:
self.useAtlasAGIS = bool(self.useAtlasAGIS)
except AttributeError:
self.useAtlasAGIS = False
# ATLAS Grid CE, requiring AGIS
try:
self.useAtlasGridCE = bool(self.useAtlasGridCE)
except AttributeError:
self.useAtlasGridCE = False
finally:
self.useAtlasAGIS = self.useAtlasAGIS or self.useAtlasGridCE
# sdf template directories of CEs
try:
self.CEtemplateDir
except AttributeError:
self.CEtemplateDir = ''
# remote condor schedd and pool name (collector), and spool option
try:
self.condorSchedd
except AttributeError:
self.condorSchedd = None
try:
self.condorPool
except AttributeError:
self.condorPool = None
# condor spool mechanism. If False, need shared FS across remote schedd
try:
self.useSpool
except AttributeError:
self.useSpool = False
# number of workers less than this number will be bulkily submitted in only one schedd
try:
self.minBulkToRamdomizedSchedd
except AttributeError:
self.minBulkToRamdomizedSchedd = 20
# record of information of CE statistics
self.ceStatsLock = threading.Lock()
self.ceStats = dict()
def __init__(self, **kwarg):
self.logBaseURL = 'http://localhost/test'
PluginBase.__init__(self, **kwarg)
self.queue_config_mapper = QueueConfigMapper()
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
tmpLog = core_utils.make_logger(baseLogger, method_name='__init__')
tmpLog.info("[{0}] SAGA adaptor will be used".format(self.adaptor))
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
self.log = core_utils.make_logger(baseLogger, 'aCT sweeper', method_name='__init__')
self.actDB = aCTDBPanda(self.log)
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
self.nProcesses = 4
self.vm_client = OS_SimpleClient(auth_config_json_file=self.authConfigFile)
def __init__(self, **kwarg):
self.n_threads = 3
PluginBase.__init__(self, **kwarg)
if self.n_threads < 1:
self.n_threads = 1
