def createNewMyProxy(self, timeleftthreshold=0, nokey=False):
"""
Handles the MyProxy creation
Let the following variables be
timeleftthreshold: the proxy in myproxy should be delegated for at least this time (14 days)
myproxytimeleft: current validity of your proxy in myproxy
usercertDaysLeft: the number of days left before your user certificate expire
myproxyDesiredValidity: delegate the proxy in myproxy for that time (30 days)
If we need to renew the proxy in myproxy because its atributes has changed or because it is valid for
less time than timeleftthreshold then we do it.
Before doing that, we check when the user certificate is expiring. If it's within the timeleftthreshold (myproxytimeleft < timeleftthreshold)
we delegate the proxy just for the time we need (checking first if we did not already do it since at some point
usercertDaysLeft ~= myproxytimeleft and we don't need to delegate it at every command even though myproxytimeleft < timeleftthreshold).
Note that a warning message is printed at every command it usercertDaysLeft < timeleftthreshold
"""
myproxy = Proxy ( self.defaultDelegation )
myproxy.userDN = myproxy.getSubjectFromCert(self.certLocation)
myproxytimeleft = 0
self.logger.debug("Getting myproxy life time left for %s" % self.defaultDelegation["myProxySvr"])
# return an integer that indicates the number of seconds to the expiration of the proxy in myproxy
myproxytimeleft = myproxy.getMyProxyTimeLeft(serverRenewer=True, nokey=nokey)
self.logger.debug("Myproxy is valid: %i" % myproxytimeleft)
trustRetrListChanged = myproxy.trustedRetrievers!=self.defaultDelegation['serverDN'] #list on the REST and on myproxy are different
if myproxytimeleft < timeleftthreshold or self.proxyChanged or trustRetrListChanged:
# checking the enddate of the user certificate
usercertDaysLeft = myproxy.getUserCertEnddate()
if usercertDaysLeft == 0:
msg = "%sYOUR USER CERTIFICATE IS EXPIRED (OR WILL EXPIRE TODAY). CANNOT SUBMIT%s"\
% (colors.RED, colors.NORMAL)
raise ProxyCreationException(msg)
#if the certificate is going to expire print a warning. This is going to bre printed at every command if
#the myproxytimeleft is inferior to the timeleftthreshold
if usercertDaysLeft < self.myproxyDesiredValidity:
self.logger.info("%sYour user certificate is going to expire in %s days. Please renew it! %s"\
% (colors.RED, usercertDaysLeft, colors.NORMAL) )
#check if usercertDaysLeft ~= myproxytimeleft which means we already delegated the proxy for as long as we could
if abs(usercertDaysLeft*60*60*24 - myproxytimeleft) < 60*60*24 and not trustRetrListChanged: #less than one day between usercertDaysLeft and myproxytimeleft
return
#adjust the myproxy delegation time accordingly to the user cert validity
self.logger.info("%sDelegating your proxy for %s days instead of %s %s"\
% (colors.RED, usercertDaysLeft, self.myproxyDesiredValidity, colors.NORMAL) )
myproxy.myproxyValidity = "%i:00" % (usercertDaysLeft*24)
# creating the proxy
self.logger.debug("Delegating a myproxy for %s hours" % self.defaultDelegation['myproxyValidity'] )
try:
myproxy.delegate(serverRenewer = True, nokey=nokey)
self.logger.debug("My-proxy delegated.")
except Exception, ex:
raise ProxyCreationException("Problems delegating My-proxy. %s"%ex._message)
def createNewMyProxy(self, timeleftthreshold=0, nokey=False):
"""
Handles the MyProxy creation
Let the following variables be
timeleftthreshold: the proxy in myproxy should be delegated for at least this time (14 days)
myproxytimeleft: current validity of your proxy in myproxy
usercertDaysLeft: the number of days left before your user certificate expire
myproxyDesiredValidity: delegate the proxy in myproxy for that time (30 days)
If we need to renew the proxy in myproxy because its atributes has changed or because it is valid for
less time than timeleftthreshold then we do it.
Before doing that, we check when the user certificate is expiring. If it's within the timeleftthreshold (myproxytimeleft < timeleftthreshold)
we delegate the proxy just for the time we need (checking first if we did not already do it since at some point
usercertDaysLeft ~= myproxytimeleft and we don't need to delegate it at every command even though myproxytimeleft < timeleftthreshold).
Note that a warning message is printed at every command it usercertDaysLeft < timeleftthreshold
"""
myproxy = Proxy(self.defaultDelegation)
myproxy.userDN = myproxy.getSubjectFromCert(self.certLocation)
myproxytimeleft = 0
self.logger.debug("Getting myproxy life time left for %s" %
self.defaultDelegation["myProxySvr"])
# return an integer that indicates the number of seconds to the expiration of the proxy in myproxy
myproxytimeleft = myproxy.getMyProxyTimeLeft(serverRenewer=True,
nokey=nokey)
self.logger.debug("Myproxy is valid: %i" % myproxytimeleft)
trustRetrListChanged = myproxy.trustedRetrievers != self.defaultDelegation[
'serverDN'] #list on the REST and on myproxy are different
if myproxytimeleft < timeleftthreshold or self.proxyChanged or trustRetrListChanged:
# checking the enddate of the user certificate
usercertDaysLeft = myproxy.getUserCertEnddate()
if usercertDaysLeft == 0:
msg = "%sYOUR USER CERTIFICATE IS EXPIRED (OR WILL EXPIRE TODAY). YOU CANNOT USE THE CRAB3 CLIENT. PLEASE REQUEST A NEW CERTIFICATE HERE https://gridca.cern.ch/gridca/ AND SEE https://ca.cern.ch/ca/Help/?kbid=024010%s"\
% (colors.RED, colors.NORMAL)
raise ProxyCreationException(msg)
#if the certificate is going to expire print a warning. This is going to bre printed at every command if
#the myproxytimeleft is inferior to the timeleftthreshold
if usercertDaysLeft < self.myproxyDesiredValidity:
self.logger.info("%sYour user certificate is going to expire in %s days. https://twiki.cern.ch/twiki/bin/view/CMSPublic/WorkBookStartingGrid#ObtainingCert %s"\
% (colors.RED, usercertDaysLeft, colors.NORMAL) )
#check if usercertDaysLeft ~= myproxytimeleft which means we already delegated the proxy for as long as we could
if abs(
usercertDaysLeft * 60 * 60 * 24 - myproxytimeleft
) < 60 * 60 * 24 and not trustRetrListChanged: #less than one day between usercertDaysLeft and myproxytimeleft
return
#adjust the myproxy delegation time accordingly to the user cert validity
self.logger.info("%sDelegating your proxy for %s days instead of %s %s"\
% (colors.RED, usercertDaysLeft, self.myproxyDesiredValidity, colors.NORMAL) )
myproxy.myproxyValidity = "%i:00" % (usercertDaysLeft * 24)
# creating the proxy
self.logger.debug("Delegating a myproxy for %s hours" %
myproxy.myproxyValidity)
try:
myproxy.delegate(serverRenewer=True, nokey=nokey)
myproxytimeleft = myproxy.getMyProxyTimeLeft(
serverRenewer=True, nokey=nokey)
if myproxytimeleft <= 0:
raise ProxyCreationException("It seems your proxy has not been delegated to myproxy. Please check the logfile for the exact error "+\
"(it might simply you typed a wrong password)")
else:
self.logger.debug("My-proxy delegated.")
except Exception as ex:
msg = ex._message if hasattr(ex, '_message') else str(ex)
raise ProxyCreationException(
"Problems delegating My-proxy. %s" % msg)
def __init__(self, confFile=None, quiet=False, debug=True, testMode=False):
"""
Initialise class members
:arg WMCore.Configuration config: input Publisher configuration
:arg bool quiet: it tells if a quiet logger is needed
:arg bool debug: it tells if needs a verbose logger
:arg bool testMode: it tells if to run in test (no subprocesses) mode.
"""
def createLogdir(dirname):
""" Create the directory dirname ignoring erors in case it exists. Exit if
the directory cannot be created.
"""
try:
os.makedirs(dirname)
except OSError as ose:
if ose.errno != 17: #ignore the "Directory already exists error"
print(str(ose))
print(
"The Publisher Worker needs to access the '%s' directory"
% dirname)
sys.exit(1)
def setRootLogger(logsDir, quiet=False, debug=True, console=False):
"""Sets the root logger with the desired verbosity level
The root logger logs to logs/log.txt and every single
logging instruction is propagated to it (not really nice
to read)
:arg bool quiet: it tells if a quiet logger is needed
:arg bool debug: it tells if needs a verbose logger
:arg bool console: it tells if to direct all printoput to console rather then files, useful for debug
:return logger: a logger with the appropriate logger level."""
createLogdir(logsDir)
createLogdir(os.path.join(logsDir, 'processes'))
createLogdir(os.path.join(logsDir, 'tasks'))
if console:
# if we are testing log to the console is easier
logging.getLogger().addHandler(logging.StreamHandler())
else:
logHandler = MultiProcessingLog(os.path.join(
logsDir, 'log.txt'),
when='midnight')
logFormatter = logging.Formatter(
"%(asctime)s:%(levelname)s:%(module)s,%(lineno)d:%(message)s"
)
logHandler.setFormatter(logFormatter)
logging.getLogger().addHandler(logHandler)
loglevel = logging.INFO
if quiet:
loglevel = logging.WARNING
if debug:
loglevel = logging.DEBUG
logging.getLogger().setLevel(loglevel)
logger = setMasterLogger()
logger.debug("PID %s.", os.getpid())
logger.debug("Logging level initialized to %s.", loglevel)
return logger
def logVersionAndConfig(config=None, logger=None):
"""
log version number and major config. parameters
args: config : a configuration object loaded from file
args: logger : the logger instance to use
"""
pubstartDict = {}
pubstartDict['version'] = __version__
pubstartDict['asoworker'] = config.General.asoworker
pubstartDict['instance'] = config.General.instance
if config.General.instance == 'other':
pubstartDict['restHost'] = config.General.restHost
pubstartDict['dbInstance'] = config.General.dbInstance
pubstartDict['max_slaves'] = config.General.max_slaves
pubstartDict['DBShost'] = config.TaskPublisher.DBShost
pubstartDict['dryRun'] = config.TaskPublisher.dryRun
# one line for automatic parsing
logger.info('PUBSTART: %s', json.dumps(pubstartDict))
# multiple lines for humans to read
for k, v in pubstartDict.items():
logger.info('%s: %s', k, v)
return
self.configurationFile = confFile # remember this, will have to pass it to TaskPublish
config = loadConfigurationFile(confFile)
self.config = config.General
self.TPconfig = config.TaskPublisher
# these are used for talking to DBS
os.putenv('X509_USER_CERT', self.config.serviceCert)
os.putenv('X509_USER_KEY', self.config.serviceKey)
self.block_publication_timeout = self.config.block_closure_timeout
self.lfn_map = {}
self.force_publication = False
self.force_failure = False
self.TestMode = testMode
self.taskFilesDir = self.config.taskFilesDir
createLogdir(self.taskFilesDir)
createLogdir(os.path.join(self.taskFilesDir, 'FailedBlocks'))
self.logger = setRootLogger(self.config.logsDir,
quiet=quiet,
debug=debug,
console=self.TestMode)
logVersionAndConfig(config, self.logger)
from WMCore.Credential.Proxy import Proxy
proxy = Proxy({'logger': self.logger})
from ServerUtilities import tempSetLogLevel
with tempSetLogLevel(self.logger, logging.ERROR):
self.myDN = proxy.getSubjectFromCert(
certFile=self.config.serviceCert)
try:
instance = self.config.instance
except:
msg = "No instance provided: need to specify config.General.instance in the configuration"
raise ConfigException(msg)
if instance in SERVICE_INSTANCES:
self.logger.info('Will connect to CRAB service: %s', instance)
restHost = SERVICE_INSTANCES[instance]['restHost']
dbInstance = SERVICE_INSTANCES[instance]['dbInstance']
else:
msg = "Invalid instance value '%s'" % instance
raise ConfigException(msg)
if instance == 'other':
self.logger.info(
'Will use restHost and dbInstance from config file')
try:
restHost = self.config.restHost
dbInstance = self.config.dbInstance
except:
msg = "Need to specify config.General.restHost and dbInstance in the configuration"
raise ConfigException(msg)
self.logger.info(
'Will connect to CRAB Data Base %s instance via URL: https://%s',
dbInstance, restHost)
# CRAB REST API's
self.max_files_per_block = self.config.max_files_per_block
self.crabServer = CRABRest(hostname=restHost,
localcert=self.config.serviceCert,
localkey=self.config.serviceKey,
retry=3,
userAgent='CRABPublisher')
self.crabServer.setDbInstance(dbInstance=dbInstance)
self.startTime = time.time()
def createNewMyProxy(self, timeleftthreshold=0, nokey=False):
"""
Handles the MyProxy creation
Let the following variables be
timeleftthreshold: the proxy in myproxy should be delegated for at least this time (14 days)
myproxytimeleft: current validity of your proxy in myproxy
usercertDaysLeft: the number of days left before your user certificate expire
myproxyDesiredValidity: delegate the proxy in myproxy for that time (30 days)
If we need to renew the proxy in myproxy because its atributes has changed or because it is valid for
less time than timeleftthreshold then we do it.
Before doing that, we check when the user certificate is expiring. If it's within the timeleftthreshold (myproxytimeleft < timeleftthreshold)
we delegate the proxy just for the time we need (checking first if we did not already do it since at some point
usercertDaysLeft ~= myproxytimeleft and we don't need to delegate it at every command even though myproxytimeleft < timeleftthreshold).
Note that a warning message is printed at every command it usercertDaysLeft < timeleftthreshold
"""
myproxy = Proxy ( self.defaultDelegation )
myproxy.userDN = myproxy.getSubjectFromCert(self.certLocation)
myproxytimeleft = 0
self.logger.debug("Getting myproxy life time left for %s" % self.defaultDelegation["myProxySvr"])
# return an integer that indicates the number of seconds to the expiration of the proxy in myproxy
# Also catch the exception in case WMCore encounters a problem with the proxy itself (one such case was #4532)
try:
myproxytimeleft = myproxy.getMyProxyTimeLeft(serverRenewer=True, nokey=nokey)
except Exception as ex:
logging.exception("Problems calculating proxy lifetime, logging stack trace and raising ProxyCreationException")
# WMException may contain the _message attribute. Otherwise, take the exception as a string.
msg = ex._message if hasattr(ex, "_message") else str(ex)
raise ProxyCreationException("Problems calculating the time left until the expiration of the proxy."
" Please reset your environment or contact [email protected] if the problem persists.\n%s" % msg)
self.logger.debug("Myproxy is valid: %i" % myproxytimeleft)
trustRetrListChanged = myproxy.trustedRetrievers!=self.defaultDelegation['serverDN'] #list on the REST and on myproxy are different
if myproxytimeleft < timeleftthreshold or self.proxyChanged or trustRetrListChanged:
# checking the enddate of the user certificate
usercertDaysLeft = myproxy.getUserCertEnddate()
if usercertDaysLeft == 0:
msg = "%sYOUR USER CERTIFICATE IS EXPIRED (OR WILL EXPIRE TODAY). YOU CANNOT USE THE CRAB3 CLIENT. PLEASE REQUEST A NEW CERTIFICATE HERE https://gridca.cern.ch/gridca/ AND SEE https://ca.cern.ch/ca/Help/?kbid=024010%s"\
% (colors.RED, colors.NORMAL)
raise ProxyCreationException(msg)
#if the certificate is going to expire print a warning. This is going to bre printed at every command if
#the myproxytimeleft is inferior to the timeleftthreshold
if usercertDaysLeft < self.myproxyDesiredValidity:
self.logger.info("%sYour user certificate is going to expire in %s days. https://twiki.cern.ch/twiki/bin/view/CMSPublic/WorkBookStartingGrid#ObtainingCert %s"\
% (colors.RED, usercertDaysLeft, colors.NORMAL) )
#check if usercertDaysLeft ~= myproxytimeleft which means we already delegated the proxy for as long as we could
if abs(usercertDaysLeft*60*60*24 - myproxytimeleft) < 60*60*24 and not trustRetrListChanged: #less than one day between usercertDaysLeft and myproxytimeleft
return
#adjust the myproxy delegation time accordingly to the user cert validity
self.logger.info("%sDelegating your proxy for %s days instead of %s %s"\
% (colors.RED, usercertDaysLeft, self.myproxyDesiredValidity, colors.NORMAL) )
myproxy.myproxyValidity = "%i:00" % (usercertDaysLeft*24)
# creating the proxy
self.logger.debug("Delegating a myproxy for %s hours" % myproxy.myproxyValidity )
try:
myproxy.delegate(serverRenewer = True, nokey=nokey)
myproxytimeleft = myproxy.getMyProxyTimeLeft(serverRenewer=True, nokey=nokey)
if myproxytimeleft <= 0:
raise ProxyCreationException("It seems your proxy has not been delegated to myproxy. Please check the logfile for the exact error "+\
"(it might simply you typed a wrong password)")
else:
self.logger.debug("My-proxy delegated.")
except Exception as ex:
msg = ex._message if hasattr(ex, '_message') else str(ex)
raise ProxyCreationException("Problems delegating My-proxy. %s" % msg)
def __init__(self, confFile=None, quiet=False, debug=True, testMode=False):
"""
Initialise class members
:arg WMCore.Configuration config: input Publisher configuration
:arg bool quiet: it tells if a quiet logger is needed
:arg bool debug: it tells if needs a verbose logger
:arg bool testMode: it tells if to run in test (no subprocesses) mode.
"""
def createLogdir(dirname):
""" Create the directory dirname ignoring erors in case it exists. Exit if
the directory cannot be created.
"""
try:
os.makedirs(dirname)
except OSError as ose:
if ose.errno != 17: #ignore the "Directory already exists error"
print(str(ose))
print(
"The Publisher Worker needs to access the '%s' directory"
% dirname)
sys.exit(1)
self.configurationFile = confFile # remember this, will have to pass it to TaskPublish
config = loadConfigurationFile(confFile)
self.config = config.General
self.TPconfig = config.TaskPublisher
# these are used for talking to DBS
os.putenv('X509_USER_CERT', self.config.serviceCert)
os.putenv('X509_USER_KEY', self.config.serviceKey)
self.block_publication_timeout = self.config.block_closure_timeout
self.lfn_map = {}
self.force_publication = False
self.force_failure = False
self.TestMode = testMode
self.taskFilesDir = self.config.taskFilesDir
createLogdir(self.taskFilesDir)
def setRootLogger(logsDir, quiet=False, debug=True, console=False):
"""Sets the root logger with the desired verbosity level
The root logger logs to logs/twlog.txt and every single
logging instruction is propagated to it (not really nice
to read)
:arg bool quiet: it tells if a quiet logger is needed
:arg bool debug: it tells if needs a verbose logger
:arg bool console: it tells if to direct all printoput to console rather then files, useful for debug
:return logger: a logger with the appropriate logger level."""
createLogdir(logsDir)
createLogdir(logsDir + '/processes')
createLogdir(logsDir + '/tasks')
if console:
# if we are testing log to the console is easier
logging.getLogger().addHandler(logging.StreamHandler())
else:
logHandler = MultiProcessingLog(logsDir + '/log.txt',
when='midnight')
logFormatter = logging.Formatter(
"%(asctime)s:%(levelname)s:%(module)s,%(lineno)d:%(message)s"
)
logHandler.setFormatter(logFormatter)
logging.getLogger().addHandler(logHandler)
loglevel = logging.INFO
if quiet:
loglevel = logging.WARNING
if debug:
loglevel = logging.DEBUG
logging.getLogger().setLevel(loglevel)
logger = setMasterLogger()
logger.debug("PID %s.", os.getpid())
logger.debug("Logging level initialized to %s.", loglevel)
return logger
self.logger = setRootLogger(self.config.logsDir,
quiet=quiet,
debug=debug,
console=self.TestMode)
from WMCore.Credential.Proxy import Proxy
proxy = Proxy({'logger': self.logger})
from ServerUtilities import tempSetLogLevel
with tempSetLogLevel(self.logger, logging.ERROR):
self.myDN = proxy.getSubjectFromCert(
certFile=self.config.serviceCert)
# CRABServer REST API's (see CRABInterface)
try:
instance = self.config.instance
except:
msg = "No instance provided: need to specify config.General.instance in the configuration"
raise ConfigException(msg)
if instance in SERVICE_INSTANCES:
self.logger.info('Will connect to CRAB service: %s', instance)
restHost = SERVICE_INSTANCES[instance]['restHost']
dbInstance = SERVICE_INSTANCES[instance]['dbInstance']
else:
msg = "Invalid instance value '%s'" % instance
raise ConfigException(msg)
if instance == 'other':
self.logger.info(
'Will use restHost and dbInstance from config file')
try:
restHost = self.config.restHost
dbInstance = self.config.dbInstance
except:
msg = "Need to specify config.General.restHost and dbInstance in the configuration"
raise ConfigException(msg)
restURInoAPI = '/crabserver/' + dbInstance
self.logger.info(
'Will connect to CRAB Data Base via URL: https://%s/%s', restHost,
restURInoAPI)
# CRAB REST API's
self.REST_filetransfers = restURInoAPI + '/filetransfers'
#self.REST_usertransfers = restURInoAPI + '/fileusertransfers'
self.REST_filemetadata = restURInoAPI + '/filemetadata'
self.REST_workflow = restURInoAPI + '/workflow'
self.REST_task = restURInoAPI + '/task'
self.max_files_per_block = self.config.max_files_per_block
self.crabServer = HTTPRequests(url=restHost,
localcert=self.config.serviceCert,
localkey=self.config.serviceKey,
retry=3)
