def __getDirsToBundle( self ):
dirsToBundle = {}
result = gConfig.getOptionsDict( "%s/DirsToBundle" % self.__csPath )
if result[ 'OK' ]:
dB = result[ 'Value' ]
for bId in dB:
dirsToBundle[ bId ] = List.fromChar( dB[ bId ] )
if gConfig.getValue( "%s/BundleCAs" % self.__csPath, True ):
dirsToBundle[ 'CAs' ] = [ "%s/*.0" % Locations.getCAsLocation(), "%s/*.signing_policy" % Locations.getCAsLocation(), "%s/*.pem" % Locations.getCAsLocation() ]
if gConfig.getValue( "%s/BundleCRLs" % self.__csPath, True ):
dirsToBundle[ 'CRLs' ] = [ "%s/*.r0" % Locations.getCAsLocation() ]
return dirsToBundle
def generateCAFile():
"""
Generate a single CA file with all the PEMs
"""
caDir = Locations.getCAsLocation()
for fn in (os.path.join(os.path.dirname(caDir), "cas.pem"),
os.path.join(os.path.dirname(HTTPSCert()), "cas.pem"), False):
if not fn:
fn = tempfile.mkstemp(prefix="cas.", suffix=".pem")[1]
try:
fd = open(fn, "w")
except IOError:
continue
for caFile in os.listdir(caDir):
caFile = os.path.join(caDir, caFile)
result = X509Chain.X509Chain.instanceFromFile(caFile)
if not result['OK']:
continue
chain = result['Value']
expired = chain.hasExpired()
if not expired['OK'] or expired['Value']:
continue
fd.write(chain.dumpAllToString()['Value'])
fd.close()
return fn
return False
def generateCAFile():
"""
Generate a single CA file with all the PEMs
"""
caDir = Locations.getCAsLocation()
for fn in ( os.path.join( os.path.dirname( caDir ), "cas.pem" ),
os.path.join( os.path.dirname( HTTPSCert() ), "cas.pem" ),
False ):
if not fn:
fn = tempfile.mkstemp( prefix = "cas.", suffix = ".pem" )
try:
fd = open( fn, "w" )
except IOError:
continue
for caFile in os.listdir( caDir ):
caFile = os.path.join( caDir, caFile )
result = X509Chain.X509Chain.instanceFromFile( caFile )
if not result[ 'OK' ]:
continue
chain = result[ 'Value' ]
expired = chain.hasExpired()
if not expired[ 'OK' ] or expired[ 'Value' ]:
continue
fd.write( chain.dumpAllToString()[ 'Value' ] )
fd.close()
return fn
return False
def generateRevokedCertsFile(location=None):
"""
Generate a single CA file with all the PEMs
:param str location: we can specify a specific location in CS
:return: file crls.pem which contains all revoked certificates
"""
caDir = Locations.getCAsLocation()
for fn in (
os.path.join(os.path.dirname(caDir), "crls.pem"),
os.path.join(
os.path.dirname(
Locations.getHostCertificateAndKeyLocation(location)[0]),
"crls.pem"), False):
if not fn:
fn = tempfile.mkstemp(prefix="crls", suffix=".pem")[1]
try:
with open(fn, "w") as fd:
for caFile in os.listdir(caDir):
caFile = os.path.join(caDir, caFile)
result = X509CRL.X509CRL.instanceFromFile(caFile)
if not result['OK']:
continue
chain = result['Value']
fd.write(chain.dumpAllToString()['Value'])
return S_OK(fn)
except IOError:
continue
return S_ERROR(caDir)
def generateCAFile():
"""
Generate a single CA file with all the PEMs
"""
caDir = Locations.getCAsLocation()
for fn in (os.path.join(os.path.dirname(caDir), "cas.pem"),
os.path.join(os.path.dirname(getCert()), "cas.pem"), False):
if not fn:
fn = tempfile.mkstemp(prefix="cas.", suffix=".pem")[1]
try:
with open(fn, "w") as fd:
for caFile in os.listdir(caDir):
caFile = os.path.join(caDir, caFile)
result = X509Chain.X509Chain.instanceFromFile(caFile)
if not result['OK']:
continue
chain = result['Value']
expired = chain.hasExpired()
if not expired['OK'] or expired['Value']:
continue
fd.write(chain.dumpAllToString()['Value'])
gLogger.info("CAs used from: %s" % str(fn))
return fn
except IOError as err:
gLogger.warn(err)
return False
def generateRevokedCertsFile(location=None):
"""
Generate a single CA file with all the PEMs
:param str location: we can specify a specific location in CS
:return file crls.pem which contains all revoked certificates
"""
caDir = Locations.getCAsLocation()
for fn in (os.path.join(os.path.dirname(caDir), "crls.pem"),
os.path.join(os.path.dirname(Locations.getHostCertificateAndKeyLocation(location)[0]), "crls.pem"),
False):
if not fn:
fn = tempfile.mkstemp(prefix="crls", suffix=".pem")[1]
try:
with open(fn, "w") as fd:
for caFile in os.listdir(caDir):
caFile = os.path.join(caDir, caFile)
result = X509CRL.X509CRL.instanceFromFile(caFile)
if not result['OK']:
continue
chain = result['Value']
fd.write(chain.dumpAllToString()['Value'])
return S_OK(fn)
except IOError:
continue
return S_ERROR(caDir)
def syncCRLs(self):
X509_CERT_DIR = False
if 'X509_CERT_DIR' in os.environ:
X509_CERT_DIR = os.environ['X509_CERT_DIR']
del os.environ['X509_CERT_DIR']
result = self.syncDir("CRLs", Locations.getCAsLocation())
if X509_CERT_DIR:
os.environ['X509_CERT_DIR'] = X509_CERT_DIR
return result
def syncCRLs( self ):
X509_CERT_DIR = False
if 'X509_CERT_DIR' in os.environ:
X509_CERT_DIR = os.environ['X509_CERT_DIR']
del os.environ['X509_CERT_DIR']
result = self.syncDir( "CRLs", Locations.getCAsLocation() )
if X509_CERT_DIR:
os.environ['X509_CERT_DIR'] = X509_CERT_DIR
return result
def syncCAs( self ):
X509_CERT_DIR = False
if 'X509_CERT_DIR' in os.environ:
X509_CERT_DIR = os.environ['X509_CERT_DIR']
del os.environ['X509_CERT_DIR']
casLocation = Locations.getCAsLocation()
if not casLocation:
casLocation = Locations.getCAsDefaultLocation()
result = self.syncDir( "CAs", casLocation )
if X509_CERT_DIR:
os.environ['X509_CERT_DIR'] = X509_CERT_DIR
return result
def syncCAs(self):
X509_CERT_DIR = False
if 'X509_CERT_DIR' in os.environ:
X509_CERT_DIR = os.environ['X509_CERT_DIR']
del os.environ['X509_CERT_DIR']
casLocation = Locations.getCAsLocation()
if not casLocation:
casLocation = Locations.getCAsDefaultLocation()
result = self.syncDir("CAs", casLocation)
if X509_CERT_DIR:
os.environ['X509_CERT_DIR'] = X509_CERT_DIR
return result
def syncCRLs(self):
"""Synchronize CRLs
:return: S_OK(bool)/S_ERROR()
"""
X509_CERT_DIR = False
if "X509_CERT_DIR" in os.environ:
X509_CERT_DIR = os.environ["X509_CERT_DIR"]
del os.environ["X509_CERT_DIR"]
result = self.syncDir("CRLs", Locations.getCAsLocation())
if X509_CERT_DIR:
os.environ["X509_CERT_DIR"] = X509_CERT_DIR
return result
def startTornado(self):
"""
Starts the tornado server when ready.
This method never returns.
"""
sLog.debug("Starting Tornado")
self._initMonitoring()
router = Application(self.urls, debug=False, compress_response=True)
certs = Locations.getHostCertificateAndKeyLocation()
if certs is False:
sLog.fatal("Host certificates not found ! Can't start the Server")
raise ImportError("Unable to load certificates")
ca = Locations.getCAsLocation()
ssl_options = {
"certfile": certs[0],
"keyfile": certs[1],
"cert_reqs": M2Crypto.SSL.verify_peer,
"ca_certs": ca,
"sslDebug": DEBUG_M2CRYPTO, # Set to true if you want to see the TLS debug messages
}
self.__monitorLastStatsUpdate = time.time()
self.__report = self.__startReportToMonitoringLoop()
# Starting monitoring, IOLoop waiting time in ms, __monitoringLoopDelay is defined in seconds
tornado.ioloop.PeriodicCallback(self.__reportToMonitoring, self.__monitoringLoopDelay * 1000).start()
# If we are running with python3, Tornado will use asyncio,
# and we have to convince it to let us run in a different thread
# Doing this ensures a consistent behavior between py2 and py3
if six.PY3:
import asyncio # pylint: disable=import-error
asyncio.set_event_loop_policy(tornado.platform.asyncio.AnyThreadEventLoopPolicy())
# Start server
server = HTTPServer(router, ssl_options=ssl_options, decompress_request=True)
try:
server.listen(self.port)
except Exception as e: # pylint: disable=broad-except
sLog.exception("Exception starting HTTPServer", e)
raise
sLog.always("Listening on port %s" % self.port)
for service in self.urls:
sLog.debug("Available service: %s" % service)
IOLoop.current().start()
def generateCAFile(location=None):
"""
Generate/find a single CA file with all the PEMs
:param str location: we can specify a specific CS location
where it's written a directory where to find the CAs and CRLs
:return: directory where the file cas.pem which contains all certificates is found/created
"""
caDir = Locations.getCAsLocation()
if not caDir:
return S_ERROR('No CAs dir found')
# look in what's normally /etc/grid-security/certificates
if os.path.isfile(os.path.join(os.path.dirname(caDir), "cas.pem")):
return S_OK(os.path.join(os.path.dirname(caDir), "cas.pem"))
# look in what's normally /opt/dirac/etc/grid-security
diracCADirPEM = os.path.join(
os.path.dirname(
Locations.getHostCertificateAndKeyLocation(location)[0]),
"cas.pem")
if os.path.isfile(diracCADirPEM):
return S_OK(diracCADirPEM)
# Now we create it in tmpdir
fn = tempfile.mkstemp(prefix="cas.", suffix=".pem")[1]
try:
with open(fn, "w") as fd:
for caFile in os.listdir(caDir):
caFile = os.path.join(caDir, caFile)
chain = X509Chain.X509Chain()
result = chain.loadChainFromFile(caFile)
if not result['OK']:
continue
expired = chain.hasExpired()
if not expired['OK'] or expired['Value']:
continue
fd.write(chain.dumpAllToString()['Value'])
gLogger.info("CAs used from: %s" % str(fn))
return S_OK(fn)
except IOError as err:
gLogger.warn(err)
return S_ERROR("Could not find/generate CAs")
def syncCAs(self):
"""Synchronize CAs
:return: S_OK(bool)/S_ERROR()
"""
X509_CERT_DIR = False
if "X509_CERT_DIR" in os.environ:
X509_CERT_DIR = os.environ["X509_CERT_DIR"]
del os.environ["X509_CERT_DIR"]
casLocation = Locations.getCAsLocation()
if not casLocation:
casLocation = Locations.getCAsDefaultLocation()
result = self.syncDir("CAs", casLocation)
if X509_CERT_DIR:
os.environ["X509_CERT_DIR"] = X509_CERT_DIR
return result
def startTornado(self):
"""
Starts the tornado server when ready.
This method never returns.
"""
sLog.debug("Starting Tornado")
self._initMonitoring()
router = Application(self.urls, debug=False, compress_response=True)
certs = Locations.getHostCertificateAndKeyLocation()
if certs is False:
sLog.fatal("Host certificates not found ! Can't start the Server")
raise ImportError("Unable to load certificates")
ca = Locations.getCAsLocation()
ssl_options = {
'certfile': certs[0],
'keyfile': certs[1],
'cert_reqs': M2Crypto.SSL.verify_peer,
'ca_certs': ca,
'sslDebug':
False, # Set to true if you want to see the TLS debug messages
}
self.__monitorLastStatsUpdate = time.time()
self.__report = self.__startReportToMonitoringLoop()
# Starting monitoring, IOLoop waiting time in ms, __monitoringLoopDelay is defined in seconds
tornado.ioloop.PeriodicCallback(self.__reportToMonitoring,
self.__monitoringLoopDelay *
1000).start()
# Start server
server = HTTPServer(router,
ssl_options=ssl_options,
decompress_request=True)
try:
server.listen(self.port)
except Exception as e: # pylint: disable=broad-except
sLog.exception("Exception starting HTTPServer", e)
raise
sLog.always("Listening on port %s" % self.port)
for service in self.urls:
sLog.debug("Available service: %s" % service)
IOLoop.current().start()
def generateRevokedCertsFile(location=None):
"""
Generate a single CA file with all the PEMs
:param str location: we can specify a specific CS location
where it's written a directory where to find the CAs and CRLs
:return: directory where the file crls.pem which contains all CRLs is created
"""
caDir = Locations.getCAsLocation()
if not caDir:
return S_ERROR("No CAs dir found")
# look in what's normally /etc/grid-security/certificates
if os.path.isfile(os.path.join(os.path.dirname(caDir), "crls.pem")):
return S_OK(os.path.join(os.path.dirname(caDir), "crls.pem"))
# look in what's normally /opt/dirac/etc/grid-security
diracCADirPEM = os.path.join(
os.path.dirname(
Locations.getHostCertificateAndKeyLocation(location)[0]),
"crls.pem")
if os.path.isfile(diracCADirPEM):
return S_OK(diracCADirPEM)
# Now we create it in tmpdir
fn = tempfile.mkstemp(prefix="crls", suffix=".pem")[1]
try:
with open(fn, "wb") as fd:
for caFile in os.listdir(caDir):
caFile = os.path.join(caDir, caFile)
result = X509CRL.X509CRL.instanceFromFile(caFile)
if not result["OK"]:
continue
chain = result["Value"]
fd.write(chain.dumpAllToString()["Value"])
return S_OK(fn)
except IOError as err:
gLogger.warn(err)
return S_ERROR("Could not find/generate CRLs")
def generateCAFile(location=None):
"""
Generate a single CA file with all the PEMs
:param str location: we can specify a specific location in CS
:return: file cas.pem which contains all certificates
"""
caDir = Locations.getCAsLocation()
for fn in (
os.path.join(os.path.dirname(caDir), "cas.pem"),
os.path.join(
os.path.dirname(
Locations.getHostCertificateAndKeyLocation(location)[0]),
"cas.pem"), False):
if not fn:
fn = tempfile.mkstemp(prefix="cas.", suffix=".pem")[1]
try:
with open(fn, "w") as fd:
for caFile in os.listdir(caDir):
caFile = os.path.join(caDir, caFile)
chain = X509Chain.X509Chain()
result = chain.loadChainFromFile(caFile)
if not result['OK']:
continue
expired = chain.hasExpired()
if not expired['OK'] or expired['Value']:
continue
fd.write(chain.dumpAllToString()['Value'])
gLogger.info("CAs used from: %s" % str(fn))
return S_OK(fn)
except IOError as err:
gLogger.warn(err)
return S_ERROR(caDir)
def initialize(self):
"""agent's initialisation
:param self: self reference
"""
self.log.info("Starting RucioRSSAgent")
# CA location
self.caCertPath = Locations.getCAsLocation()
# configured VOs
res = getVOs()
if res["OK"]:
voList = getVOs().get("Value", [])
else:
return S_ERROR(res["Message"])
# configHelper test
if isinstance(voList, str):
voList = [voList]
self.clientConfig = configHelper(voList)
self.log.debug(" VO-specific Rucio Client config parameters: ",
self.clientConfig)
return S_OK()
def generateCAFile(location=None):
"""
Generate a single CA file with all the PEMs
:param str location: we can specify a specific location in CS
:return file cas.pem which contains all certificates
"""
caDir = Locations.getCAsLocation()
for fn in (os.path.join(os.path.dirname(caDir), "cas.pem"),
os.path.join(os.path.dirname(Locations.getHostCertificateAndKeyLocation(location)[0]), "cas.pem"),
False):
if not fn:
fn = tempfile.mkstemp(prefix="cas.", suffix=".pem")[1]
try:
with open(fn, "w") as fd:
for caFile in os.listdir(caDir):
caFile = os.path.join(caDir, caFile)
result = X509Chain.X509Chain.instanceFromFile(caFile)
if not result['OK']:
continue
chain = result['Value']
expired = chain.hasExpired()
if not expired['OK'] or expired['Value']:
continue
fd.write(chain.dumpAllToString()['Value'])
gLogger.info("CAs used from: %s" % str(fn))
return S_OK(fn)
except IOError as err:
gLogger.warn(err)
return S_ERROR(caDir)
def getM2SSLContext(ctx=None, **kwargs):
""" Gets an M2Crypto.SSL.Context configured using the standard
DIRAC connection keywords from kwargs. The keywords are:
- clientMode: Boolean, if False hostcerts are always used. If True
a proxy is used unless other flags are set.
- useCertificates: Boolean, Set to true to use hostcerts in client
mode.
- proxyString: String, no-longer supported, used to allow a literal
proxy string to be provided.
- proxyLocation: String, Path to file to use as proxy, defaults to
usual location(s) if not set.
- skipCACheck: Boolean, if True, don't verify peer certificates.
- sslMethod: String, List of SSL algorithms to enable in OpenSSL style
cipher format, e.g. "SSLv3:TLSv1".
- sslCiphers: String, OpenSSL style cipher string of ciphers to allow
on this connection.
If an existing context "ctx" is provided, it is just reconfigured with
the selected arguments.
Returns the new or updated context.
"""
if not ctx:
ctx = SSL.Context()
# Set certificates for connection
# CHRIS: I think clientMode was just an internal of pyGSI implementation
# if kwargs.get('clientMode', False) and not kwargs.get('useCertificates', False):
# if not kwargs.get('useCertificates', False):
if kwargs.get('bServerMode', False) or kwargs.get('useCertificates',
False):
# Server mode always uses hostcert
__loadM2SSLCTXHostcert(ctx)
else:
# Client mode has a choice of possible options
if kwargs.get('proxyString', None):
# We don't support this any more, there is no easy way
# to convert a proxy string to something usable by M2Crypto SSL
# Try writing it to a temp file and use proxyLocation instead?
raise RuntimeError("Proxy string no longer supported.")
else:
# Use normal proxy
__loadM2SSLCTXProxy(ctx,
proxyPath=kwargs.get('proxyLocation', None))
# Set peer verification
if kwargs.get('skipCACheck', False):
# Don't validate peer, but still request creds
ctx.set_verify(SSL.verify_none, VERIFY_DEPTH)
else:
# Do validate peer
ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert,
VERIFY_DEPTH)
# Set CA location
caPath = Locations.getCAsLocation()
if not caPath:
raise RuntimeError("Failed to find CA location")
if not os.path.isdir(caPath):
raise RuntimeError("CA path (%s) is not a valid directory" %
caPath)
ctx.load_verify_locations(capath=caPath)
# Other parameters
sslMethod = kwargs.get('sslMethod', None)
if sslMethod:
# Pylint can't see the m2 constants due to the way the library is loaded
# We just have to disable that warning for the next bit...
# pylint: disable=no-member
methods = [('SSLv2', m2.SSL_OP_NO_SSLv2),
('SSLv3', m2.SSL_OP_NO_SSLv3),
('TLSv1', m2.SSL_OP_NO_TLSv1)]
allowed_methods = sslMethod.split(':')
# If a method isn't explicitly allowed, set the flag to disable it...
for method, method_flag in methods:
if method not in allowed_methods:
ctx.set_options(method_flag)
# SSL_OP_NO_SSLv2, SSL_OP_NO_SSLv3, SSL_OP_NO_TLSv1
ciphers = kwargs.get('sslCiphers', DEFAULT_SSL_CIPHERS)
ctx.set_cipher_list(ciphers)
# log the debug messages
# ctx.set_info_callback()
return ctx
def __getCAStore(self):
SocketInfo.__cachedCAsCRLsLoadLock.acquire()
try:
if not SocketInfo.__cachedCAsCRLs or time.time() - SocketInfo.__cachedCAsCRLsLastLoaded > 900:
# Need to generate the CA Store
casDict = {}
crlsDict = {}
casPath = Locations.getCAsLocation()
if not casPath:
return S_ERROR("No valid CAs location found")
gLogger.debug("CAs location is %s" % casPath)
casFound = 0
crlsFound = 0
SocketInfo.__caStore = GSI.crypto.X509Store()
for fileName in os.listdir(casPath):
filePath = os.path.join(casPath, fileName)
if not os.path.isfile(filePath):
continue
fObj = file(filePath, "rb")
pemData = fObj.read()
fObj.close()
# Try to load CA Cert
try:
caCert = GSI.crypto.load_certificate(GSI.crypto.FILETYPE_PEM, pemData)
if caCert.has_expired():
continue
caID = (caCert.get_subject().one_line(),
caCert.get_issuer().one_line())
caNotAfter = caCert.get_not_after()
if caID not in casDict:
casDict[caID] = (caNotAfter, caCert)
casFound += 1
else:
if casDict[caID][0] < caNotAfter:
casDict[caID] = (caNotAfter, caCert)
continue
except BaseException:
if fileName.find(".0") == len(fileName) - 2:
gLogger.exception("LOADING %s" % filePath)
if 'IgnoreCRLs' not in self.infoDict or not self.infoDict['IgnoreCRLs']:
# Try to load CRL
try:
crl = GSI.crypto.load_crl(GSI.crypto.FILETYPE_PEM, pemData)
if crl.has_expired():
continue
crlID = crl.get_issuer().one_line()
crlsDict[crlID] = crl
crlsFound += 1
continue
except Exception as e:
if fileName.find(".r0") == len(fileName) - 2:
gLogger.exception("LOADING %s ,Exception: %s" %
(filePath, str(e)))
gLogger.debug("Loaded %s CAs [%s CRLs]" % (casFound, crlsFound))
SocketInfo.__cachedCAsCRLs = ([casDict[k][1] for k in casDict],
[crlsDict[k] for k in crlsDict])
SocketInfo.__cachedCAsCRLsLastLoaded = time.time()
except BaseException:
gLogger.exception("Failed to init CA store")
finally:
SocketInfo.__cachedCAsCRLsLoadLock.release()
# Generate CA Store
caStore = GSI.crypto.X509Store()
caList = SocketInfo.__cachedCAsCRLs[0]
for caCert in caList:
caStore.add_cert(caCert)
crlList = SocketInfo.__cachedCAsCRLs[1]
for crl in crlList:
caStore.add_crl(crl)
return S_OK(caStore)
def startTornado(self):
"""
Starts the tornado server when ready.
This method never returns.
"""
# If there is no services loaded:
if not self.__calculateAppSettings():
raise Exception(
"There is no services loaded, please check your configuration")
sLog.debug("Starting Tornado")
# Prepare SSL settings
certs = Locations.getHostCertificateAndKeyLocation()
if certs is False:
sLog.fatal("Host certificates not found ! Can't start the Server")
raise ImportError("Unable to load certificates")
ca = Locations.getCAsLocation()
ssl_options = {
"certfile": certs[0],
"keyfile": certs[1],
"cert_reqs": M2Crypto.SSL.verify_peer,
"ca_certs": ca,
"sslDebug":
DEBUG_M2CRYPTO, # Set to true if you want to see the TLS debug messages
}
# Init monitoring
if self.activityMonitoring:
from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter
self.activityMonitoringReporter = MonitoringReporter(
monitoringType="ServiceMonitoring")
self.__monitorLastStatsUpdate = time.time()
self.__report = self.__startReportToMonitoringLoop()
# Response time
# Starting monitoring, IOLoop waiting time in ms, __monitoringLoopDelay is defined in seconds
tornado.ioloop.PeriodicCallback(
self.__reportToMonitoring(self.__elapsedTime),
self.__monitoringLoopDelay * 1000).start()
# If we are running with python3, Tornado will use asyncio,
# and we have to convince it to let us run in a different thread
# Doing this ensures a consistent behavior between py2 and py3
asyncio.set_event_loop_policy(
tornado.platform.asyncio.AnyThreadEventLoopPolicy())
for port, app in self.__appsSettings.items():
sLog.debug(" - %s" % "\n - ".join(
["%s = %s" % (k, ssl_options[k]) for k in ssl_options]))
# Default server configuration
settings = dict(compress_response=True, cookie_secret="secret")
# Merge appllication settings
settings.update(app["settings"])
# Start server
router = Application(app["routes"],
default_handler_class=NotFoundHandler,
**settings)
server = HTTPServer(router,
ssl_options=ssl_options,
decompress_request=True)
try:
server.listen(int(port))
except Exception as e: # pylint: disable=broad-except
sLog.exception("Exception starting HTTPServer", e)
raise
sLog.always("Listening on port %s" % port)
tornado.ioloop.IOLoop.current().start()
def getM2SSLContext(ctx=None, **kwargs):
"""Gets an M2Crypto.SSL.Context configured using the standard
DIRAC connection keywords from kwargs. The keywords are:
- clientMode: Boolean, if False hostcerts are always used. If True
a proxy is used unless other flags are set.
- useCertificates: Boolean, Set to true to use hostcerts in client
mode.
- proxyString: String, allow a literal proxy string to be provided.
- proxyLocation: String, Path to file to use as proxy, defaults to
usual location(s) if not set.
- skipCACheck: Boolean, if True, don't verify peer certificates.
- sslMethods: String, List of SSL algorithms to enable in OpenSSL style
cipher format, e.g. "SSLv3:TLSv1".
- sslCiphers: String, OpenSSL style cipher string of ciphers to allow
on this connection.
If an existing context "ctx" is provided, it is just reconfigured with
the selected arguments.
Returns the new or updated context.
"""
if not ctx:
ctx = SSL.Context()
# Set certificates for connection
# CHRIS: I think clientMode was just an internal of pyGSI implementation
# if kwargs.get('clientMode', False) and not kwargs.get('useCertificates', False):
# if not kwargs.get('useCertificates', False):
if kwargs.get("bServerMode", False) or kwargs.get("useCertificates",
False):
# Server mode always uses hostcert
__loadM2SSLCTXHostcert(ctx)
else:
# Client mode has a choice of possible options
if kwargs.get("proxyString", None):
# M2Crypto cannot take an inmemory location or a string, so
# so write it to a temp file and use proxyLocation
with tempfile.NamedTemporaryFile(mode="w") as tmpFile:
tmpFilePath = tmpFile.name
tmpFile.write(kwargs["proxyString"])
__loadM2SSLCTXProxy(ctx, proxyPath=tmpFilePath)
else:
# Use normal proxy
__loadM2SSLCTXProxy(ctx,
proxyPath=kwargs.get("proxyLocation", None))
verify_callback = ssl_verify_callback_print_error if DEBUG_M2CRYPTO else None
# Set peer verification
if kwargs.get("skipCACheck", False):
# Don't validate peer, but still request creds
ctx.set_verify(SSL.verify_none, VERIFY_DEPTH, callback=verify_callback)
else:
# Do validate peer
ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert,
VERIFY_DEPTH,
callback=verify_callback)
# Set CA location
caPath = Locations.getCAsLocation()
if not caPath:
raise RuntimeError("Failed to find CA location")
if not os.path.isdir(caPath):
raise RuntimeError("CA path (%s) is not a valid directory" %
caPath)
ctx.load_verify_locations(capath=caPath)
# If the version of M2Crypto is recent enough, there is an API
# to accept proxy certificate, and we do not need to rely on
# OPENSSL_ALLOW_PROXY_CERT environment variable
# which was removed as of openssl 1.1
# We need this to be merged in M2Crypto: https://gitlab.com/m2crypto/m2crypto/merge_requests/236
# We set the proper verify flag to the X509Store of the context
# as described here https://www.openssl.org/docs/man1.1.1/man7/proxy-certificates.html
if hasattr(SSL, "verify_allow_proxy_certs"):
ctx.get_cert_store().set_flags(SSL.verify_allow_proxy_certs) # pylint: disable=no-member
# As of M2Crypto 0.37, the `verify_allow_proxy_certs` flag was moved
# to X509 (https://gitlab.com/m2crypto/m2crypto/-/merge_requests/238)
# It is more consistent with all the other flags,
# but pySSL had it in SSL. Well...
if hasattr(X509, "verify_allow_proxy_certs"):
ctx.get_cert_store().set_flags(X509.verify_allow_proxy_certs) # pylint: disable=no-member
# Other parameters
sslMethods = kwargs.get("sslMethods", DEFAULT_SSL_METHODS)
if sslMethods:
# Pylint can't see the m2 constants due to the way the library is loaded
# We just have to disable that warning for the next bit...
# pylint: disable=no-member
methods = [("SSLv2", m2.SSL_OP_NO_SSLv2),
("SSLv3", m2.SSL_OP_NO_SSLv3),
("TLSv1", m2.SSL_OP_NO_TLSv1)]
allowed_methods = sslMethods.split(":")
# If a method isn't explicitly allowed, set the flag to disable it...
for method, method_flag in methods:
if method not in allowed_methods:
ctx.set_options(method_flag)
# SSL_OP_NO_SSLv2, SSL_OP_NO_SSLv3, SSL_OP_NO_TLSv1
ciphers = kwargs.get("sslCiphers", DEFAULT_SSL_CIPHERS)
ctx.set_cipher_list(ciphers)
# log the debug messages
if DEBUG_M2CRYPTO:
ctx.set_info_callback()
return ctx
def __init__(self, **options):
"""
Constructor. Takes options defined in Resources and Operations for this client.
:param options: options dict
"""
self.diracScopeAlg = options.get("DiracScopeAlg", "dirac")
self.useDiracCS = False # use a Rucio config file
self.convertUnicode = True
proxyInfo = {"OK": False}
if os.getenv("RUCIO_AUTH_TYPE"
) == "x509_proxy" and not os.getenv("X509_USER_PROXY"):
proxyInfo = getProxyInfo(disableVOMS=True)
if proxyInfo["OK"]:
os.environ["X509_USER_PROXY"] = proxyInfo["Value"]["path"]
sLog.debug("X509_USER_PROXY not defined. Using %s" %
proxyInfo["Value"]["path"])
try:
try:
self._client = Client()
self.account = self._client.account
except (CannotAuthenticate, MissingClientParameter):
if os.getenv("RUCIO_AUTH_TYPE") == "x509_proxy":
if not proxyInfo["OK"]:
proxyInfo = getProxyInfo(disableVOMS=True)
if proxyInfo["OK"]:
dn = proxyInfo["Value"]["identity"]
username = proxyInfo["Value"]["username"]
self.account = username
sLog.debug(
"Switching to account %s mapped to proxy %s" %
(username, dn))
try:
self._client = Client(account=self.account)
self.scopes = self._client.list_scopes()
except Exception as err:
sLog.error(
"Cannot instantiate RucioFileCatalog interface using a config file",
"error : %s" % repr(err))
sLog.info(("will try using Dirac CS"))
except Exception as err:
# instantiate the client w/o a config file
sLog.debug(
"instantiate the client w/o a config file - take config params from the CS"
)
self.useDiracCS = True
proxyInfo = getProxyInfo(disableVOMS=True)
if proxyInfo["OK"]:
proxyDict = proxyInfo["Value"]
self.proxyPath = proxyDict.get("path", None)
self.username = proxyDict.get("username", None)
else:
sLog.error("Cannot instantiate RucioFileCatalog interface",
proxyInfo["Message"])
return
self.VO = getVOfromProxyGroup()["Value"]
self.rucioHost = options.get("RucioHost", None)
self.authHost = options.get("AuthHost", None)
self.caCertPath = Locations.getCAsLocation()
try:
sLog.info("Logging in with a proxy located at: %s" %
self.proxyPath)
sLog.debug("account: ", self.username)
sLog.debug("rucio host: ", self.rucioHost)
sLog.debug("auth host: ", self.authHost)
sLog.debug("CA cert path: ", self.caCertPath)
sLog.debug("VO: ", self.VO)
self._client = Client(
account=self.username,
rucio_host=self.rucioHost,
auth_host=self.authHost,
ca_cert=self.caCertPath,
auth_type="x509_proxy",
creds={"client_proxy": self.proxyPath},
timeout=600,
user_agent="rucio-clients",
vo=self.VO,
)
sLog.debug(
"Rucio client instantiated successfully for VO %s and account %s "
% (self.VO, self.username))
except Exception as err:
sLog.error("Cannot instantiate RucioFileCatalog interface",
"error : %s" % repr(err))
def _request(self, retry=0, outputFile=None, **kwargs):
"""
Sends the request to server
:param retry: internal parameters for recursive call. TODO: remove ?
:param outputFile: (default None) path to a file where to store the received data.
If set, the server response will be streamed for optimization
purposes, and the response data will not go through the
JDecode process
:param **kwargs: Any argument there is used as a post parameter. They are detailed bellow.
:param method: (mandatory) name of the distant method
:param args: (mandatory) json serialized list of argument for the procedure
:returns: The received data. If outputFile is set, return always S_OK
"""
# Adding some informations to send
if self.__extraCredentials:
kwargs[self.KW_EXTRA_CREDENTIALS] = encode(self.__extraCredentials)
kwargs["clientVO"] = self.vo
kwargs["clientSetup"] = self.setup
# Getting URL
url = self.__findServiceURL()
if not url["OK"]:
return url
url = url["Value"]
# Getting CA file (or skip verification)
verify = not self.kwargs.get(self.KW_SKIP_CA_CHECK)
if verify:
if not self.__ca_location:
self.__ca_location = Locations.getCAsLocation()
if not self.__ca_location:
gLogger.error("No CAs found!")
return S_ERROR("No CAs found!")
verify = self.__ca_location
# getting certificate
# Do we use the server certificate ?
if self.kwargs[self.KW_USE_CERTIFICATES]:
cert = Locations.getHostCertificateAndKeyLocation()
elif self.kwargs.get(self.KW_PROXY_STRING):
tmpHandle, cert = tempfile.mkstemp()
fp = os.fdopen(tmpHandle, "wb")
fp.write(self.kwargs[self.KW_PROXY_STRING])
fp.close()
# CHRIS 04.02.21
# TODO: add proxyLocation check ?
else:
cert = Locations.getProxyLocation()
if not cert:
gLogger.error("No proxy found")
return S_ERROR("No proxy found")
# We have a try/except for all the exceptions
# whose default behavior is to try again,
# maybe to different server
try:
# And we have a second block to handle specific exceptions
# which makes it not worth retrying
try:
rawText = None
# Default case, just return the result
if not outputFile:
call = requests.post(url, data=kwargs, timeout=self.timeout, verify=verify, cert=cert)
# raising the exception for status here
# means essentialy that we are losing here the information of what is returned by the server
# as error message, since it is not passed to the exception
# However, we can store the text and return it raw as an error,
# since there is no guarantee that it is any JEncoded text
# Note that we would get an exception only if there is an exception on the server side which
# is not handled.
# Any standard S_ERROR will be transfered as an S_ERROR with a correct code.
rawText = call.text
call.raise_for_status()
return decode(rawText)[0]
else:
# Instruct the server not to encode the response
kwargs["rawContent"] = True
rawText = None
# Stream download
# https://requests.readthedocs.io/en/latest/user/advanced/#body-content-workflow
with requests.post(
url, data=kwargs, timeout=self.timeout, verify=verify, cert=cert, stream=True
) as r:
rawText = r.text
r.raise_for_status()
with open(outputFile, "wb") as f:
for chunk in r.iter_content(4096):
# if chunk: # filter out keep-alive new chuncks
f.write(chunk)
return S_OK()
# Some HTTPError are not worth retrying
except requests.exceptions.HTTPError as e:
status_code = e.response.status_code
if status_code == http_client.NOT_IMPLEMENTED:
return S_ERROR(errno.ENOSYS, "%s is not implemented" % kwargs.get("method"))
elif status_code in (http_client.FORBIDDEN, http_client.UNAUTHORIZED):
return S_ERROR(errno.EACCES, "No access to %s" % url)
# if it is something else, retry
raise
# Whatever exception we have here, we deem worth retrying
except Exception as e:
# CHRIS TODO review this part: retry logic is fishy
# self.__bannedUrls is emptied in findServiceURLs
if url not in self.__bannedUrls:
self.__bannedUrls += [url]
if retry < self.__nbOfUrls - 1:
self._request(retry=retry + 1, outputFile=outputFile, **kwargs)
errStr = "%s: %s" % (str(e), rawText)
return S_ERROR(errStr)
def checkSanity(urlTuple, kwargs):
"""
Check that all ssl environment is ok
"""
useCerts = False
if "useCertificates" in kwargs and kwargs['useCertificates']:
certTuple = Locations.getHostCertificateAndKeyLocation()
if not certTuple:
gLogger.error("No cert/key found! ")
return S_ERROR("No cert/key found! ")
certFile = certTuple[0]
useCerts = True
elif "proxyString" in kwargs:
if type(kwargs['proxyString']) != types.StringType:
gLogger.error("proxyString parameter is not a valid type")
return S_ERROR("proxyString parameter is not a valid type")
else:
if "proxyLocation" in kwargs:
certFile = kwargs["proxyLocation"]
else:
certFile = Locations.getProxyLocation()
if not certFile:
gLogger.error("No proxy found")
return S_ERROR("No proxy found")
elif not os.path.isfile(certFile):
gLogger.error("%s proxy file does not exist" % certFile)
return S_ERROR("%s proxy file does not exist" % certFile)
#For certs always check CA's. For clients skipServerIdentityCheck
if 'skipCACheck' not in kwargs or not kwargs['skipCACheck']:
if not Locations.getCAsLocation():
gLogger.error("No CAs found!")
return S_ERROR("No CAs found!")
if "proxyString" in kwargs:
certObj = X509Chain()
retVal = certObj.loadChainFromString(kwargs['proxyString'])
if not retVal['OK']:
gLogger.error("Can't load proxy string")
return S_ERROR("Can't load proxy string")
else:
if useCerts:
certObj = X509Certificate()
certObj.loadFromFile(certFile)
else:
certObj = X509Chain()
certObj.loadChainFromFile(certFile)
retVal = certObj.hasExpired()
if not retVal['OK']:
gLogger.error("Can't verify file %s:%s" %
(certFile, retVal['Message']))
return S_ERROR("Can't verify file %s:%s" %
(certFile, retVal['Message']))
else:
if retVal['Value']:
notAfter = certObj.getNotAfterDate()
if notAfter['OK']:
notAfter = notAfter['Value']
else:
notAfter = "unknown"
gLogger.error("PEM file has expired",
"%s is not valid after %s" % (certFile, notAfter))
return S_ERROR("PEM file %s has expired, not valid after %s" %
(certFile, notAfter))
idDict = {}
retVal = certObj.getDIRACGroup(ignoreDefault=True)
if retVal['OK'] and retVal['Value'] != False:
idDict['group'] = retVal['Value']
if useCerts:
idDict['DN'] = certObj.getSubjectDN()['Value']
else:
idDict['DN'] = certObj.getIssuerCert()['Value'].getSubjectDN()['Value']
return S_OK(idDict)
def checkSanity(urlTuple, kwargs):
"""
Check that all ssl environment is ok
"""
useCerts = False
certFile = ""
if "useCertificates" in kwargs and kwargs["useCertificates"]:
certTuple = Locations.getHostCertificateAndKeyLocation()
if not certTuple:
gLogger.error("No cert/key found! ")
return S_ERROR("No cert/key found! ")
certFile = certTuple[0]
useCerts = True
elif "proxyString" in kwargs:
if not isinstance(kwargs["proxyString"], six.string_types if six.PY2 else bytes):
gLogger.error("proxyString parameter is not a valid type", str(type(kwargs["proxyString"])))
return S_ERROR("proxyString parameter is not a valid type")
else:
if "proxyLocation" in kwargs:
certFile = kwargs["proxyLocation"]
else:
certFile = Locations.getProxyLocation()
if not certFile:
gLogger.error("No proxy found")
return S_ERROR("No proxy found")
elif not os.path.isfile(certFile):
gLogger.error("Proxy file does not exist", certFile)
return S_ERROR("%s proxy file does not exist" % certFile)
# For certs always check CA's. For clients skipServerIdentityCheck
if "skipCACheck" not in kwargs or not kwargs["skipCACheck"]:
if not Locations.getCAsLocation():
gLogger.error("No CAs found!")
return S_ERROR("No CAs found!")
if "proxyString" in kwargs:
certObj = X509Chain()
retVal = certObj.loadChainFromString(kwargs["proxyString"])
if not retVal["OK"]:
gLogger.error("Can't load proxy string")
return S_ERROR("Can't load proxy string")
else:
if useCerts:
certObj = X509Certificate()
certObj.loadFromFile(certFile)
else:
certObj = X509Chain()
certObj.loadChainFromFile(certFile)
retVal = certObj.hasExpired()
if not retVal["OK"]:
gLogger.error("Can't verify proxy or certificate file", "%s:%s" % (certFile, retVal["Message"]))
return S_ERROR("Can't verify file %s:%s" % (certFile, retVal["Message"]))
else:
if retVal["Value"]:
notAfter = certObj.getNotAfterDate()
if notAfter["OK"]:
notAfter = notAfter["Value"]
else:
notAfter = "unknown"
gLogger.error("PEM file has expired", "%s is not valid after %s" % (certFile, notAfter))
return S_ERROR("PEM file %s has expired, not valid after %s" % (certFile, notAfter))
idDict = {}
retVal = certObj.getDIRACGroup(ignoreDefault=True)
if retVal["OK"] and retVal["Value"] is not False:
idDict["group"] = retVal["Value"]
if useCerts:
idDict["DN"] = certObj.getSubjectDN()["Value"]
else:
idDict["DN"] = certObj.getIssuerCert()["Value"].getSubjectDN()["Value"]
return S_OK(idDict)
def __getCAStore( self ):
SocketInfo.__cachedCAsCRLsLoadLock.acquire()
try:
if not SocketInfo.__cachedCAsCRLs or time.time() - SocketInfo.__cachedCAsCRLsLastLoaded > 900:
#Need to generate the CA Store
casDict = {}
crlsDict = {}
casPath = Locations.getCAsLocation()
if not casPath:
return S_ERROR( "No valid CAs location found" )
gLogger.debug( "CAs location is %s" % casPath )
casFound = 0
crlsFound = 0
SocketInfo.__caStore = GSI.crypto.X509Store()
for fileName in os.listdir( casPath ):
filePath = os.path.join( casPath, fileName )
if not os.path.isfile( filePath ):
continue
fObj = file( filePath, "rb" )
pemData = fObj.read()
fObj.close()
#Try to load CA Cert
try:
caCert = GSI.crypto.load_certificate( GSI.crypto.FILETYPE_PEM, pemData )
if caCert.has_expired():
continue
caID = ( caCert.get_subject().one_line(), caCert.get_issuer().one_line() )
caNotAfter = caCert.get_not_after()
if caID not in casDict:
casDict[ caID ] = ( caNotAfter, caCert )
casFound += 1
else:
if casDict[ caID ][0] < caNotAfter:
casDict[ caID ] = ( caNotAfter, caCert )
continue
except:
if fileName.find( ".0" ) == len( fileName ) - 2:
gLogger.exception( "LOADING %s" % filePath )
if 'IgnoreCRLs' not in self.infoDict or not self.infoDict[ 'IgnoreCRLs' ]:
#Try to load CRL
try:
crl = GSI.crypto.load_crl( GSI.crypto.FILETYPE_PEM, pemData )
if crl.has_expired():
continue
crlID = crl.get_issuer().one_line()
crlNotAfter = crl.get_not_after()
if crlID not in crlsDict:
crlsDict[ crlID ] = ( crlNotAfter, crl )
crlsFound += 1
else:
if crlsDict[ crlID ][0] < crlNotAfter:
crlsDict[ crlID ] = ( crlNotAfter, crl )
continue
except:
if fileName.find( ".r0" ) == len( fileName ) - 2:
gLogger.exception( "LOADING %s" % filePath )
gLogger.debug( "Loaded %s CAs [%s CRLs]" % ( casFound, crlsFound ) )
SocketInfo.__cachedCAsCRLs = ( [ casDict[k][1] for k in casDict ],
[ crlsDict[k][1] for k in crlsDict ] )
SocketInfo.__cachedCAsCRLsLastLoaded = time.time()
except:
gLogger.exception( "ASD" )
finally:
SocketInfo.__cachedCAsCRLsLoadLock.release()
#Generate CA Store
caStore = GSI.crypto.X509Store()
caList = SocketInfo.__cachedCAsCRLs[0]
for caCert in caList:
caStore.add_cert( caCert )
crlList = SocketInfo.__cachedCAsCRLs[1]
for crl in crlList:
caStore.add_crl( crl )
return S_OK( caStore )
def checkSanity( urlTuple, kwargs ):
"""
Check that all ssl environment is ok
"""
useCerts = False
certFile = ''
if "useCertificates" in kwargs and kwargs[ 'useCertificates' ]:
certTuple = Locations.getHostCertificateAndKeyLocation()
if not certTuple:
gLogger.error( "No cert/key found! " )
return S_ERROR( "No cert/key found! " )
certFile = certTuple[0]
useCerts = True
elif "proxyString" in kwargs:
if not isinstance( kwargs[ 'proxyString' ], basestring ):
gLogger.error( "proxyString parameter is not a valid type", str( type( kwargs[ 'proxyString' ] ) ) )
return S_ERROR( "proxyString parameter is not a valid type" )
else:
if "proxyLocation" in kwargs:
certFile = kwargs[ "proxyLocation" ]
else:
certFile = Locations.getProxyLocation()
if not certFile:
gLogger.error( "No proxy found" )
return S_ERROR( "No proxy found" )
elif not os.path.isfile( certFile ):
gLogger.error( "Proxy file does not exist", certFile )
return S_ERROR( "%s proxy file does not exist" % certFile )
#For certs always check CA's. For clients skipServerIdentityCheck
if 'skipCACheck' not in kwargs or not kwargs[ 'skipCACheck' ]:
if not Locations.getCAsLocation():
gLogger.error( "No CAs found!" )
return S_ERROR( "No CAs found!" )
if "proxyString" in kwargs:
certObj = X509Chain()
retVal = certObj.loadChainFromString( kwargs[ 'proxyString' ] )
if not retVal[ 'OK' ]:
gLogger.error( "Can't load proxy string" )
return S_ERROR( "Can't load proxy string" )
else:
if useCerts:
certObj = X509Certificate()
certObj.loadFromFile( certFile )
else:
certObj = X509Chain()
certObj.loadChainFromFile( certFile )
retVal = certObj.hasExpired()
if not retVal[ 'OK' ]:
gLogger.error( "Can't verify proxy or certificate file", "%s:%s" % ( certFile, retVal[ 'Message' ] ) )
return S_ERROR( "Can't verify file %s:%s" % ( certFile, retVal[ 'Message' ] ) )
else:
if retVal[ 'Value' ]:
notAfter = certObj.getNotAfterDate()
if notAfter[ 'OK' ]:
notAfter = notAfter[ 'Value' ]
else:
notAfter = "unknown"
gLogger.error( "PEM file has expired", "%s is not valid after %s" % ( certFile, notAfter ) )
return S_ERROR( "PEM file %s has expired, not valid after %s" % ( certFile, notAfter ) )
idDict = {}
retVal = certObj.getDIRACGroup( ignoreDefault = True )
if retVal[ 'OK' ] and retVal[ 'Value' ] != False:
idDict[ 'group' ] = retVal[ 'Value' ]
if useCerts:
idDict[ 'DN' ] = certObj.getSubjectDN()[ 'Value' ]
else:
idDict[ 'DN' ] = certObj.getIssuerCert()[ 'Value' ].getSubjectDN()[ 'Value' ]
return S_OK( idDict )
from DIRAC.ConfigurationSystem.Client.Utilities import getAuthorizationServerMetadata
# Description of default DIRAC OAuth2 clients
DEFAULT_CLIENTS = {
# Description of default public DIRAC client which is installed in the terminal
"DIRACCLI":
dict(
client_id="DIRAC_CLI",
scope="proxy g: lifetime:",
response_types=["device"],
grant_types=[
"urn:ietf:params:oauth:grant-type:device_code", "refresh_token"
],
token_endpoint_auth_method="none",
ProviderType="OAuth2",
verify=Locations.getCAsLocation(),
),
# These settings are for the web server
"DIRACWeb":
dict(
client_id="DIRAC_Web",
scope="g:",
response_types=["code"],
grant_types=["authorization_code", "refresh_token"],
ProviderType="OAuth2",
verify=Locations.getCAsLocation(),
),
}
def getDIRACClients():
