class UPHandler(WebHandler):
__tc = ThreadConfig()
def prepare(self):
if not self.isRegisteredUser():
raise WErr(401, "Not a registered user")
self.set_header("Pragma", "no-cache")
self.set_header("Cache-Control",
"max-age=0, no-store, no-cache, must-revalidate")
@asyncGen
def web_saveAppState(self):
self.__tc.setSetup(False)
try:
app = self.request.arguments['app'][-1]
name = self.request.arguments['name'][-1]
state = self.request.arguments['state'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
data = base64.b64encode(zlib.compress(DEncode.encode(state), 9))
up = UserProfileClient("Web/App/%s" % app)
result = yield self.threadTask(up.storeVar, name, data)
if not result['OK']:
raise WErr.fromSERROR(result)
self.set_status(200)
self.finish()
@asyncGen
def web_loadAppState(self):
self.__tc.setSetup(False)
try:
app = self.request.arguments['app'][-1]
name = self.request.arguments['name'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
up = UserProfileClient("Web/App/%s" % app)
result = yield self.threadTask(up.retrieveVar, name)
if not result['OK']:
raise WErr.fromSERROR(result)
data = result['Value']
data, count = DEncode.decode(zlib.decompress(base64.b64decode(data)))
self.set_header("Content-Type", "application/json")
self.finish(data)
@asyncGen
def web_listAppState(self):
self.__tc.setSetup(False)
try:
app = self.request.arguments['app'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
up = UserProfileClient("Web/App/%s" % app)
result = yield self.threadTask(up.listAvailableVars,
{'UserName': [self.getUserName()]})
if not result['OK']:
raise WErr.fromSERROR(result)
data = result['Value']
self.finish({'app': [e[-1] for e in data]})
class WebHandler(tornado.web.RequestHandler):
__threadPool = getGlobalThreadPool()
__disetConfig = ThreadConfig()
__log = False
#Auth requirements
AUTH_PROPS = None
#Location of the handler in the URL
LOCATION = ""
#URL Schema with holders to generate handler urls
URLSCHEMA = ""
#RE to extract group and setup
PATH_RE = ""
#Helper function to create threaded gen.Tasks with automatic callback and execption handling
@classmethod
def threadTask(cls, method, *args, **kwargs):
"""
Helper method to generate a gen.Task and automatically call the callback when the real
method ends. THIS IS SPARTAAAAAAAAAA
"""
#Save the task to access the runner
genTask = False
#This runs in the separate thread, calls the callback on finish and takes into account exceptions
def cbMethod(*cargs, **ckwargs):
cb = ckwargs.pop('callback')
method = cargs[0]
disetConf = cargs[1]
cargs = cargs[2]
cls.__disetConfig.load(disetConf)
ioloop = tornado.ioloop.IOLoop.instance()
try:
result = method(*cargs, **ckwargs)
ioloop.add_callback(functools.partial(cb, result))
except Exception, excp:
exc_info = sys.exc_info()
ioloop.add_callback(
lambda: genTask.runner.handle_exception(*exc_info))
#Put the task in the thread :)
def threadJob(tmethod, *targs, **tkwargs):
tkwargs['callback'] = tornado.stack_context.wrap(
tkwargs['callback'])
targs = (tmethod, cls.__disetConfig.dump(), targs)
cls.__threadPool.generateJobAndQueueIt(cbMethod,
args=targs,
kwargs=tkwargs)
#Return a YieldPoint
genTask = tornado.gen.Task(threadJob, method, *args, **kwargs)
return genTask
class BaseClient:
VAL_EXTRA_CREDENTIALS_HOST = "hosts"
KW_USE_CERTIFICATES = "useCertificates"
KW_EXTRA_CREDENTIALS = "extraCredentials"
KW_TIMEOUT = "timeout"
KW_SETUP = "setup"
KW_VO = "VO"
KW_DELEGATED_DN = "delegatedDN"
KW_DELEGATED_GROUP = "delegatedGroup"
KW_IGNORE_GATEWAYS = "ignoreGateways"
KW_PROXY_LOCATION = "proxyLocation"
KW_PROXY_STRING = "proxyString"
KW_PROXY_CHAIN = "proxyChain"
KW_SKIP_CA_CHECK = "skipCACheck"
KW_KEEP_ALIVE_LAPSE = "keepAliveLapse"
__threadConfig = ThreadConfig()
def __init__( self, serviceName, **kwargs ):
if type( serviceName ) not in types.StringTypes:
raise TypeError( "Service name expected to be a string. Received %s type %s" %
( str( serviceName ), type( serviceName ) ) )
self._destinationSrv = serviceName
self._serviceName = serviceName
self.kwargs = kwargs
self.__initStatus = S_OK()
self.__idDict = {}
self.__extraCredentials = ""
self.__enableThreadCheck = False
self.__retry = 0
self.__retryDelay = 0
self.__nbOfUrls = 1 #by default we always have 1 url for example: RPCClient('dips://volhcb38.cern.ch:9162/Framework/SystemAdministrator')
self.__nbOfRetry = 3 # by default we try try times
self.__bannedUrls = []
for initFunc in ( self.__discoverSetup, self.__discoverVO, self.__discoverTimeout,
self.__discoverURL, self.__discoverCredentialsToUse,
self.__checkTransportSanity,
self.__setKeepAliveLapse ):
result = initFunc()
if not result[ 'OK' ] and self.__initStatus[ 'OK' ]:
self.__initStatus = result
self._initialize()
#HACK for thread-safety:
self.__allowedThreadID = False
def _initialize( self ):
pass
def getDestinationService( self ):
return self._destinationSrv
def getServiceName( self ):
return self._serviceName
def __discoverSetup( self ):
#Which setup to use?
if self.KW_SETUP in self.kwargs and self.kwargs[ self.KW_SETUP ]:
self.setup = str( self.kwargs[ self.KW_SETUP ] )
else:
self.setup = self.__threadConfig.getSetup()
if not self.setup:
self.setup = gConfig.getValue( "/DIRAC/Setup", "Test" )
return S_OK()
def __discoverVO( self ):
#Which setup to use?
if self.KW_VO in self.kwargs and self.kwargs[ self.KW_VO ]:
self.vo = str( self.kwargs[ self.KW_VO ] )
else:
self.vo = gConfig.getValue( "/DIRAC/VirtualOrganization", "unknown" )
return S_OK()
def __discoverURL( self ):
#Calculate final URL
try:
result = self.__findServiceURL()
except Exception, e:
return S_ERROR( str( e ) )
if not result[ 'OK' ]:
return result
self.serviceURL = result[ 'Value' ]
retVal = Network.splitURL( self.serviceURL )
if not retVal[ 'OK' ]:
return S_ERROR( "URL is malformed: %s" % retVal[ 'Message' ] )
self.__URLTuple = retVal[ 'Value' ]
self._serviceName = self.__URLTuple[-1]
res = gConfig.getOptionsDict( "/DIRAC/ConnConf/%s:%s" % self.__URLTuple[1:3] )
if res[ 'OK' ]:
opts = res[ 'Value' ]
for k in opts:
if k not in self.kwargs:
self.kwargs[k] = opts[k]
return S_OK()
class TornadoBaseClient(object):
"""
This class contain initialization method and all utilities method used for RPC
"""
__threadConfig = ThreadConfig()
VAL_EXTRA_CREDENTIALS_HOST = "hosts"
KW_USE_CERTIFICATES = "useCertificates"
KW_EXTRA_CREDENTIALS = "extraCredentials"
KW_TIMEOUT = "timeout"
KW_SETUP = "setup"
KW_VO = "VO"
KW_DELEGATED_DN = "delegatedDN"
KW_DELEGATED_GROUP = "delegatedGroup"
KW_IGNORE_GATEWAYS = "ignoreGateways"
KW_PROXY_LOCATION = "proxyLocation"
KW_PROXY_STRING = "proxyString"
KW_PROXY_CHAIN = "proxyChain"
KW_SKIP_CA_CHECK = "skipCACheck"
KW_KEEP_ALIVE_LAPSE = "keepAliveLapse"
def __init__(self, serviceName, **kwargs):
"""
:param serviceName: URL of the service (proper uri or just System/Component)
:param useCertificates: If set to True, use the server certificate
:param extraCredentials:
:param timeout: Timeout of the call (default 600 s)
:param setup: Specify the Setup
:param VO: Specify the VO
:param delegatedDN: Not clear what it can be used for.
:param delegatedGroup: Not clear what it can be used for.
:param ignoreGateways: Ignore the DIRAC Gatways settings
:param proxyLocation: Specify the location of the proxy
:param proxyString: Specify the proxy string
:param proxyChain: Specify the proxy chain
:param skipCACheck: Do not check the CA
:param keepAliveLapse: Duration for keepAliveLapse (heartbeat like) (now managed by requests)
"""
if not isinstance(serviceName, six.string_types):
raise TypeError(
"Service name expected to be a string. Received %s type %s" % (str(serviceName), type(serviceName))
)
self._destinationSrv = serviceName
self._serviceName = serviceName
self.__ca_location = False
self.kwargs = kwargs
self.__useCertificates = None
# The CS useServerCertificate option can be overridden by explicit argument
self.__forceUseCertificates = self.kwargs.get(self.KW_USE_CERTIFICATES)
self.__initStatus = S_OK()
self.__idDict = {}
self.__extraCredentials = ""
# by default we always have 1 url for example:
# RPCClient('dips://volhcb38.cern.ch:9162/Framework/SystemAdministrator')
self.__nbOfUrls = 1
self.__bannedUrls = []
# For pylint...
self.setup = None
self.vo = None
self.serviceURL = None
for initFunc in (
self.__discoverTimeout,
self.__discoverSetup,
self.__discoverVO,
self.__discoverCredentialsToUse,
self.__discoverExtraCredentials,
self.__discoverURL,
):
result = initFunc()
if not result["OK"] and self.__initStatus["OK"]:
self.__initStatus = result
def __discoverSetup(self):
"""Discover which setup to use and stores it in self.setup
The setup is looked for:
* kwargs of the constructor (see KW_SETUP)
* in the CS /DIRAC/Setup
* default to 'Test'
"""
if self.KW_SETUP in self.kwargs and self.kwargs[self.KW_SETUP]:
self.setup = str(self.kwargs[self.KW_SETUP])
else:
self.setup = self.__threadConfig.getSetup()
if not self.setup:
self.setup = gConfig.getValue("/DIRAC/Setup", "Test")
return S_OK()
def __discoverURL(self):
"""Calculate the final URL. It is called at initialization and in connect in case of issue
It sets:
* self.serviceURL: the url (dips) selected as target using __findServiceURL
* self.__URLTuple: a split of serviceURL obtained by Network.splitURL
* self._serviceName: the last part of URLTuple (typically System/Component)
WARNING: COPY PASTE FROM BaseClient
"""
# Calculate final URL
try:
result = self.__findServiceURL()
except Exception as e:
return S_ERROR(repr(e))
if not result["OK"]:
return result
self.serviceURL = result["Value"]
retVal = Network.splitURL(self.serviceURL)
if not retVal["OK"]:
return retVal
self.__URLTuple = retVal["Value"]
self._serviceName = self.__URLTuple[-1]
res = gConfig.getOptionsDict("/DIRAC/ConnConf/%s:%s" % self.__URLTuple[1:3])
if res["OK"]:
opts = res["Value"]
for k in opts:
if k not in self.kwargs:
self.kwargs[k] = opts[k]
return S_OK()
def __discoverVO(self):
"""Discover which VO to use and stores it in self.vo
The VO is looked for:
* kwargs of the constructor (see KW_VO)
* in the CS /DIRAC/VirtualOrganization
* default to 'unknown'
WARNING: COPY/PASTE FROM Core/Diset/private/BaseClient FOR NOW
"""
if self.KW_VO in self.kwargs and self.kwargs[self.KW_VO]:
self.vo = str(self.kwargs[self.KW_VO])
else:
self.vo = gConfig.getValue("/DIRAC/VirtualOrganization", "unknown")
return S_OK()
def __discoverCredentialsToUse(self):
"""Discovers which credentials to use for connection.
* Server certificate:
-> If KW_USE_CERTIFICATES in kwargs, sets it in self.__useCertificates
-> If not, check gConfig.useServerCertificate(), and sets it in self.__useCertificates
and kwargs[KW_USE_CERTIFICATES]
* Certification Authorities check:
-> if KW_SKIP_CA_CHECK is not in kwargs and we are using the certificates,
set KW_SKIP_CA_CHECK to false in kwargs
-> if KW_SKIP_CA_CHECK is not in kwargs and we are not using the certificate, check the skipCACheck
* Proxy Chain
WARNING: MOSTLY COPY/PASTE FROM Core/Diset/private/BaseClient
"""
# Use certificates?
if self.KW_USE_CERTIFICATES in self.kwargs:
self.__useCertificates = self.kwargs[self.KW_USE_CERTIFICATES]
else:
self.__useCertificates = gConfig.useServerCertificate()
self.kwargs[self.KW_USE_CERTIFICATES] = self.__useCertificates
if self.KW_SKIP_CA_CHECK not in self.kwargs:
if self.__useCertificates:
self.kwargs[self.KW_SKIP_CA_CHECK] = False
else:
self.kwargs[self.KW_SKIP_CA_CHECK] = skipCACheck()
# Rewrite a little bit from here: don't need the proxy string, we use the file
if self.KW_PROXY_CHAIN in self.kwargs:
try:
self.kwargs[self.KW_PROXY_STRING] = self.kwargs[self.KW_PROXY_CHAIN].dumpAllToString()["Value"]
del self.kwargs[self.KW_PROXY_CHAIN]
except Exception:
return S_ERROR("Invalid proxy chain specified on instantiation")
# ==== REWRITED FROM HERE ====
# For certs always check CA's. For clients skipServerIdentityCheck
return S_OK()
def __discoverExtraCredentials(self):
"""Add extra credentials informations.
* self.__extraCredentials
-> if KW_EXTRA_CREDENTIALS in kwargs, we set it
-> Otherwise, if we use the server certificate, we set it to VAL_EXTRA_CREDENTIALS_HOST
-> If we have a delegation (see bellow), we set it to (delegatedDN, delegatedGroup)
-> otherwise it is an empty string
* delegation:
-> if KW_DELEGATED_DN in kwargs, or delegatedDN in threadConfig, put in in self.kwargs
-> If we have a delegated DN but not group, we find the corresponding group in the CS
WARNING: COPY/PASTE FROM Core/Diset/private/BaseClient
"""
# which extra credentials to use?
if self.__useCertificates:
self.__extraCredentials = self.VAL_EXTRA_CREDENTIALS_HOST
else:
self.__extraCredentials = ""
if self.KW_EXTRA_CREDENTIALS in self.kwargs:
self.__extraCredentials = self.kwargs[self.KW_EXTRA_CREDENTIALS]
# Are we delegating something?
delegatedDN, delegatedGroup = self.__threadConfig.getID()
if self.KW_DELEGATED_DN in self.kwargs and self.kwargs[self.KW_DELEGATED_DN]:
delegatedDN = self.kwargs[self.KW_DELEGATED_DN]
elif delegatedDN:
self.kwargs[self.KW_DELEGATED_DN] = delegatedDN
if self.KW_DELEGATED_GROUP in self.kwargs and self.kwargs[self.KW_DELEGATED_GROUP]:
delegatedGroup = self.kwargs[self.KW_DELEGATED_GROUP]
elif delegatedGroup:
self.kwargs[self.KW_DELEGATED_GROUP] = delegatedGroup
if delegatedDN:
if not delegatedGroup:
result = findDefaultGroupForDN(self.kwargs[self.KW_DELEGATED_DN])
if not result["OK"]:
return result
self.__extraCredentials = (delegatedDN, delegatedGroup)
return S_OK()
def __discoverTimeout(self):
"""Discover which timeout to use and stores it in self.timeout
The timeout can be specified kwargs of the constructor (see KW_TIMEOUT),
with a minimum of 120 seconds.
If unspecified, the timeout will be 600 seconds.
The value is set in self.timeout, as well as in self.kwargs[KW_TIMEOUT]
WARNING: COPY/PASTE FROM Core/Diset/private/BaseClient
"""
if self.KW_TIMEOUT in self.kwargs:
self.timeout = self.kwargs[self.KW_TIMEOUT]
else:
self.timeout = False
if self.timeout:
self.timeout = max(120, self.timeout)
else:
self.timeout = 600
self.kwargs[self.KW_TIMEOUT] = self.timeout
return S_OK()
def __findServiceURL(self):
"""
Discovers the URL of a service, taking into account gateways, multiple URLs, banned URLs
If the site on which we run is configured to use gateways (/DIRAC/Gateways/<siteName>),
these URLs will be used. To ignore the gateway, it is possible to set KW_IGNORE_GATEWAYS
to False in kwargs.
If self._destinationSrv (given as constructor attribute) is a properly formed URL,
we just return this one. If we have to use a gateway, we just replace the server name in the url.
The list of URLs defined in the CS (<System>/URLs/<Component>) is randomized
This method also sets some attributes:
* self.__nbOfUrls = number of URLs
* self.__nbOfRetry removed in HTTPS (Managed by requests)
* self.__bannedUrls is reinitialized if all the URLs are banned
:return: the selected URL
WARNING (Mostly) COPY PASTE FROM BaseClient (protocols list is changed to https)
"""
if not self.__initStatus["OK"]:
return self.__initStatus
# Load the Gateways URLs for the current site Name
gatewayURL = False
if not self.kwargs.get(self.KW_IGNORE_GATEWAYS):
gatewayURLs = getGatewayURLs()
if gatewayURLs:
gatewayURL = "/".join(gatewayURLs[0].split("/")[:3])
# If what was given as constructor attribute is a properly formed URL,
# we just return this one.
# If we have to use a gateway, we just replace the server name in it
if self._destinationSrv.startswith("https://"):
gLogger.debug("Already given a valid url", self._destinationSrv)
if not gatewayURL:
return S_OK(self._destinationSrv)
gLogger.debug("Reconstructing given URL to pass through gateway")
path = "/".join(self._destinationSrv.split("/")[3:])
finalURL = "%s/%s" % (gatewayURL, path)
gLogger.debug("Gateway URL conversion:\n %s -> %s" % (self._destinationSrv, finalURL))
return S_OK(finalURL)
if gatewayURL:
gLogger.debug("Using gateway", gatewayURL)
return S_OK("%s/%s" % (gatewayURL, self._destinationSrv))
# If nor url is given as constructor, we extract the list of URLs from the CS (System/URLs/Component)
try:
# We randomize the list, and add at the end the failover URLs (System/FailoverURLs/Component)
urlsList = getServiceURLs(self._destinationSrv, setup=self.setup, failover=True)
except Exception as e:
return S_ERROR("Cannot get URL for %s in setup %s: %s" % (self._destinationSrv, self.setup, repr(e)))
if not urlsList:
return S_ERROR("URL for service %s not found" % self._destinationSrv)
self.__nbOfUrls = len(urlsList)
# __nbOfRetry removed in HTTPS (managed by requests)
if self.__nbOfUrls == len(self.__bannedUrls):
self.__bannedUrls = [] # retry all urls
gLogger.debug("Retrying again all URLs")
if self.__bannedUrls and len(urlsList) > 1:
# we have host which is not accessible. We remove that host from the list.
# We only remove if we have more than one instance
for i in self.__bannedUrls:
gLogger.debug("Removing banned URL", "%s" % i)
urlsList.remove(i)
sURL = urlsList[0]
# If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL
# is not on a host which is banned. If it is, we take the next one in the list using __selectUrl
if self.__bannedUrls and self.__nbOfUrls > 2: # when we have multiple services then we can
# have a situation when two services are running on the same machine with different ports...
retVal = Network.splitURL(sURL)
nexturl = None
if retVal["OK"]:
nexturl = retVal["Value"]
found = False
for i in self.__bannedUrls:
retVal = Network.splitURL(i)
if retVal["OK"]:
bannedurl = retVal["Value"]
else:
break
# We found a banned URL on the same host as the one we are running on
if nexturl[1] == bannedurl[1]:
found = True
break
if found:
nexturl = self.__selectUrl(nexturl, urlsList[1:])
if nexturl: # an url found which is in different host
sURL = nexturl
gLogger.debug("Discovering URL for service", "%s -> %s" % (self._destinationSrv, sURL))
return S_OK(sURL)
def __selectUrl(self, notselect, urls):
"""In case when multiple services are running in the same host, a new url has to be in a different host
Note: If we do not have different host we will use the selected url...
:param notselect: URL that should NOT be selected
:param urls: list of potential URLs
:return: selected URL
WARNING: COPY/PASTE FROM Core/Diset/private/BaseClient
"""
url = None
for i in urls:
retVal = Network.splitURL(i)
if retVal["OK"]:
if retVal["Value"][1] != notselect[1]: # the hots are different
url = i
break
else:
gLogger.error(retVal["Message"])
return url
def getServiceName(self):
"""
Returns the name of the service, if you had given a url at init, returns the URL.
"""
return self._serviceName
def getDestinationService(self):
"""
Returns the url the service.
"""
urls = getServiceURLs(self._serviceName)
return urls[0] if urls else ""
def _getBaseStub(self):
"""Returns a tuple with (self._destinationSrv, newKwargs)
self._destinationSrv is what was given as first parameter of the init serviceName
newKwargs is an updated copy of kwargs:
* if set, we remove the useCertificates (KW_USE_CERTIFICATES) in newKwargs
This method is just used to return information in case of error in the InnerRPCClient
WARNING: COPY/PASTE FROM Core/Diset/private/BaseClient
"""
newKwargs = dict(self.kwargs)
# Remove useCertificates as the forwarder of the call will have to
# independently decide whether to use their cert or not anyway.
if "useCertificates" in newKwargs:
del newKwargs["useCertificates"]
return (self._destinationSrv, newKwargs)
def _request(self, retry=0, outputFile=None, **kwargs):
"""
Sends the request to server
:param retry: internal parameters for recursive call. TODO: remove ?
:param outputFile: (default None) path to a file where to store the received data.
If set, the server response will be streamed for optimization
purposes, and the response data will not go through the
JDecode process
:param **kwargs: Any argument there is used as a post parameter. They are detailed bellow.
:param method: (mandatory) name of the distant method
:param args: (mandatory) json serialized list of argument for the procedure
:returns: The received data. If outputFile is set, return always S_OK
"""
# Adding some informations to send
if self.__extraCredentials:
kwargs[self.KW_EXTRA_CREDENTIALS] = encode(self.__extraCredentials)
kwargs["clientVO"] = self.vo
kwargs["clientSetup"] = self.setup
# Getting URL
url = self.__findServiceURL()
if not url["OK"]:
return url
url = url["Value"]
# Getting CA file (or skip verification)
verify = not self.kwargs.get(self.KW_SKIP_CA_CHECK)
if verify:
if not self.__ca_location:
self.__ca_location = Locations.getCAsLocation()
if not self.__ca_location:
gLogger.error("No CAs found!")
return S_ERROR("No CAs found!")
verify = self.__ca_location
# getting certificate
# Do we use the server certificate ?
if self.kwargs[self.KW_USE_CERTIFICATES]:
cert = Locations.getHostCertificateAndKeyLocation()
elif self.kwargs.get(self.KW_PROXY_STRING):
tmpHandle, cert = tempfile.mkstemp()
fp = os.fdopen(tmpHandle, "wb")
fp.write(self.kwargs[self.KW_PROXY_STRING])
fp.close()
# CHRIS 04.02.21
# TODO: add proxyLocation check ?
else:
cert = Locations.getProxyLocation()
if not cert:
gLogger.error("No proxy found")
return S_ERROR("No proxy found")
# We have a try/except for all the exceptions
# whose default behavior is to try again,
# maybe to different server
try:
# And we have a second block to handle specific exceptions
# which makes it not worth retrying
try:
rawText = None
# Default case, just return the result
if not outputFile:
call = requests.post(url, data=kwargs, timeout=self.timeout, verify=verify, cert=cert)
# raising the exception for status here
# means essentialy that we are losing here the information of what is returned by the server
# as error message, since it is not passed to the exception
# However, we can store the text and return it raw as an error,
# since there is no guarantee that it is any JEncoded text
# Note that we would get an exception only if there is an exception on the server side which
# is not handled.
# Any standard S_ERROR will be transfered as an S_ERROR with a correct code.
rawText = call.text
call.raise_for_status()
return decode(rawText)[0]
else:
# Instruct the server not to encode the response
kwargs["rawContent"] = True
rawText = None
# Stream download
# https://requests.readthedocs.io/en/latest/user/advanced/#body-content-workflow
with requests.post(
url, data=kwargs, timeout=self.timeout, verify=verify, cert=cert, stream=True
) as r:
rawText = r.text
r.raise_for_status()
with open(outputFile, "wb") as f:
for chunk in r.iter_content(4096):
# if chunk: # filter out keep-alive new chuncks
f.write(chunk)
return S_OK()
# Some HTTPError are not worth retrying
except requests.exceptions.HTTPError as e:
status_code = e.response.status_code
if status_code == http_client.NOT_IMPLEMENTED:
return S_ERROR(errno.ENOSYS, "%s is not implemented" % kwargs.get("method"))
elif status_code in (http_client.FORBIDDEN, http_client.UNAUTHORIZED):
return S_ERROR(errno.EACCES, "No access to %s" % url)
# if it is something else, retry
raise
# Whatever exception we have here, we deem worth retrying
except Exception as e:
# CHRIS TODO review this part: retry logic is fishy
# self.__bannedUrls is emptied in findServiceURLs
if url not in self.__bannedUrls:
self.__bannedUrls += [url]
if retry < self.__nbOfUrls - 1:
self._request(retry=retry + 1, outputFile=outputFile, **kwargs)
errStr = "%s: %s" % (str(e), rawText)
return S_ERROR(errStr)
class BaseClient(object):
""" Glues together stubs with threading, credentials, and URLs discovery (by DIRAC vo and setup).
Basically what needs to be done to enable RPC calls, and transfer, to find a URL.
"""
VAL_EXTRA_CREDENTIALS_HOST = "hosts"
KW_USE_CERTIFICATES = "useCertificates"
KW_EXTRA_CREDENTIALS = "extraCredentials"
KW_TIMEOUT = "timeout"
KW_SETUP = "setup"
KW_VO = "VO"
KW_DELEGATED_DN = "delegatedDN"
KW_DELEGATED_GROUP = "delegatedGroup"
KW_IGNORE_GATEWAYS = "ignoreGateways"
KW_PROXY_LOCATION = "proxyLocation"
KW_PROXY_STRING = "proxyString"
KW_PROXY_CHAIN = "proxyChain"
KW_SKIP_CA_CHECK = "skipCACheck"
KW_KEEP_ALIVE_LAPSE = "keepAliveLapse"
__threadConfig = ThreadConfig()
def __init__(self, serviceName, **kwargs):
""" Constructor
:param serviceName: URL of the service (proper uri or just System/Component)
:param useCertificates: If set to True, use the server certificate
:param extraCredentials:
:param timeout: Timeout of the call (default 600 s)
:param setup: Specify the Setup
:param VO: Specify the VO
:param delegatedDN: Not clear what it can be used for.
:param delegatedGroup: Not clear what it can be used for.
:param ignoreGateways: Ignore the DIRAC Gatways settings
:param proxyLocation: Specify the location of the proxy
:param proxyString: Specify the proxy string
:param proxyChain: Specify the proxy chain
:param skipCACheck: Do not check the CA
:param keepAliveLapse: Duration for keepAliveLapse (heartbeat like)
"""
if not isinstance(serviceName, six.string_types):
raise TypeError("Service name expected to be a string. Received %s type %s" %
(str(serviceName), type(serviceName)))
# Explicitly convert to a str to avoid Python 2 M2Crypto issues with unicode objects
self._destinationSrv = str(serviceName)
self._serviceName = str(serviceName)
self.kwargs = kwargs
self.__useCertificates = None
# The CS useServerCertificate option can be overridden by explicit argument
self.__forceUseCertificates = self.kwargs.get(self.KW_USE_CERTIFICATES)
self.__initStatus = S_OK()
self.__idDict = {}
self.__extraCredentials = ""
self.__enableThreadCheck = False
self.__retry = 0
self.__retryDelay = 0
# by default we always have 1 url for example:
# RPCClient('dips://volhcb38.cern.ch:9162/Framework/SystemAdministrator')
self.__nbOfUrls = 1
self.__nbOfRetry = 3 # by default we try try times
self.__retryCounter = 1
self.__bannedUrls = []
for initFunc in (self.__discoverSetup, self.__discoverVO, self.__discoverTimeout,
self.__discoverURL, self.__discoverCredentialsToUse,
self.__checkTransportSanity,
self.__setKeepAliveLapse):
result = initFunc()
if not result['OK'] and self.__initStatus['OK']:
self.__initStatus = result
self.numberOfURLs = 0
self._initialize()
# HACK for thread-safety:
self.__allowedThreadID = False
def _initialize(self):
pass
def getDestinationService(self):
""" Return service destination
:return: str
"""
return self._destinationSrv
def getServiceName(self):
""" Return service name
:return: str
"""
return self._serviceName
def __discoverSetup(self):
""" Discover which setup to use and stores it in self.setup
The setup is looked for:
* kwargs of the constructor (see KW_SETUP)
* the ThreadConfig
* in the CS /DIRAC/Setup
* default to 'Test'
:return: S_OK()/S_ERROR()
"""
if self.KW_SETUP in self.kwargs and self.kwargs[self.KW_SETUP]:
self.setup = str(self.kwargs[self.KW_SETUP])
else:
self.setup = self.__threadConfig.getSetup()
if not self.setup:
self.setup = gConfig.getValue("/DIRAC/Setup", "Test")
return S_OK()
def __discoverVO(self):
""" Discover which VO to use and stores it in self.vo
The VO is looked for:
* kwargs of the constructor (see KW_VO)
* in the CS /DIRAC/VirtualOrganization
* default to 'unknown'
:return: S_OK()/S_ERROR()
"""
if self.KW_VO in self.kwargs and self.kwargs[self.KW_VO]:
self.vo = str(self.kwargs[self.KW_VO])
else:
self.vo = gConfig.getValue("/DIRAC/VirtualOrganization", "unknown")
return S_OK()
def __discoverURL(self):
""" Calculate the final URL. It is called at initialization and in connect in case of issue
It sets:
* self.serviceURL: the url (dips) selected as target using __findServiceURL
* self.__URLTuple: a split of serviceURL obtained by Network.splitURL
* self._serviceName: the last part of URLTuple (typically System/Component)
:return: S_OK()/S_ERROR()
"""
# Calculate final URL
try:
result = self.__findServiceURL()
except Exception as e:
return S_ERROR(repr(e))
if not result['OK']:
return result
self.serviceURL = result['Value']
retVal = Network.splitURL(self.serviceURL)
if not retVal['OK']:
return retVal
self.__URLTuple = retVal['Value']
self._serviceName = self.__URLTuple[-1]
res = gConfig.getOptionsDict("/DIRAC/ConnConf/%s:%s" % self.__URLTuple[1:3])
if res['OK']:
opts = res['Value']
for k in opts:
if k not in self.kwargs:
self.kwargs[k] = opts[k]
return S_OK()
def __discoverTimeout(self):
""" Discover which timeout to use and stores it in self.timeout
The timeout can be specified kwargs of the constructor (see KW_TIMEOUT),
with a minimum of 120 seconds.
If unspecified, the timeout will be 600 seconds.
The value is set in self.timeout, as well as in self.kwargs[KW_TIMEOUT]
:return: S_OK()/S_ERROR()
"""
if self.KW_TIMEOUT in self.kwargs:
self.timeout = self.kwargs[self.KW_TIMEOUT]
else:
self.timeout = False
if self.timeout:
self.timeout = max(120, self.timeout)
else:
self.timeout = 600
self.kwargs[self.KW_TIMEOUT] = self.timeout
return S_OK()
def __discoverCredentialsToUse(self):
""" Discovers which credentials to use for connection.
* Server certificate:
-> If KW_USE_CERTIFICATES in kwargs, sets it in self.__useCertificates
-> If not, check gConfig.useServerCertificate(),
and sets it in self.__useCertificates and kwargs[KW_USE_CERTIFICATES]
* Certification Authorities check:
-> if KW_SKIP_CA_CHECK is not in kwargs and we are using the certificates,
set KW_SKIP_CA_CHECK to false in kwargs
-> if KW_SKIP_CA_CHECK is not in kwargs and we are not using the certificate, check the CS.skipCACheck
* Proxy Chain
-> if KW_PROXY_CHAIN in kwargs, we remove it and dump its string form into kwargs[KW_PROXY_STRING]
:return: S_OK()/S_ERROR()
"""
# Use certificates?
if self.KW_USE_CERTIFICATES in self.kwargs:
self.__useCertificates = self.kwargs[self.KW_USE_CERTIFICATES]
else:
self.__useCertificates = gConfig.useServerCertificate()
self.kwargs[self.KW_USE_CERTIFICATES] = self.__useCertificates
if self.KW_SKIP_CA_CHECK not in self.kwargs:
if self.__useCertificates:
self.kwargs[self.KW_SKIP_CA_CHECK] = False
else:
self.kwargs[self.KW_SKIP_CA_CHECK] = skipCACheck()
if self.KW_PROXY_CHAIN in self.kwargs:
try:
self.kwargs[self.KW_PROXY_STRING] = self.kwargs[self.KW_PROXY_CHAIN].dumpAllToString()['Value']
del self.kwargs[self.KW_PROXY_CHAIN]
except BaseException:
return S_ERROR("Invalid proxy chain specified on instantiation")
return S_OK()
def __discoverExtraCredentials(self):
""" Add extra credentials informations.
* self.__extraCredentials
-> if KW_EXTRA_CREDENTIALS in kwargs, we set it
-> Otherwise, if we use the server certificate, we set it to VAL_EXTRA_CREDENTIALS_HOST
-> If we have a delegation (see bellow), we set it to (delegatedDN, delegatedGroup)
-> otherwise it is an empty string
* delegation:
-> if KW_DELEGATED_DN in kwargs, or delegatedDN in threadConfig, put in in self.kwargs
-> if KW_DELEGATED_GROUP in kwargs or delegatedGroup in threadConfig, put it in self.kwargs
-> If we have a delegated DN but not group, we find the corresponding group in the CS
:return: S_OK()/S_ERROR()
"""
# which extra credentials to use?
self.__extraCredentials = self.VAL_EXTRA_CREDENTIALS_HOST if self.__useCertificates else ""
if self.KW_EXTRA_CREDENTIALS in self.kwargs:
self.__extraCredentials = self.kwargs[self.KW_EXTRA_CREDENTIALS]
# Are we delegating something?
delegatedDN = self.kwargs.get(self.KW_DELEGATED_DN) or self.__threadConfig.getDN()
delegatedGroup = self.kwargs.get(self.KW_DELEGATED_GROUP) or self.__threadConfig.getGroup()
if delegatedDN:
self.kwargs[self.KW_DELEGATED_DN] = delegatedDN
if not delegatedGroup:
result = Registry.findDefaultGroupForDN(delegatedDN)
if not result['OK']:
return result
delegatedGroup = result['Value']
self.kwargs[self.KW_DELEGATED_GROUP] = delegatedGroup
self.__extraCredentials = (delegatedDN, delegatedGroup)
return S_OK()
def __findServiceURL(self):
""" Discovers the URL of a service, taking into account gateways, multiple URLs, banned URLs
If the site on which we run is configured to use gateways (/DIRAC/Gateways/<siteName>),
these URLs will be used. To ignore the gateway, it is possible to set KW_IGNORE_GATEWAYS
to False in kwargs.
If self._destinationSrv (given as constructor attribute) is a properly formed URL,
we just return this one. If we have to use a gateway, we just replace the server name in the url.
The list of URLs defined in the CS (<System>/URLs/<Component>) is randomized
This method also sets some attributes:
* self.__nbOfUrls = number of URLs
* self.__nbOfRetry = 2 if we have more than 2 urls, otherwise 3
* self.__bannedUrls is reinitialized if all the URLs are banned
:return: S_OK(str)/S_ERROR() -- the selected URL
"""
if not self.__initStatus['OK']:
return self.__initStatus
# Load the Gateways URLs for the current site Name
gatewayURL = False
if not self.kwargs.get(self.KW_IGNORE_GATEWAYS):
dRetVal = gConfig.getOption("/DIRAC/Gateways/%s" % DIRAC.siteName())
if dRetVal['OK']:
rawGatewayURL = List.randomize(List.fromChar(dRetVal['Value'], ","))[0]
gatewayURL = "/".join(rawGatewayURL.split("/")[:3])
# If what was given as constructor attribute is a properly formed URL,
# we just return this one.
# If we have to use a gateway, we just replace the server name in it
for protocol in gProtocolDict:
if self._destinationSrv.find("%s://" % protocol) == 0:
gLogger.debug("Already given a valid url", self._destinationSrv)
if not gatewayURL:
return S_OK(self._destinationSrv)
gLogger.debug("Reconstructing given URL to pass through gateway")
path = "/".join(self._destinationSrv.split("/")[3:])
finalURL = "%s/%s" % (gatewayURL, path)
gLogger.debug("Gateway URL conversion:\n %s -> %s" % (self._destinationSrv, finalURL))
return S_OK(finalURL)
if gatewayURL:
gLogger.debug("Using gateway", gatewayURL)
return S_OK("%s/%s" % (gatewayURL, self._destinationSrv))
# We extract the list of URLs from the CS (System/URLs/Component)
try:
urls = getServiceURL(self._destinationSrv, setup=self.setup)
except Exception as e:
return S_ERROR("Cannot get URL for %s in setup %s: %s" % (self._destinationSrv, self.setup, repr(e)))
if not urls:
return S_ERROR("URL for service %s not found" % self._destinationSrv)
failoverUrls = []
# Try if there are some failover URLs to use as last resort
try:
failoverUrlsStr = getServiceFailoverURL(self._destinationSrv, setup=self.setup)
if failoverUrlsStr:
failoverUrls = failoverUrlsStr.split(',')
except Exception as e:
pass
# We randomize the list, and add at the end the failover URLs (System/FailoverURLs/Component)
urlsList = List.randomize(List.fromChar(urls, ",")) + failoverUrls
self.__nbOfUrls = len(urlsList)
self.__nbOfRetry = 2 if self.__nbOfUrls > 2 else 3 # we retry 2 times all services, if we run more than 2 services
if self.__nbOfUrls == len(self.__bannedUrls):
self.__bannedUrls = [] # retry all urls
gLogger.debug("Retrying again all URLs")
if len(self.__bannedUrls) > 0 and len(urlsList) > 1:
# we have host which is not accessible. We remove that host from the list.
# We only remove if we have more than one instance
for i in self.__bannedUrls:
gLogger.debug("Removing banned URL", "%s" % i)
urlsList.remove(i)
# Take the first URL from the list
# randUrls = List.randomize( urlsList ) + failoverUrls
sURL = urlsList[0]
# If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL
# is not on a host which is banned. If it is, we take the next one in the list using __selectUrl
# If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL
# is not on a host which is banned. If it is, we take the next one in the list using __selectUrl
if len(self.__bannedUrls) > 0 and self.__nbOfUrls > 2: # when we have multiple services then we can
# have a situation when two services are running on the same machine with different ports...
retVal = Network.splitURL(sURL)
nexturl = None
if retVal['OK']:
nexturl = retVal['Value']
found = False
for i in self.__bannedUrls:
retVal = Network.splitURL(i)
if retVal['OK']:
bannedurl = retVal['Value']
else:
break
# We found a banned URL on the same host as the one we are running on
if nexturl[1] == bannedurl[1]:
found = True
break
if found:
nexturl = self.__selectUrl(nexturl, urlsList[1:])
if nexturl: # an url found which is in different host
sURL = nexturl
gLogger.debug("Discovering URL for service", "%s -> %s" % (self._destinationSrv, sURL))
return S_OK(sURL)
def __selectUrl(self, notselect, urls):
""" In case when multiple services are running in the same host, a new url has to be in a different host
Note: If we do not have different host we will use the selected url...
:param notselect: URL that should NOT be selected
:param list urls: list of potential URLs
:return: str -- selected URL
"""
url = None
for i in urls:
retVal = Network.splitURL(i)
if retVal['OK']:
if retVal['Value'][1] != notselect[1]: # the hosts are different
url = i
break
else:
gLogger.error(retVal['Message'])
return url
def __checkThreadID(self):
"""
..warning:: just guessing....
This seems to check that we are not creating a client and then using it
in a multithreaded environment.
However, it is triggered only if self.__enableThreadCheck is to True, but it is
hardcoded to False, and does not seem to be modified anywhere in the code.
"""
if not self.__initStatus['OK']:
return self.__initStatus
cThID = thread.get_ident()
if not self.__allowedThreadID:
self.__allowedThreadID = cThID
elif cThID != self.__allowedThreadID:
msgTxt = """
=======DISET client thread safety error========================
Client %s
can only run on thread %s
and this is thread %s
===============================================================""" % (str(self),
self.__allowedThreadID,
cThID)
gLogger.error("DISET client thread safety error", msgTxt)
# raise Exception( msgTxt )
def _connect(self):
""" Establish the connection.
It uses the URL discovered in __discoverURL.
In case the connection cannot be established, __discoverURL
is called again, and _connect calls itself.
We stop after trying self.__nbOfRetry * self.__nbOfUrls
:return: S_OK()/S_ERROR()
"""
# Check if the useServerCertificate configuration changed
# Note: I am not really sure that all this block makes
# any sense at all since all these variables are
# evaluated in __discoverCredentialsToUse
if gConfig.useServerCertificate() != self.__useCertificates:
if self.__forceUseCertificates is None:
self.__useCertificates = gConfig.useServerCertificate()
self.kwargs[self.KW_USE_CERTIFICATES] = self.__useCertificates
# The server certificate use context changed, rechecking the transport sanity
result = self.__checkTransportSanity()
if not result['OK']:
return result
# Take all the extra credentials
self.__discoverExtraCredentials()
if not self.__initStatus['OK']:
return self.__initStatus
if self.__enableThreadCheck:
self.__checkThreadID()
gLogger.debug("Trying to connect to: %s" % self.serviceURL)
try:
# Calls the transport method of the apropriate protocol.
# self.__URLTuple[1:3] = [server name, port, System/Component]
transport = gProtocolDict[self.__URLTuple[0]]['transport'](self.__URLTuple[1:3], **self.kwargs)
# the socket timeout is the default value which is 1.
# later we increase to 5
retVal = transport.initAsClient()
# We try at most __nbOfRetry each URLs
if not retVal['OK']:
gLogger.warn("Issue getting socket:", "%s : %s : %s" % (transport, self.__URLTuple, retVal['Message']))
# We try at most __nbOfRetry each URLs
if self.__retry < self.__nbOfRetry * self.__nbOfUrls - 1:
# Recompose the URL (why not using self.serviceURL ? )
url = "%s://%s:%d/%s" % (self.__URLTuple[0], self.__URLTuple[1], int(self.__URLTuple[2]), self.__URLTuple[3])
# Add the url to the list of banned URLs if it is not already there. (Can it happen ? I don't think so)
if url not in self.__bannedUrls:
gLogger.warn("Non-responding URL temporarily banned", "%s" % url)
self.__bannedUrls += [url]
# Increment the retry counter
self.__retry += 1
# 16.07.20 CHRIS: I guess this setSocketTimeout does not behave as expected.
# If the initasClient did not work, we anyway re-enter the whole method,
# so a new transport object is created.
# However, it migh be that this timeout value was propagated down to the
# SocketInfoFactory singleton, and thus used, but that means that the timeout
# specified in parameter was then void.
# If it is our last attempt for each URL, we increase the timeout
if self.__retryCounter == self.__nbOfRetry - 1:
transport.setSocketTimeout(5) # we increase the socket timeout in case the network is not good
gLogger.info("Retry connection", ": %d to %s" % (self.__retry, self.serviceURL))
# If we tried all the URL, we increase the global counter (__retryCounter), and sleep
if len(self.__bannedUrls) == self.__nbOfUrls:
self.__retryCounter += 1
# we run only one service! In that case we increase the retry delay.
self.__retryDelay = 3. / self.__nbOfUrls if self.__nbOfUrls > 1 else 2
gLogger.info("Waiting %f seconds before retry all service(s)" % self.__retryDelay)
time.sleep(self.__retryDelay)
# rediscover the URL
self.__discoverURL()
# try to reconnect
return self._connect()
else:
return retVal
except Exception as e:
gLogger.exception(lException=True, lExcInfo=True)
return S_ERROR("Can't connect to %s: %s" % (self.serviceURL, repr(e)))
# We add the connection to the transport pool
gLogger.debug("Connected to: %s" % self.serviceURL)
trid = getGlobalTransportPool().add(transport)
return S_OK((trid, transport))
def _disconnect(self, trid):
""" Disconnect the connection.
:param str trid: Transport ID in the transportPool
"""
getGlobalTransportPool().close(trid)
@staticmethod
def _serializeStConnectionInfo(stConnectionInfo):
""" We want to send tuple but we need to convert
into a list
"""
serializedTuple = [list(x) if isinstance(x, tuple) else x for x in stConnectionInfo]
return serializedTuple
def _proposeAction(self, transport, action):
""" Proposes an action by sending a tuple containing
* System/Component
* Setup
* VO
* action
* extraCredentials
It is kind of a handshake.
The server might ask for a delegation, in which case it is done here.
The result of the delegation is then returned.
:param transport: the Transport object returned by _connect
:param action: tuple (<action type>, <action name>). It depends on the
subclasses of BaseClient. <action type> can be for example
'RPC' or 'FileTransfer'
:return: whatever the server sent back
"""
if not self.__initStatus['OK']:
return self.__initStatus
stConnectionInfo = ((self.__URLTuple[3], self.setup, self.vo),
action,
self.__extraCredentials,
DIRAC.version)
# Send the connection info and get the answer back
retVal = transport.sendData(S_OK(BaseClient._serializeStConnectionInfo(stConnectionInfo)))
if not retVal['OK']:
return retVal
serverReturn = transport.receiveData()
# TODO: Check if delegation is required. This seems to be used only for the GatewayService
if serverReturn['OK'] and 'Value' in serverReturn and isinstance(serverReturn['Value'], dict):
gLogger.debug("There is a server requirement")
serverRequirements = serverReturn['Value']
if 'delegate' in serverRequirements:
gLogger.debug("A delegation is requested")
serverReturn = self.__delegateCredentials(transport, serverRequirements['delegate'])
return serverReturn
def __delegateCredentials(self, transport, delegationRequest):
""" Perform a credential delegation. This seems to be used only for the GatewayService.
It calls the delegation mechanism of the Transport class. Note that it is not used when
delegating credentials to the ProxyDB
:param transport: the Transport object returned by _connect
:param delegationRequest: delegation request
:return: S_OK()/S_ERROR()
"""
retVal = gProtocolDict[self.__URLTuple[0]]['delegation'](delegationRequest, self.kwargs)
if not retVal['OK']:
return retVal
retVal = transport.sendData(retVal['Value'])
if not retVal['OK']:
return retVal
return transport.receiveData()
def __checkTransportSanity(self):
""" Calls the sanity check of the underlying Transport object
and stores the result in self.__idDict.
It is checked at the creation of the BaseClient, and when connecting
if the use of the certificate has changed.
:return: S_OK()/S_ERROR()
"""
if not self.__initStatus['OK']:
return self.__initStatus
retVal = gProtocolDict[self.__URLTuple[0]]['sanity'](self.__URLTuple[1:3], self.kwargs)
if not retVal['OK']:
return retVal
idDict = retVal['Value']
for key in idDict:
self.__idDict[key] = idDict[key]
return S_OK()
def __setKeepAliveLapse(self):
""" Select the maximum Keep alive lapse between
150 seconds and what is specifind in kwargs[KW_KEEP_ALIVE_LAPSE],
and sets it in kwargs[KW_KEEP_ALIVE_LAPSE]
:return: S_OK()/S_ERROR()
"""
kaa = 1
if self.KW_KEEP_ALIVE_LAPSE in self.kwargs:
try:
kaa = max(0, int(self.kwargs[self.KW_KEEP_ALIVE_LAPSE]))
except BaseException:
pass
if kaa:
kaa = max(150, kaa)
self.kwargs[self.KW_KEEP_ALIVE_LAPSE] = kaa
return S_OK()
def _getBaseStub(self):
""" Returns a list with [self._destinationSrv, newKwargs]
self._destinationSrv is what was given as first parameter of the init serviceName
newKwargs is an updated copy of kwargs:
* if set, we remove the useCertificates (KW_USE_CERTIFICATES) in newKwargs
This method is just used to return information in case of error in the InnerRPCClient
:return: tuple
"""
newKwargs = dict(self.kwargs)
# Remove useCertificates as the forwarder of the call will have to
# independently decide whether to use their cert or not anyway.
if 'useCertificates' in newKwargs:
del newKwargs['useCertificates']
return [self._destinationSrv, newKwargs]
def __bool__(self):
return True
# For Python 2 compatibility
__nonzero__ = __bool__
def __str__(self):
return "<DISET Client %s %s>" % (self.serviceURL, self.__extraCredentials)
class BaseClient:
VAL_EXTRA_CREDENTIALS_HOST = "hosts"
KW_USE_CERTIFICATES = "useCertificates"
KW_EXTRA_CREDENTIALS = "extraCredentials"
KW_TIMEOUT = "timeout"
KW_SETUP = "setup"
KW_VO = "VO"
KW_DELEGATED_DN = "delegatedDN"
KW_DELEGATED_GROUP = "delegatedGroup"
KW_IGNORE_GATEWAYS = "ignoreGateways"
KW_PROXY_LOCATION = "proxyLocation"
KW_PROXY_STRING = "proxyString"
KW_PROXY_CHAIN = "proxyChain"
KW_SKIP_CA_CHECK = "skipCACheck"
KW_KEEP_ALIVE_LAPSE = "keepAliveLapse"
__threadConfig = ThreadConfig()
def __init__(self, serviceName, **kwargs):
if type(serviceName) not in types.StringTypes:
raise TypeError(
"Service name expected to be a string. Received %s type %s" %
(str(serviceName), type(serviceName)))
self._destinationSrv = serviceName
self._serviceName = serviceName
self.kwargs = kwargs
self.__initStatus = S_OK()
self.__idDict = {}
self.__extraCredentials = ""
self.__enableThreadCheck = False
self.__retry = 0
self.__retryDelay = 0
self.__nbOfUrls = 1 #by default we always have 1 url for example: RPCClient('dips://volhcb38.cern.ch:9162/Framework/SystemAdministrator')
self.__nbOfRetry = 3 # by default we try try times
self.__retryCounter = 1
self.__bannedUrls = []
for initFunc in (self.__discoverSetup, self.__discoverVO,
self.__discoverTimeout, self.__discoverURL,
self.__discoverCredentialsToUse,
self.__checkTransportSanity,
self.__setKeepAliveLapse):
result = initFunc()
if not result['OK'] and self.__initStatus['OK']:
self.__initStatus = result
self.numberOfURLs = 0
self._initialize()
#HACK for thread-safety:
self.__allowedThreadID = False
def _initialize(self):
pass
def getDestinationService(self):
return self._destinationSrv
def getServiceName(self):
return self._serviceName
def __discoverSetup(self):
#Which setup to use?
if self.KW_SETUP in self.kwargs and self.kwargs[self.KW_SETUP]:
self.setup = str(self.kwargs[self.KW_SETUP])
else:
self.setup = self.__threadConfig.getSetup()
if not self.setup:
self.setup = gConfig.getValue("/DIRAC/Setup", "Test")
return S_OK()
def __discoverVO(self):
#Which setup to use?
if self.KW_VO in self.kwargs and self.kwargs[self.KW_VO]:
self.vo = str(self.kwargs[self.KW_VO])
else:
self.vo = gConfig.getValue("/DIRAC/VirtualOrganization", "unknown")
return S_OK()
def __discoverURL(self):
#Calculate final URL
try:
result = self.__findServiceURL()
except Exception as e:
return S_ERROR(repr(e))
if not result['OK']:
return result
self.serviceURL = result['Value']
retVal = Network.splitURL(self.serviceURL)
if not retVal['OK']:
return retVal
self.__URLTuple = retVal['Value']
self._serviceName = self.__URLTuple[-1]
res = gConfig.getOptionsDict("/DIRAC/ConnConf/%s:%s" %
self.__URLTuple[1:3])
if res['OK']:
opts = res['Value']
for k in opts:
if k not in self.kwargs:
self.kwargs[k] = opts[k]
return S_OK()
def __discoverTimeout(self):
if self.KW_TIMEOUT in self.kwargs:
self.timeout = self.kwargs[self.KW_TIMEOUT]
else:
self.timeout = False
if self.timeout:
self.timeout = max(120, self.timeout)
else:
self.timeout = 600
self.kwargs[self.KW_TIMEOUT] = self.timeout
return S_OK()
def __discoverCredentialsToUse(self):
#Use certificates?
if self.KW_USE_CERTIFICATES in self.kwargs:
self.useCertificates = self.kwargs[self.KW_USE_CERTIFICATES]
else:
self.useCertificates = gConfig.useServerCertificate()
self.kwargs[self.KW_USE_CERTIFICATES] = self.useCertificates
if self.KW_SKIP_CA_CHECK not in self.kwargs:
if self.useCertificates:
self.kwargs[self.KW_SKIP_CA_CHECK] = False
else:
self.kwargs[self.KW_SKIP_CA_CHECK] = CS.skipCACheck()
if self.KW_PROXY_CHAIN in self.kwargs:
try:
self.kwargs[self.KW_PROXY_STRING] = self.kwargs[
self.KW_PROXY_CHAIN].dumpAllToString()['Value']
del self.kwargs[self.KW_PROXY_CHAIN]
except:
return S_ERROR(
"Invalid proxy chain specified on instantiation")
return S_OK()
def __discoverExtraCredentials(self):
#Wich extra credentials to use?
if self.useCertificates:
self.__extraCredentials = self.VAL_EXTRA_CREDENTIALS_HOST
else:
self.__extraCredentials = ""
if self.KW_EXTRA_CREDENTIALS in self.kwargs:
self.__extraCredentials = self.kwargs[self.KW_EXTRA_CREDENTIALS]
#Are we delegating something?
delegatedDN, delegatedGroup = self.__threadConfig.getID()
if self.KW_DELEGATED_DN in self.kwargs and self.kwargs[
self.KW_DELEGATED_DN]:
delegatedDN = self.kwargs[self.KW_DELEGATED_DN]
elif delegatedDN:
self.kwargs[self.KW_DELEGATED_DN] = delegatedDN
if self.KW_DELEGATED_GROUP in self.kwargs and self.kwargs[
self.KW_DELEGATED_GROUP]:
delegatedGroup = self.kwargs[self.KW_DELEGATED_GROUP]
elif delegatedGroup:
self.kwargs[self.KW_DELEGATED_GROUP] = delegatedGroup
if delegatedDN:
if not delegatedGroup:
result = CS.findDefaultGroupForDN(
self.kwargs[self.KW_DELEGATED_DN])
if not result['OK']:
return result
self.__extraCredentials = (delegatedDN, delegatedGroup)
return S_OK()
def __findServiceURL(self):
if not self.__initStatus['OK']:
return self.__initStatus
gatewayURL = False
if self.KW_IGNORE_GATEWAYS not in self.kwargs or not self.kwargs[
self.KW_IGNORE_GATEWAYS]:
dRetVal = gConfig.getOption("/DIRAC/Gateways/%s" %
DIRAC.siteName())
if dRetVal['OK']:
rawGatewayURL = List.randomize(
List.fromChar(dRetVal['Value'], ","))[0]
gatewayURL = "/".join(rawGatewayURL.split("/")[:3])
for protocol in gProtocolDict.keys():
if self._destinationSrv.find("%s://" % protocol) == 0:
gLogger.debug("Already given a valid url",
self._destinationSrv)
if not gatewayURL:
return S_OK(self._destinationSrv)
gLogger.debug(
"Reconstructing given URL to pass through gateway")
path = "/".join(self._destinationSrv.split("/")[3:])
finalURL = "%s/%s" % (gatewayURL, path)
gLogger.debug("Gateway URL conversion:\n %s -> %s" %
(self._destinationSrv, finalURL))
return S_OK(finalURL)
if gatewayURL:
gLogger.debug("Using gateway", gatewayURL)
return S_OK("%s/%s" % (gatewayURL, self._destinationSrv))
try:
urls = getServiceURL(self._destinationSrv, setup=self.setup)
except Exception as e:
return S_ERROR("Cannot get URL for %s in setup %s: %s" %
(self._destinationSrv, self.setup, repr(e)))
if not urls:
return S_ERROR("URL for service %s not found" %
self._destinationSrv)
urlsList = List.fromChar(urls, ",")
self.__nbOfUrls = len(urlsList)
self.__nbOfRetry = 2 if self.__nbOfUrls > 2 else 3 # we retry 2 times all services, if we run more than 2 services
if len(urlsList) == len(self.__bannedUrls):
self.__bannedUrls = [] # retry all urls
gLogger.debug("Retrying again all URLs")
if len(self.__bannedUrls) > 0 and len(urlsList) > 1:
# we have host which is not accessible. We remove that host from the list.
# We only remove if we have more than one instance
for i in self.__bannedUrls:
gLogger.debug("Removing banned URL", "%s" % i)
urlsList.remove(i)
randUrls = List.randomize(urlsList)
sURL = randUrls[0]
if len(
self.__bannedUrls
) > 0 and self.__nbOfUrls > 2: # when we have multiple services then we can have a situation
# when two service are running on the same machine with different port...
retVal = Network.splitURL(sURL)
nexturl = None
if retVal['OK']:
nexturl = retVal['Value']
found = False
for i in self.__bannedUrls:
retVal = Network.splitURL(i)
if retVal['OK']:
bannedurl = retVal['Value']
else:
break
if nexturl[1] == bannedurl[1]:
found = True
break
if found:
nexturl = self.__selectUrl(nexturl, randUrls[1:])
if nexturl: # an url found which is in different host
sURL = nexturl
gLogger.debug("Discovering URL for service",
"%s -> %s" % (self._destinationSrv, sURL))
return S_OK(sURL)
def __selectUrl(self, notselect, urls):
"""In case when multiple services are running in the same host, a new url has to be in a different host
Note: If we do not have different host we will use the selected url...
"""
url = None
for i in urls:
retVal = Network.splitURL(i)
if retVal['OK']:
if retVal['Value'][1] != notselect[1]: # the hots are different
url = i
break
else:
gLogger.error(retVal['Message'])
return url
def __checkThreadID(self):
if not self.__initStatus['OK']:
return self.__initStatus
cThID = thread.get_ident()
if not self.__allowedThreadID:
self.__allowedThreadID = cThID
elif cThID != self.__allowedThreadID:
msgTxt = """
=======DISET client thread safety error========================
Client %s
can only run on thread %s
and this is thread %s
===============================================================""" % (
str(self), self.__allowedThreadID, cThID)
gLogger.error("DISET client thread safety error", msgTxt)
#raise Exception( msgTxt )
def _connect(self):
self.__discoverExtraCredentials()
if not self.__initStatus['OK']:
return self.__initStatus
if self.__enableThreadCheck:
self.__checkThreadID()
gLogger.debug("Connecting to: %s" % self.serviceURL)
try:
transport = gProtocolDict[self.__URLTuple[0]]['transport'](
self.__URLTuple[1:3], **self.kwargs)
#the socket timeout is the default value which is 1.
#later we increase to 5
retVal = transport.initAsClient()
if not retVal['OK']:
if self.__retry < self.__nbOfRetry * self.__nbOfUrls - 1:
url = "%s://%s:%d/%s" % (
self.__URLTuple[0], self.__URLTuple[1],
int(self.__URLTuple[2]), self.__URLTuple[3])
if url not in self.__bannedUrls:
self.__bannedUrls += [url]
if len(self.__bannedUrls) < self.__nbOfUrls:
gLogger.notice(
"Non-responding URL temporarily banned",
"%s" % url)
self.__retry += 1
if self.__retryCounter == self.__nbOfRetry - 1:
transport.setSocketTimeout(
5
) # we increase the socket timeout in case the network is not good
gLogger.info("Retry connection: ", "%d" % self.__retry)
if len(self.__bannedUrls) == self.__nbOfUrls:
self.__retryCounter += 1
self.__retryDelay = 3. / self.__nbOfUrls if self.__nbOfUrls > 1 else 2 # we run only one service! In that case we increase the retry delay.
gLogger.info(
"Waiting %f second before retry all service(s)" %
self.__retryDelay)
time.sleep(self.__retryDelay)
self.__discoverURL()
return self._connect()
else:
return retVal
except Exception as e:
return S_ERROR("Can't connect to %s: %s" %
(self.serviceURL, repr(e)))
trid = getGlobalTransportPool().add(transport)
return S_OK((trid, transport))
def _disconnect(self, trid):
getGlobalTransportPool().close(trid)
def _proposeAction(self, transport, action):
if not self.__initStatus['OK']:
return self.__initStatus
stConnectionInfo = ((self.__URLTuple[3], self.setup, self.vo), action,
self.__extraCredentials)
retVal = transport.sendData(S_OK(stConnectionInfo))
if not retVal['OK']:
return retVal
serverReturn = transport.receiveData()
#TODO: Check if delegation is required
if serverReturn['OK'] and 'Value' in serverReturn and type(
serverReturn['Value']) == types.DictType:
gLogger.debug("There is a server requirement")
serverRequirements = serverReturn['Value']
if 'delegate' in serverRequirements:
gLogger.debug("A delegation is requested")
serverReturn = self.__delegateCredentials(
transport, serverRequirements['delegate'])
return serverReturn
def __delegateCredentials(self, transport, delegationRequest):
retVal = gProtocolDict[self.__URLTuple[0]]['delegation'](
delegationRequest, self.kwargs)
if not retVal['OK']:
return retVal
retVal = transport.sendData(retVal['Value'])
if not retVal['OK']:
return retVal
return transport.receiveData()
def __checkTransportSanity(self):
if not self.__initStatus['OK']:
return self.__initStatus
retVal = gProtocolDict[self.__URLTuple[0]]['sanity'](
self.__URLTuple[1:3], self.kwargs)
if not retVal['OK']:
return retVal
idDict = retVal['Value']
for key in idDict:
self.__idDict[key] = idDict[key]
return S_OK()
def __setKeepAliveLapse(self):
kaa = 1
if self.KW_KEEP_ALIVE_LAPSE in self.kwargs:
try:
kaa = max(0, int(self.kwargs))
except:
pass
if kaa:
kaa = max(150, kaa)
self.kwargs[self.KW_KEEP_ALIVE_LAPSE] = kaa
return S_OK()
def _getBaseStub(self):
newKwargs = dict(self.kwargs)
#Set DN
tDN, tGroup = self.__threadConfig.getID()
if not self.KW_DELEGATED_DN in newKwargs:
if tDN:
newKwargs[self.KW_DELEGATED_DN] = tDN
elif 'DN' in self.__idDict:
newKwargs[self.KW_DELEGATED_DN] = self.__idDict['DN']
#Discover group
if not self.KW_DELEGATED_GROUP in newKwargs:
if 'group' in self.__idDict:
newKwargs[self.KW_DELEGATED_GROUP] = self.__idDict['group']
elif tGroup:
newKwargs[self.KW_DELEGATED_GROUP] = tGroup
else:
if self.KW_DELEGATED_DN in newKwargs:
if CS.getUsernameForDN(
newKwargs[self.KW_DELEGATED_DN])['OK']:
result = CS.findDefaultGroupForDN(
newKwargs[self.KW_DELEGATED_DN])
if result['OK']:
newKwargs[
self.KW_DELEGATED_GROUP] = result['Value']
if CS.getHostnameForDN(
newKwargs[self.KW_DELEGATED_DN])['OK']:
newKwargs[
self.
KW_DELEGATED_GROUP] = self.VAL_EXTRA_CREDENTIALS_HOST
if 'useCertificates' in newKwargs:
del (newKwargs['useCertificates'])
return (self._destinationSrv, newKwargs)
def __nonzero__(self):
return True
def __str__(self):
return "<DISET Client %s %s>" % (self.serviceURL,
self.__extraCredentials)
class SessionData(object):
__disetConfig = ThreadConfig()
__handlers = {}
__groupMenu = {}
__extensions = []
__extVersion = False
@classmethod
def setHandlers(cls, handlers):
cls.__handlers = {}
for k in handlers:
handler = handlers[k]
cls.__handlers[handler.LOCATION.strip("/")] = handler
#Calculate extensions
cls.__extensions = []
for ext in CSGlobals.getInstalledExtensions():
if ext in ("EiscatWebDIRAC", "WebAppDIRAC", "DIRAC"):
continue
cls.__extensions.append(ext)
cls.__extensions.append("DIRAC")
cls.__extensions.append("EiscatWebDIRAC")
print "cls.__extensions in SessionData Lib"
print cls.__extensions
def __init__(self, credDict, setup):
self.__credDict = credDict
self.__setup = setup
def __isGroupAuthApp(self, appLoc):
handlerLoc = "/".join(List.fromChar(appLoc, ".")[1:])
if not handlerLoc:
return False
if handlerLoc not in self.__handlers:
gLogger.error("Handler %s required by %s does not exist!" %
(handlerLoc, appLoc))
return False
handler = self.__handlers[handlerLoc]
auth = AuthManager(Conf.getAuthSectionForHandler(handlerLoc))
return auth.authQuery("", dict(self.__credDict), handler.AUTH_PROPS)
def __generateSchema(self, base, path):
"""
Generate a menu schema based on the user credentials
"""
#Calculate schema
schema = []
fullName = "%s/%s" % (base, path)
result = gConfig.getSections(fullName)
if not result['OK']:
return schema
sectionsList = result['Value']
for sName in sectionsList:
subSchema = self.__generateSchema(base, "%s/%s" % (path, sName))
if subSchema:
schema.append((sName, subSchema))
result = gConfig.getOptions(fullName)
if not result['OK']:
return schema
optionsList = result['Value']
for opName in optionsList:
opVal = gConfig.getValue("%s/%s" % (fullName, opName))
if opVal.find("link|") == 0:
schema.append(("link", opName, opVal[5:]))
continue
if self.__isGroupAuthApp(opVal):
schema.append(("app", opName, opVal))
return schema
def __getGroupMenu(self):
"""
Load the schema from the CS and filter based on the group
"""
#Somebody coming from HTTPS and not with a valid group
group = self.__credDict.get("group", "")
#Cache time!
if group not in self.__groupMenu:
base = "%s/Schema" % (Conf.BASECS)
self.__groupMenu[group] = self.__generateSchema(base, "")
return self.__groupMenu[group]
@classmethod
def getWebAppPath(cls):
return os.path.join(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))),
"WebApp")
@classmethod
def getExtJSVersion(cls):
if not cls.__extVersion:
extPath = os.path.join(cls.getWebAppPath(), "static", "extjs")
extVersionPath = []
for entryName in os.listdir(extPath):
if entryName.find("ext-") == 0:
extVersionPath.append(entryName)
cls.__extVersion = sorted(extVersionPath)[-1]
return cls.__extVersion
def getData(self):
data = {
'menu': self.__getGroupMenu(),
'user': self.__credDict,
'validGroups': [],
'setup': self.__setup,
'validSetups': gConfig.getSections("/DIRAC/Setups")['Value'],
'extensions': self.__extensions,
'extVersion': self.getExtJSVersion()
}
#Add valid groups if known
DN = self.__credDict.get("DN", "")
if DN:
result = Registry.getGroupsForDN(DN)
if result['OK']:
data['validGroups'] = result['Value']
#Calculate baseURL
baseURL = [
Conf.rootURL().strip("/"),
"s:%s" % data['setup'],
"g:%s" % self.__credDict.get('group', '')
]
data['baseURL'] = "/%s" % "/".join(baseURL)
return data
class WebHandler(tornado.web.RequestHandler):
__disetConfig = ThreadConfig()
__log = False
#Auth requirements
AUTH_PROPS = None
#Location of the handler in the URL
LOCATION = ""
#URL Schema with holders to generate handler urls
URLSCHEMA = ""
#RE to extract group and setup
PATH_RE = ""
#Helper function to create threaded gen.Tasks with automatic callback and execption handling
def threadTask(self, method, *args, **kwargs):
"""
Helper method to generate a gen.Task and automatically call the callback when the real
method ends. THIS IS SPARTAAAAAAAAAA. SPARTA has improved using futures ;)
"""
#Save the task to access the runner
genTask = False
#This runs in the separate thread, calls the callback on finish and takes into account exceptions
def cbMethod(*cargs, **ckwargs):
cb = ckwargs.pop('callback')
method = cargs[0]
disetConf = cargs[1]
cargs = cargs[2]
self.__disetConfig.reset()
self.__disetConfig.load(disetConf)
ioloop = tornado.ioloop.IOLoop.instance()
try:
result = method(*cargs, **ckwargs)
ioloop.add_callback(functools.partial(cb, result))
except Exception as excp:
gLogger.error("Following exception occured %s" % excp)
exc_info = sys.exc_info()
genTask.set_exc_info(exc_info)
ioloop.add_callback(lambda: genTask.exception())
#Put the task in the thread :)
def threadJob(tmethod, *targs, **tkwargs):
tkwargs['callback'] = tornado.stack_context.wrap(
tkwargs['callback'])
targs = (tmethod, self.__disetDump, targs)
gThreadPool.submit(cbMethod, *targs, **tkwargs)
#Return a YieldPoint
genTask = tornado.gen.Task(threadJob, method, *args, **kwargs)
return genTask
def __disetBlockDecor(self, func):
def wrapper(*args, **kwargs):
raise RuntimeError(
"All DISET calls must be made from inside a Threaded Task! Bad boy!"
)
return wrapper
def __init__(self, *args, **kwargs):
"""
Initialize the handler
"""
super(WebHandler, self).__init__(*args, **kwargs)
if not WebHandler.__log:
WebHandler.__log = gLogger.getSubLogger(self.__class__.__name__)
self.__credDict = {}
self.__setup = Conf.setup()
self.__processCredentials()
self.__disetConfig.reset()
self.__disetConfig.setDecorator(self.__disetBlockDecor)
self.__disetDump = self.__disetConfig.dump()
match = self.PATH_RE.match(self.request.path)
self._pathResult = self.__checkPath(*match.groups())
self.__sessionData = SessionData(self.__credDict, self.__setup)
def __processCredentials(self):
"""
Extract the user credentials based on the certificate or what comes from the balancer
"""
#NGINX
if Conf.balancer() == "nginx":
headers = self.request.headers
if headers['X-Scheme'] == "https" and headers[
'X-Ssl_client_verify'] == 'SUCCESS':
DN = headers['X-Ssl_client_s_dn']
self.__credDict['DN'] = DN
self.__credDict['issuer'] = headers['X-Ssl_client_i_dn']
result = Registry.getUsernameForDN(DN)
if not result['OK']:
self.__credDict['validDN'] = False
else:
self.__credDict['validDN'] = True
self.__credDict['username'] = result['Value']
return
#TORNADO
if not self.request.protocol == "https":
return
derCert = self.request.get_ssl_certificate(binary_form=True)
if not derCert:
return
pemCert = ssl.DER_cert_to_PEM_cert(derCert)
chain = X509Chain()
chain.loadChainFromString(pemCert)
result = chain.getCredentials()
if not result['OK']:
self.log.error("Could not get client credentials %s" %
result['Message'])
return
self.__credDict = result['Value']
#Hack. Data coming from OSSL directly and DISET difer in DN/subject
try:
self.__credDict['DN'] = self.__credDict['subject']
except KeyError:
pass
def _request_summary(self):
"""
Return a string returning the summary of the request
"""
summ = super(WebHandler, self)._request_summary()
cl = []
if self.__credDict.get('validDN', False):
cl.append(self.__credDict['username'])
if self.__credDict.get('validGroup', False):
cl.append("@%s" % self.__credDict['group'])
cl.append(" (%s)" % self.__credDict['DN'])
summ = "%s %s" % (summ, "".join(cl))
return summ
@property
def log(self):
return self.__log
@classmethod
def getLog(cls):
return cls.__log
def getUserDN(self):
return self.__credDict.get('DN', '')
def getUserName(self):
return self.__credDict.get('username', '')
def getUserGroup(self):
return self.__credDict.get('group', '')
def getUserSetup(self):
return self.__setup
def getUserProperties(self):
return self.__sessionData.getData().properties
def isRegisteredUser(self):
return self.__credDict.get('validDN', "") and self.__credDict.get(
'validGroup', "")
def getSessionData(self):
return self.__sessionData.getData()
def actionURL(self, action=""):
"""
Given an action name for the handler, return the URL
"""
if action == "index":
action = ""
group = self.getUserGroup()
if group:
group = "/g:%s" % group
setup = self.getUserSetup()
if setup:
setup = "/s:%s" % setup
location = self.LOCATION
if location:
location = "/%s" % location
ats = dict(action=action, group=group, setup=setup, location=location)
return self.URLSCHEMA % ats
def __auth(self, handlerRoute, group):
"""
Authenticate request
"""
userDN = self.getUserDN()
if group:
self.__credDict['group'] = group
else:
if userDN:
result = Registry.findDefaultGroupForDN(userDN)
if result['OK']:
self.__credDict['group'] = result['Value']
self.__credDict['validGroup'] = False
if type(self.AUTH_PROPS) not in (types.ListType, types.TupleType):
self.AUTH_PROPS = [
p.strip() for p in self.AUTH_PROPS.split(",") if p.strip()
]
allAllowed = False
for p in self.AUTH_PROPS:
if p.lower() in ('all', 'any'):
allAllowed = True
auth = AuthManager(Conf.getAuthSectionForHandler(handlerRoute))
ok = auth.authQuery("", self.__credDict, self.AUTH_PROPS)
if ok:
if userDN:
self.__credDict['validGroup'] = True
self.log.info("AUTH OK: %s by %s@%s (%s)" %
(handlerRoute, self.__credDict['username'],
self.__credDict['group'], userDN))
else:
self.__credDict['validDN'] = False
self.log.info("AUTH OK: %s by visitor" % (handlerRoute))
elif allAllowed:
self.log.info("AUTH ALL: %s by %s" % (handlerRoute, userDN))
ok = True
else:
self.log.info("AUTH KO: %s by %s@%s" %
(handlerRoute, userDN, group))
return ok
def __checkPath(self, setup, group, route):
"""
Check the request, auth, credentials and DISET config
"""
if route[-1] == "/":
methodName = "index"
handlerRoute = route
else:
iP = route.rfind("/")
methodName = route[iP + 1:]
handlerRoute = route[:iP]
if setup:
self.__setup = setup
if not self.__auth(handlerRoute, group):
return WErr(401, "Unauthorized, bad boy!")
DN = self.getUserDN()
if DN:
self.__disetConfig.setDN(DN)
group = self.getUserGroup()
if group:
self.__disetConfig.setGroup(group)
self.__disetConfig.setSetup(setup)
self.__disetDump = self.__disetConfig.dump()
return WOK(methodName)
def get(self, setup, group, route):
if not self._pathResult.ok:
raise self._pathResult
methodName = "web_%s" % self._pathResult.data
try:
mObj = getattr(self, methodName)
except AttributeError as e:
self.log.fatal("This should not happen!! %s" % e)
raise tornado.web.HTTPError(404)
return mObj()
def post(self, *args, **kwargs):
return self.get(*args, **kwargs)
def write_error(self, status_code, **kwargs):
self.set_status(status_code)
cType = "text/plain"
data = self._reason
if 'exc_info' in kwargs:
ex = kwargs['exc_info'][1]
trace = traceback.format_exception(*kwargs["exc_info"])
if not isinstance(ex, WErr):
data += "\n".join(trace)
else:
if self.settings.get("debug"):
self.log.error("Request ended in error:\n %s" %
"\n ".join(trace))
data = ex.msg
if type(data) == types.DictType:
cType = "application/json"
data = json.dumps(data)
self.set_header('Content-Type', cType)
self.finish(data)
class S3GatewayHandler(RequestHandler):
"""
.. class:: S3GatewayHandler
"""
# FC instance to check whether access is permitted or not
_fc = None
# Mapping between the S3 methods and the DFC methods
_s3ToFC_methods = {
"head_object": "getFileMetadata",
"get_object":
"getFileMetadata", # consider that if we are allowed to see the file metadata
# we can also download it
"put_object": "addFile",
"delete_object": "removeFile",
}
_S3Storages = {}
# This allows us to perform the DFC queries on behalf of a user
# without having to recreate a DFC object every time and
# pass it the "delegatedDN" and "delegatedGroup" values
_tc = ThreadConfig()
@classmethod
def initializeHandler(cls, serviceInfoDict):
"""initialize handler"""
log = LOG.getSubLogger("initializeHandler")
for seName in DMSHelpers().getStorageElements():
se = StorageElement(seName)
# TODO: once we finally merge _allProtocolParameters with the
# standard paramaters in the StorageBase, this will be much neater
for storagePlugin in se.storages:
storageParam = storagePlugin._allProtocolParameters # pylint: disable=protected-access
if (storageParam.get("Protocol") == "s3"
and "Aws_access_key_id" in storageParam
and "Aws_secret_access_key" in storageParam):
cls._S3Storages[seName] = storagePlugin
log.debug("Add %s to the list of usable S3 storages" %
seName)
break
log.info("S3Gateway initialized storages",
"%s" % list(cls._S3Storages))
cls._fc = FileCatalog()
return S_OK()
def _hasAccess(self, lfn, s3_method):
"""Check if we have permission to execute given operation on the given file (if exists) or its directory"""
opType = self._s3ToFC_methods.get(s3_method)
if not opType:
return S_ERROR(errno.EINVAL, "Unknown S3 method %s" % s3_method)
return returnSingleResult(self._fc.hasAccess(lfn, opType))
types_createPresignedUrl = [
six.string_types, six.string_types, (dict, list), six.integer_types
]
def export_createPresignedUrl(self, storageName, s3_method, urls,
expiration):
"""Generate a presigned URL for a given object, given method, and given storage
Permissions are checked against the DFC
:param storageName: SE name
:param s3_method: name of the S3 client method we want to perform.
:param urls: Iterable of urls. If s3_method is put_object, it must be a dict <url:fields> where fields
is a dictionary (see ~DIRAC.Resources.Storage.S3Storage.S3Storage.createPresignedUrl)
:param expiration: duration of the token
"""
log = LOG.getSubLogger("createPresignedUrl")
if s3_method == "put_object" and not isinstance(urls, dict):
return S_ERROR(errno.EINVAL, "urls has to be a dict <url:fields>")
# Fetch the remote credentials, and set them in the ThreadConfig
# This allows to perform the FC operations on behalf of the user
credDict = self.getRemoteCredentials()
if not credDict:
# If we can't obtain remote credentials, consider it permission denied
return S_ERROR(errno.EACCES, "Could not obtain remote credentials")
self._tc.setDN(credDict["DN"])
self._tc.setGroup(credDict["group"])
successful = {}
failed = {}
s3Plugin = self._S3Storages[storageName]
for url in urls:
try:
log.verbose(
"Creating presigned URL",
"SE: %s Method: %s URL: %s Expiration: %s" %
(storageName, s3_method, url, expiration),
)
# Finding the LFN to query the FC
# I absolutely hate doing such path mangling but well....
res = s3Plugin._getKeyFromURL(url) # pylint: disable=protected-access
if not res["OK"]:
failed[url] = res["Message"]
log.debug("Could not parse the url %s %s" % (url, res))
continue
lfn = "/" + res["Value"]
log.debug("URL: %s -> LFN %s" % (url, lfn))
# Checking whether access is permitted
res = self._hasAccess(lfn, s3_method)
if not res["OK"]:
failed[url] = res["Message"]
continue
if not res["Value"]:
failed[url] = "Permission denied"
continue
res = returnSingleResult(
s3Plugin.createPresignedUrl({url: urls.get("Fields")},
s3_method,
expiration=expiration))
log.debug("Presigned URL for %s: %s" % (url, res))
if res["OK"]:
successful[url] = res["Value"]
else:
failed["url"] = res["Message"]
except Exception as e:
log.exception("Exception presigning URL")
failed[url] = repr(e)
return S_OK({"Successful": successful, "Failed": failed})
class WebHandler(tornado.web.RequestHandler):
__disetConfig = ThreadConfig()
__log = False
# Auth requirements
AUTH_PROPS = None
# Location of the handler in the URL
LOCATION = ""
# URL Schema with holders to generate handler urls
URLSCHEMA = ""
# RE to extract group and setup
PATH_RE = ""
def threadTask(self, method, *args, **kwargs):
if tornado.version < '5.0.0':
return self.threadTaskOld(method, *args, **kwargs)
else:
return self.threadTaskExecutor(method, *args, **kwargs)
# Helper function to create threaded gen.Tasks with automatic callback and execption handling
@deprecated("Only for Tornado 4.x.x and DIRAC v6r20")
def threadTaskOld(self, method, *args, **kwargs):
"""
Helper method to generate a gen.Task and automatically call the callback when the real
method ends. THIS IS SPARTAAAAAAAAAA. SPARTA has improved using futures ;)
"""
# Save the task to access the runner
genTask = False
# This runs in the separate thread, calls the callback on finish and takes into account exceptions
def cbMethod(*cargs, **ckwargs):
cb = ckwargs.pop('callback')
method = cargs[0]
disetConf = cargs[1]
cargs = cargs[2]
self.__disetConfig.reset()
self.__disetConfig.load(disetConf)
ioloop = tornado.ioloop.IOLoop.instance()
try:
result = method(*cargs, **ckwargs)
ioloop.add_callback(functools.partial(cb, result))
except Exception as excp:
gLogger.error("Following exception occured %s" % excp)
exc_info = sys.exc_info()
genTask.set_exc_info(exc_info)
ioloop.add_callback(lambda: genTask.exception())
# Put the task in the thread :)
def threadJob(tmethod, *targs, **tkwargs):
tkwargs['callback'] = tornado.stack_context.wrap(
tkwargs['callback'])
targs = (tmethod, self.__disetDump, targs)
gThreadPool.submit(cbMethod, *targs, **tkwargs)
# Return a YieldPoint
genTask = tornado.gen.Task(threadJob, method, *args, **kwargs)
return genTask
def threadTaskExecutor(self, method, *args, **kwargs):
def threadJob(*targs, **tkwargs):
args = targs[0]
disetConf = targs[1]
self.__disetConfig.reset()
self.__disetConfig.load(disetConf)
return method(*args, **tkwargs)
targs = (args, self.__disetDump)
return tornado.ioloop.IOLoop.current().run_in_executor(
gThreadPool, functools.partial(threadJob, *targs, **kwargs))
def __disetBlockDecor(self, func):
def wrapper(*args, **kwargs):
raise RuntimeError(
"All DISET calls must be made from inside a Threaded Task!")
return wrapper
def __init__(self, *args, **kwargs):
"""
Initialize the handler
"""
super(WebHandler, self).__init__(*args, **kwargs)
if not WebHandler.__log:
WebHandler.__log = gLogger.getSubLogger(self.__class__.__name__)
self.__credDict = {}
self.__setup = Conf.setup()
self.__processCredentials()
self.__disetConfig.reset()
self.__disetConfig.setDecorator(self.__disetBlockDecor)
self.__disetDump = self.__disetConfig.dump()
match = self.PATH_RE.match(self.request.path)
self._pathResult = self.__checkPath(*match.groups())
self.__sessionData = SessionData(self.__credDict, self.__setup)
def __processCredentials(self):
"""
Extract the user credentials based on the certificate or what comes from the balancer
"""
if not self.request.protocol == "https":
return
# OIDC auth method
def oAuth2():
if self.get_secure_cookie("AccessToken"):
access_token = self.get_secure_cookie("AccessToken")
url = Conf.getCSValue(
"TypeAuths/%s/authority" % typeAuth) + '/userinfo'
heads = {
'Authorization': 'Bearer ' + access_token,
'Content-Type': 'application/json'
}
if 'error' in requests.get(url, headers=heads,
verify=False).json():
self.log.error('OIDC request error: %s' % requests.get(
url, headers=heads, verify=False).json()['error'])
return
ID = requests.get(url, headers=heads,
verify=False).json()['sub']
result = getUsernameForID(ID)
if result['OK']:
self.__credDict['username'] = result['Value']
result = getDNForUsername(self.__credDict['username'])
if result['OK']:
self.__credDict['validDN'] = True
self.__credDict['DN'] = result['Value'][0]
result = getCAForUsername(self.__credDict['username'])
if result['OK']:
self.__credDict['issuer'] = result['Value'][0]
return
# Type of Auth
if not self.get_secure_cookie("TypeAuth"):
self.set_secure_cookie("TypeAuth", 'Certificate')
typeAuth = self.get_secure_cookie("TypeAuth")
self.log.info("Type authentication: %s" % str(typeAuth))
if typeAuth == "Visitor":
return
retVal = Conf.getCSSections("TypeAuths")
if retVal['OK']:
if typeAuth in retVal.get("Value"):
method = Conf.getCSValue("TypeAuths/%s/method" % typeAuth,
'default')
if method == "oAuth2":
oAuth2()
# NGINX
if Conf.balancer() == "nginx":
headers = self.request.headers
if headers['X-Scheme'] == "https" and headers[
'X-Ssl_client_verify'] == 'SUCCESS':
DN = headers['X-Ssl_client_s_dn']
if not DN.startswith('/'):
items = DN.split(',')
items.reverse()
DN = '/' + '/'.join(items)
self.__credDict['DN'] = DN
self.__credDict['issuer'] = headers['X-Ssl_client_i_dn']
result = Registry.getUsernameForDN(DN)
if not result['OK']:
self.__credDict['validDN'] = False
else:
self.__credDict['validDN'] = True
self.__credDict['username'] = result['Value']
return
# TORNADO
derCert = self.request.get_ssl_certificate(binary_form=True)
if not derCert:
return
pemCert = ssl.DER_cert_to_PEM_cert(derCert)
chain = X509Chain()
chain.loadChainFromString(pemCert)
result = chain.getCredentials()
if not result['OK']:
self.log.error("Could not get client credentials %s" %
result['Message'])
return
self.__credDict = result['Value']
# Hack. Data coming from OSSL directly and DISET difer in DN/subject
try:
self.__credDict['DN'] = self.__credDict['subject']
except KeyError:
pass
def _request_summary(self):
"""
Return a string returning the summary of the request
"""
summ = super(WebHandler, self)._request_summary()
cl = []
if self.__credDict.get('validDN', False):
cl.append(self.__credDict['username'])
if self.__credDict.get('validGroup', False):
cl.append("@%s" % self.__credDict['group'])
cl.append(" (%s)" % self.__credDict['DN'])
summ = "%s %s" % (summ, "".join(cl))
return summ
@property
def log(self):
return self.__log
@classmethod
def getLog(cls):
return cls.__log
def getUserDN(self):
return self.__credDict.get('DN', '')
def getUserName(self):
return self.__credDict.get('username', '')
def getUserGroup(self):
return self.__credDict.get('group', '')
def getUserSetup(self):
return self.__setup
def getUserProperties(self):
return self.__sessionData.getData().properties
def isRegisteredUser(self):
return self.__credDict.get('validDN', "") and self.__credDict.get(
'validGroup', "")
def getSessionData(self):
return self.__sessionData.getData()
def actionURL(self, action=""):
"""
Given an action name for the handler, return the URL
"""
if action == "index":
action = ""
group = self.getUserGroup()
if group:
group = "/g:%s" % group
setup = self.getUserSetup()
if setup:
setup = "/s:%s" % setup
location = self.LOCATION
if location:
location = "/%s" % location
ats = dict(action=action, group=group, setup=setup, location=location)
return self.URLSCHEMA % ats
def __auth(self, handlerRoute, group, method):
"""
Authenticate request
:param str handlerRoute: the name of the handler
:param str group: DIRAC group
:param str method: the name of the method
:return: bool
"""
userDN = self.getUserDN()
if group:
self.__credDict['group'] = group
else:
if userDN:
result = Registry.findDefaultGroupForDN(userDN)
if result['OK']:
self.__credDict['group'] = result['Value']
self.__credDict['validGroup'] = False
if type(self.AUTH_PROPS) not in (types.ListType, types.TupleType):
self.AUTH_PROPS = [
p.strip() for p in self.AUTH_PROPS.split(",") if p.strip()
]
auth = AuthManager(Conf.getAuthSectionForHandler(handlerRoute))
ok = auth.authQuery(method, self.__credDict, self.AUTH_PROPS)
if ok:
if userDN:
self.__credDict['validGroup'] = True
self.log.info("AUTH OK: %s by %s@%s (%s)" %
(handlerRoute, self.__credDict['username'],
self.__credDict['group'], userDN))
else:
self.__credDict['validDN'] = False
self.log.info("AUTH OK: %s by visitor" % (handlerRoute))
elif self.isTrustedHost(self.__credDict.get('DN')):
self.log.info("Request is coming from Trusted host")
return True
else:
self.log.info("AUTH KO: %s by %s@%s" %
(handlerRoute, userDN, group))
return ok
def isTrustedHost(self, dn):
"""
Check if the request coming from a TrustedHost
:param str dn: certificate DN
:return: bool if the host is Trusrted it return true otherwise false
"""
retVal = CS.getHostnameForDN(dn)
if retVal['OK']:
hostname = retVal['Value']
if Properties.TRUSTED_HOST in CS.getPropertiesForHost(
hostname, []):
return True
return False
def __checkPath(self, setup, group, route):
"""
Check the request, auth, credentials and DISET config
"""
if route[-1] == "/":
methodName = "index"
handlerRoute = route
else:
iP = route.rfind("/")
methodName = route[iP + 1:]
handlerRoute = route[:iP]
if setup:
self.__setup = setup
if not self.__auth(handlerRoute, group, methodName):
return WErr(401, "Unauthorized.")
DN = self.getUserDN()
if DN:
self.__disetConfig.setDN(DN)
group = self.getUserGroup()
if group:
self.__disetConfig.setGroup(group)
self.__disetConfig.setSetup(setup)
self.__disetDump = self.__disetConfig.dump()
return WOK(methodName)
def get(self, setup, group, route):
if not self._pathResult.ok:
raise self._pathResult
methodName = "web_%s" % self._pathResult.data
try:
mObj = getattr(self, methodName)
except AttributeError as e:
self.log.fatal("This should not happen!! %s" % e)
raise tornado.web.HTTPError(404)
return mObj()
def post(self, *args, **kwargs):
return self.get(*args, **kwargs)
def write_error(self, status_code, **kwargs):
self.set_status(status_code)
cType = "text/plain"
data = self._reason
if 'exc_info' in kwargs:
ex = kwargs['exc_info'][1]
trace = traceback.format_exception(*kwargs["exc_info"])
if not isinstance(ex, WErr):
data += "\n".join(trace)
else:
if self.settings.get("debug"):
self.log.error("Request ended in error:\n %s" %
"\n ".join(trace))
data = ex.msg
if isinstance(data, dict):
cType = "application/json"
data = json.dumps(data)
self.set_header('Content-Type', cType)
self.finish(data)
class UPHandler(WebHandler):
AUTH_PROPS = "authenticated"
__tc = ThreadConfig()
def prepare(self):
if not self.isRegisteredUser():
raise WErr(401, "Not a registered user")
self.set_header("Pragma", "no-cache")
self.set_header("Cache-Control",
"max-age=0, no-store, no-cache, must-revalidate")
#Do not use the defined user setup. Use the web one to show the same profile independenly of
# user setup
self.__tc.setSetup(False)
def __getUP(self):
try:
obj = self.request.arguments['obj'][-1]
app = self.request.arguments['app'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
return UserProfileClient("Web/%s/%s" % (obj, app))
@asyncGen
def web_saveAppState(self):
up = self.__getUP()
try:
name = self.request.arguments['name'][-1]
state = self.request.arguments['state'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
data = base64.b64encode(zlib.compress(DEncode.encode(state), 9))
result = yield self.threadTask(up.storeVar, name, data)
if not result['OK']:
raise WErr.fromSERROR(result)
self.set_status(200)
self.finish()
@asyncGen
def web_loadAppState(self):
up = self.__getUP()
try:
name = self.request.arguments['name'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
result = yield self.threadTask(up.retrieveVar, name)
if not result['OK']:
raise WErr.fromSERROR(result)
data = result['Value']
data, count = DEncode.decode(zlib.decompress(base64.b64decode(data)))
self.finish(data)
@asyncGen
def web_listAppState(self):
up = self.__getUP()
result = yield self.threadTask(up.retrieveAllVars)
if not result['OK']:
raise WErr.fromSERROR(result)
data = result['Value']
for k in data:
#Unpack data
data[k] = json.loads(
DEncode.decode(zlib.decompress(base64.b64decode(data[k])))[0])
self.finish(data)
@asyncGen
def web_delAppState(self):
up = self.__getUP()
try:
name = self.request.arguments['name'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
result = yield self.threadTask(up.deleteVar, name)
if not result['OK']:
raise WErr.fromSERROR(result)
self.finish()
class UPHandler(WebHandler):
AUTH_PROPS = "authenticated"
__tc = ThreadConfig()
def prepare(self):
if not self.isRegisteredUser():
raise WErr(401, "Not a registered user")
self.set_header("Pragma", "no-cache")
self.set_header("Cache-Control",
"max-age=0, no-store, no-cache, must-revalidate")
# Do not use the defined user setup. Use the web one to show the same profile independenly of
# user setup
self.__tc.setSetup(False)
def __getUP(self):
try:
obj = self.request.arguments['obj'][-1]
app = self.request.arguments['app'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
return UserProfileClient("Web/%s/%s" % (obj, app))
@asyncGen
def web_saveAppState(self):
up = self.__getUP()
try:
name = self.request.arguments['name'][-1]
state = self.request.arguments['state'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
data = base64.b64encode(zlib.compress(DEncode.encode(state), 9))
# before we save the state (modify the state) we have to remeber the actual access: ReadAccess and PublishAccess
result = yield self.threadTask(up.getVarPermissions, name)
if result['OK']:
access = result['Value']
else:
access = {
'ReadAccess': 'USER',
'PublishAccess': 'USER'
} # this is when the application/desktop does not exists.
result = yield self.threadTask(up.storeVar, name, data)
if not result['OK']:
raise WErr.fromSERROR(result)
# change the access to the application/desktop
result = yield self.threadTask(up.setVarPermissions, name, access)
if not result['OK']:
raise WErr.fromSERROR(result)
self.set_status(200)
self.finish()
@asyncGen
def web_makePublicAppState(self):
up = self.__getUP()
try:
name = self.request.arguments['name'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
try:
access = self.request.arguments['access'][-1].upper()
except KeyError as excp:
access = 'ALL'
if access not in ('ALL', 'VO', 'GROUP', 'USER'):
raise WErr(400, "Invalid access")
revokeAccess = {'ReadAccess': access}
if access == 'USER': # if we make private a state,
# we have to revoke from the public as well
revokeAccess['PublishAccess'] = 'USER'
# TODO: Check access is in either 'ALL', 'VO' or 'GROUP'
result = yield self.threadTask(up.setVarPermissions, name,
revokeAccess)
if not result['OK']:
raise WErr.fromSERROR(result)
self.set_status(200)
self.finish()
@asyncGen
def web_loadAppState(self):
up = self.__getUP()
try:
name = self.request.arguments['name'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
result = yield self.threadTask(up.retrieveVar, name)
if not result['OK']:
raise WErr.fromSERROR(result)
data = result['Value']
data, count = DEncode.decode(zlib.decompress(base64.b64decode(data)))
self.finish(data)
@asyncGen
def web_loadUserAppState(self):
up = self.__getUP()
try:
user = self.request.arguments['user'][-1]
group = self.request.arguments['group'][-1]
name = self.request.arguments['name'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
result = yield self.threadTask(up.retrieveVarFromUser, user, group,
name)
if not result['OK']:
raise WErr.fromSERROR(result)
data = result['Value']
data, count = DEncode.decode(zlib.decompress(base64.b64decode(data)))
self.finish(data)
@asyncGen
def web_listAppState(self):
up = self.__getUP()
result = yield self.threadTask(up.retrieveAllVars)
if not result['OK']:
raise WErr.fromSERROR(result)
data = result['Value']
for k in data:
# Unpack data
data[k] = json.loads(
DEncode.decode(zlib.decompress(base64.b64decode(data[k])))[0])
self.finish(data)
@asyncGen
def web_delAppState(self):
up = self.__getUP()
try:
name = self.request.arguments['name'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
result = yield self.threadTask(up.deleteVar, name)
if not result['OK']:
raise WErr.fromSERROR(result)
self.finish()
@asyncGen
def web_listPublicDesktopStates(self):
up = self.__getUP()
result = yield self.threadTask(up.listAvailableVars)
if not result['OK']:
raise WErr.fromSERROR(result)
data = result['Value']
paramNames = ['UserName', 'Group', 'VO', 'desktop']
records = []
for i in data:
records += [dict(zip(paramNames, i))]
sharedDesktops = {}
for i in records:
result = yield self.threadTask(up.getVarPermissions, i['desktop'])
if not result['OK']:
raise WErr.fromSERROR(result)
if result['Value']['ReadAccess'] == 'ALL':
print i['UserName'], i['Group'], i
result = yield self.threadTask(up.retrieveVarFromUser,
i['UserName'], i['Group'],
i['desktop'])
if not result['OK']:
raise WErr.fromSERROR(result)
if i['UserName'] not in sharedDesktops:
sharedDesktops[i['UserName']] = {}
sharedDesktops[i['UserName']][i['desktop']] = json.loads(
DEncode.decode(
zlib.decompress(base64.b64decode(
result['Value'])))[0])
sharedDesktops[i['UserName']]['Metadata'] = i
else:
sharedDesktops[i['UserName']][i['desktop']] = json.loads(
DEncode.decode(
zlib.decompress(base64.b64decode(
result['Value'])))[0])
sharedDesktops[i['UserName']]['Metadata'] = i
self.finish(sharedDesktops)
@asyncGen
def web_makePublicDesktopState(self):
up = UserProfileClient("Web/application/desktop")
try:
name = self.request.arguments['name'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
try:
access = self.request.arguments['access'][-1].upper()
except KeyError as excp:
access = 'ALL'
if access not in ('ALL', 'VO', 'GROUP', 'USER'):
raise WErr(400, "Invalid access")
# TODO: Check access is in either 'ALL', 'VO' or 'GROUP'
result = yield self.threadTask(up.setVarPermissions, name,
{'ReadAccess': access})
if not result['OK']:
raise WErr.fromSERROR(result)
self.set_status(200)
self.finish()
@asyncGen
def web_changeView(self):
up = self.__getUP()
try:
desktopName = self.request.arguments['desktop'][-1]
view = self.request.arguments['view'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
result = yield self.threadTask(up.retrieveVar, desktopName)
if not result['OK']:
raise WErr.fromSERROR(result)
data = result['Value']
oDesktop = json.loads(
DEncode.decode(zlib.decompress(base64.b64decode(data)))[0])
oDesktop[unicode('view')] = unicode(view)
oDesktop = json.dumps(oDesktop)
data = base64.b64encode(zlib.compress(DEncode.encode(oDesktop), 9))
result = yield self.threadTask(up.storeVar, desktopName, data)
if not result['OK']:
raise WErr.fromSERROR(result)
self.set_status(200)
self.finish()
@asyncGen
def web_listPublicStates(self):
session = self.getSessionData()
user = session["user"]["username"]
up = self.__getUP()
retVal = yield self.threadTask(up.getUserProfileNames,
{'PublishAccess': 'ALL'})
if not retVal['OK']:
raise WErr.fromSERROR(retVal)
data = retVal['Value']
if data == None:
raise WErr(404, "There are no public states!")
paramNames = ['user', 'group', 'vo', 'name']
mydesktops = {
'name': 'My Desktops',
'group': '',
'vo': '',
'user': '',
'iconCls': 'my-desktop',
'children': []
}
shareddesktops = {
'name': 'Shared Desktops',
'group': '',
'vo': '',
'user': '',
'expanded': 'true',
'iconCls': 'shared-desktop',
'children': []
}
myapplications = {
'name': 'My Applications',
'group': '',
'vo': '',
'user': '',
'children': []
}
sharedapplications = {
'name': 'Shared Applications',
'group': '',
'vo': '',
'user': '',
'expanded': 'true',
'iconCls': 'shared-desktop',
'children': []
}
desktopsApplications = {
'text':
'.',
'children': [{
'name': 'Desktops',
'group': '',
'vo': '',
'user': '',
'children': [mydesktops, shareddesktops]
}, {
'name': 'Applications',
'group': '',
'vo': '',
'user': '',
'children': [myapplications, sharedapplications]
}]
}
type = ''
for i in data:
application = i.replace('Web/application/', '')
up = UserProfileClient(i)
retVal = up.listAvailableVars()
if not retVal['OK']:
raise WErr.fromSERROR(retVal)
else:
states = retVal['Value']
for state in states:
record = dict(zip(paramNames, state))
record['app'] = application
retVal = yield self.threadTask(up.getVarPermissions,
record['name'])
if not retVal['OK']:
raise WErr.fromSERROR(retVal)
else:
permissions = retVal['Value']
if permissions['PublishAccess'] == 'ALL':
if application == 'desktop':
record['type'] = 'desktop'
record['leaf'] = 'true'
record['iconCls'] = 'core-desktop-icon',
if record['user'] == user:
mydesktops['children'].append(record)
else:
shareddesktops['children'].append(record)
else:
record['type'] = 'application'
record['leaf'] = 'true'
record['iconCls'] = 'core-application-icon'
if record['user'] == user:
myapplications['children'].append(record)
else:
sharedapplications['children'].append(
record)
self.finish(desktopsApplications)
@asyncGen
def web_publishAppState(self):
up = self.__getUP()
try:
name = self.request.arguments['name'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
try:
access = self.request.arguments['access'][-1].upper()
except KeyError as excp:
access = 'ALL'
if access not in ('ALL', 'VO', 'GROUP', 'USER'):
raise WErr(400, "Invalid access")
result = yield self.threadTask(up.setVarPermissions, name, {
'PublishAccess': access,
'ReadAccess': access
})
if not result['OK']:
raise WErr.fromSERROR(result)
self.set_status(200)
self.finish()