def __getProxyStatus(self, secondsOverride=None): from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient proxyManager = ProxyManagerClient() userData = self.getSessionData() group = str(userData["user"]["group"]) if group == "visitor": return {"success":"false", "error":"User is anonymous or is not registered in the system"} userDN = str(userData["user"]["DN"]) defaultSeconds = 24 * 3600 + 60 # 24H + 1min validSeconds = gConfig.getValue("/Registry/DefaultProxyLifeTime", defaultSeconds) gLogger.info("\033[0;31m userHasProxy(%s, %s, %s) \033[0m" % (userDN, group, validSeconds)) result = proxyManager.userHasProxy(userDN, group, validSeconds) if result["OK"]: if result["Value"]: return {"success":"true", "result":"true"} else: return {"success":"true", "result":"false"} else: return {"success":"false", "error":"false"} gLogger.info("\033[0;31m PROXY: \033[0m", result)
def __getProxyStatus(self,secondsOverride = None): from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient proxyManager = ProxyManagerClient() group = str(credentials.getSelectedGroup()) if group == "visitor": return {"success":"false","error":"User is anonymous or is not registered in the system"} userDN = str(credentials.getUserDN()) if secondsOverride and str(secondsOverride).isdigit(): validSeconds = int(secondsOverride) else: defaultSeconds = 24 * 3600 + 60 # 24H + 1min validSeconds = gConfig.getValue("/Registry/DefaultProxyLifeTime",defaultSeconds) gLogger.info("\033[0;31m userHasProxy(%s, %s, %s) \033[0m" % (userDN,group,validSeconds)) result = proxyManager.userHasProxy(userDN,group,validSeconds) if result["OK"]: if result["Value"]: c.result = {"success":"true","result":"true"} else: c.result = {"success":"true","result":"false"} else: c.result = {"success":"false","error":"false"} gLogger.info("\033[0;31m PROXY: \033[0m",result) return c.result
def __init__(self): self.db = AuthDB() # place to store session information self.log = sLog self.idps = IdProviderFactory() self.proxyCli = ProxyManagerClient() # take care about proxies self.tokenCli = TokenManagerClient() # take care about tokens # The authorization server has its own settings, but they are standardized self.metadata = collectMetadata() self.metadata.validate() # Initialize AuthorizationServer _AuthorizationServer.__init__(self, scopes_supported=self.metadata["scopes_supported"]) # authlib requires the following methods: # The following `save_token` method is called when requesting a new access token to save it after it is generated. # Let's skip this step, because getting tokens and saving them if necessary has already taken place in `generate_token` method. self.save_token = lambda x, y: None # Framework integration can re-implement this method to support signal system. # But in this implementation, this system is not used. self.send_signal = lambda *x, **y: None # The main method that will return an access token to the user (this can be a proxy) self.generate_token = self.generateProxyOrToken # Register configured grants self.register_grant(RefreshTokenGrant) # Enable refreshing tokens # Enable device code flow self.register_grant(DeviceCodeGrant) self.register_endpoint(DeviceAuthorizationEndpoint) self.register_endpoint(RevocationEndpoint) # Enable revokation tokens self.register_grant(AuthorizationCodeGrant, [CodeChallenge(required=True)]) # Enable authorization code flow
def __init__(self, **kwargs): GridBackend.__init__(self, catalogue_prefix='', **kwargs) from DIRAC.Core.Base import Script Script.initialize() from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient self.pm = ProxyManagerClient() proxy = self.pm.getUserProxiesInfo() if not proxy['OK']: raise BackendException("Proxy error.") from DIRAC.Interfaces.API.Dirac import Dirac self.dirac = Dirac() from DIRAC.Resources.Catalog.FileCatalog import FileCatalog self.fc = FileCatalog() from DIRAC.DataManagementSystem.Client.DataManager import DataManager self.dm = DataManager() self._xattr_cmd = sh.Command('gfal-xattr').bake(_tty_out=False) self._replica_checksum_cmd = sh.Command('gfal-sum').bake(_tty_out=False) self._bringonline_cmd = sh.Command('gfal-legacy-bringonline').bake(_tty_out=False) self._cp_cmd = sh.Command('gfal-copy').bake(_tty_out=False) self._ls_se_cmd = sh.Command('gfal-ls').bake(color='never', _tty_out=False) self._move_cmd = sh.Command('gfal-rename').bake(_tty_out=False) self._mkdir_cmd = sh.Command('gfal-mkdir').bake(_tty_out=False) self._replicate_cmd = sh.Command('dirac-dms-replicate-lfn').bake(_tty_out=False) self._add_cmd = sh.Command('dirac-dms-add-file').bake(_tty_out=False)
def __getProxyStatus( self, validSeconds = 0 ): from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient proxyManager = ProxyManagerClient() group = str(credentials.getSelectedGroup()) if group == "visitor": return {"success":"false","error":"User not registered"} userDN = str(credentials.getUserDN()) gLogger.info("\033[0;31m userHasProxy(%s, %s, %s) \033[0m" % (userDN,group,validSeconds)) result = proxyManager.userHasProxy(userDN,group,validSeconds) if result["OK"]: if result["Value"]: c.result = {"success":"true","result":"true"} else: c.result = {"success":"true","result":"false"} else: c.result = {"success":"false","error":"false"} gLogger.info("\033[0;31m PROXY: \033[0m",result) return c.result
def __getProxyStatus(self, validSeconds=0): from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient proxyManager = ProxyManagerClient() group = str(credentials.getSelectedGroup()) if group == "visitor": return {"success": "false", "error": "User not registered"} userDN = str(credentials.getUserDN()) gLogger.info("\033[0;31m userHasProxy(%s, %s, %s) \033[0m" % (userDN, group, validSeconds)) result = proxyManager.userHasProxy(userDN, group, validSeconds) if result["OK"]: if result["Value"]: c.result = {"success": "true", "result": "true"} else: c.result = {"success": "true", "result": "false"} else: c.result = {"success": "false", "error": "false"} gLogger.info("\033[0;31m PROXY: \033[0m", result) return c.result
def _renewCloudProxy(self): """Takes short lived proxy from the site director and promotes it to a long lived proxy keeping the DIRAC group. :returns: True on success, false otherwise. :rtype: bool """ if not self._cloudDN or not self._cloudGroup: self.log.error( "Could not renew cloud proxy, DN and/or Group not set.") return False proxyLifetime = int( self.ceParameters.get("Context_ProxyLifetime", DEF_PROXYLIFETIME)) # only renew proxy if lifetime is less than configured lifetime # self.valid is a datetime if self.valid - datetime.datetime.utcnow( ) > proxyLifetime * datetime.timedelta(seconds=1): return True proxyLifetime += DEF_PROXYGRACE proxyManager = ProxyManagerClient() self.log.info("Downloading proxy with cloudDN and cloudGroup: %s, %s" % (self._cloudDN, self._cloudGroup)) res = proxyManager.downloadProxy(self._cloudDN, self._cloudGroup, limited=True, requiredTimeLeft=proxyLifetime) if not res["OK"]: self.log.error("Could not download proxy", res["Message"]) return False resdump = res["Value"].dumpAllToString() if not resdump["OK"]: self.log.error("Failed to dump proxy to string", resdump["Message"]) return False self.proxy = resdump["Value"] self.valid = datetime.datetime.utcnow( ) + proxyLifetime * datetime.timedelta(seconds=1) return True
'issuer']: print "You can only query info about yourself!" sys.exit(1) result = CS.getDNForUsername(userName) if not result['OK']: print "Oops %s" % result['Message'] dnList = result['Value'] if not dnList: print "User %s has no DN defined!" % userName sys.exit(1) userDNs = dnList else: userDNs = [userName] print "Checking for DNs %s" % " | ".join(userDNs) pmc = ProxyManagerClient() result = pmc.getDBContents({'UserDN': userDNs}) if not result['OK']: print "Could not retrieve the proxy list: %s" % result['Value'] sys.exit(1) data = result['Value'] colLengths = [] for pN in data['ParameterNames']: colLengths.append(len(pN)) for row in data['Records']: for i in range(len(row)): colLengths[i] = max(colLengths[i], len(str(row[i]))) lines = [""] for i in range(len(data['ParameterNames'])):
def main(): global userName Script.registerSwitch("u:", "user="******"User to query (by default oneself)", setUser) Script.parseCommandLine() result = getProxyInfo() if not result["OK"]: gLogger.notice("Do you have a valid proxy?") gLogger.notice(result["Message"]) sys.exit(1) proxyProps = result["Value"] userName = userName or proxyProps.get("username") if not userName: gLogger.notice("Your proxy don`t have username extension") sys.exit(1) if userName in Registry.getAllUsers(): if Properties.PROXY_MANAGEMENT not in proxyProps["groupProperties"]: if userName != proxyProps["username"] and userName != proxyProps["issuer"]: gLogger.notice("You can only query info about yourself!") sys.exit(1) result = Registry.getDNForUsername(userName) if not result["OK"]: gLogger.notice("Oops %s" % result["Message"]) dnList = result["Value"] if not dnList: gLogger.notice("User %s has no DN defined!" % userName) sys.exit(1) userDNs = dnList else: userDNs = [userName] gLogger.notice("Checking for DNs %s" % " | ".join(userDNs)) pmc = ProxyManagerClient() result = pmc.getDBContents({"UserDN": userDNs}) if not result["OK"]: gLogger.notice("Could not retrieve the proxy list: %s" % result["Value"]) sys.exit(1) data = result["Value"] colLengths = [] for pN in data["ParameterNames"]: colLengths.append(len(pN)) for row in data["Records"]: for i in range(len(row)): colLengths[i] = max(colLengths[i], len(str(row[i]))) lines = [""] for i in range(len(data["ParameterNames"])): pN = data["ParameterNames"][i] lines[0] += "| %s " % pN.ljust(colLengths[i]) lines[0] += "|" tL = len(lines[0]) lines.insert(0, "-" * tL) lines.append("-" * tL) for row in data["Records"]: nL = "" for i in range(len(row)): nL += "| %s " % str(row[i]).ljust(colLengths[i]) nL += "|" lines.append(nL) lines.append("-" * tL) gLogger.notice("\n".join(lines))
print "You can only query info about yourself!" sys.exit( 1 ) result = CS.getDNForUsername( userName ) if not result[ 'OK' ]: print "Oops %s" % result[ 'Message' ] dnList = result[ 'Value' ] if not dnList: print "User %s has no DN defined!" % userName sys.exit( 1 ) userDNs = dnList else: userDNs = [ userName ] print "Checking for DNs %s" % " | ".join( userDNs ) pmc = ProxyManagerClient() result = pmc.getDBContents( { 'UserDN' : userDNs } ) if not result[ 'OK' ]: print "Could not retrieve the proxy list: %s" % result[ 'Value' ] sys.exit( 1 ) data = result[ 'Value' ] colLengths = [] for pN in data[ 'ParameterNames' ]: colLengths.append( len( pN ) ) for row in data[ 'Records' ] : for i in range( len( row ) ): colLengths[ i ] = max( colLengths[i], len( str( row[i] ) ) ) lines = [""] for i in range( len( data[ 'ParameterNames' ] ) ):
def web_proxy(self): """ Proxy management endpoint, use: GET /proxy?<options> -- retrieve personal proxy * options: * voms - to get VOMSproxy(optional) * lifetime - requested proxy live time(optional) GET /proxy/<user>/<group>?<options> -- retrieve proxy * user - user name * group - group name * options: * voms - to get VOMSproxy(optional) * lifetime - requested proxy live time(optional) GET /proxy/metadata?<options> -- retrieve proxy metadata.. * options: :return: json """ voms = self.args.get('voms') proxyLifeTime = 3600 * 12 if re.match('[0-9]+', self.args.get('lifetime') or ''): proxyLifeTime = int(self.args.get('lifetime')) optns = self.overpath.strip('/').split('/') # GET if self.request.method == 'GET': # Return content of Proxy DB if 'metadata' in optns: pass # Return personal proxy elif not self.overpath: result = yield self.threadTask(ProxyManagerClient().downloadPersonalProxy, self.getUserName(), self.getUserGroup(), requiredTimeLeft=proxyLifeTime, voms=voms) if not result['OK']: raise WErr(500, result['Message']) self.log.notice('Proxy was created.') result = result['Value'].dumpAllToString() if not result['OK']: raise WErr(500, result['Message']) self.finishJEncode(result['Value']) # Return proxy elif len(optns) == 2: user = optns[0] group = optns[1] # Get proxy to string result = getDNForUsernameInGroup(user, group) if not result['OK'] or not result.get('Value'): raise WErr(500, '%s@%s has no registred DN: %s' % (user, group, result.get('Message') or "")) if voms: result = yield self.threadTask(ProxyManagerClient().downloadVOMSProxy, user, group, requiredTimeLeft=proxyLifeTime) else: result = yield self.threadTask(ProxyManagerClient().downloadProxy, user, group, requiredTimeLeft=proxyLifeTime) if not result['OK']: raise WErr(500, result['Message']) self.log.notice('Proxy was created.') result = result['Value'].dumpAllToString() if not result['OK']: raise WErr(500, result['Message']) self.finishJEncode(result['Value']) else: raise WErr(404, "Wrone way")
class AuthServer(_AuthorizationServer): """Implementation of the :class:`authlib.oauth2.rfc6749.AuthorizationServer`. This framework has been changed and simplified to be used for DIRAC purposes, namely authorization on the third party side and saving the received extended long-term access tokens on the DIRAC side with the possibility of their future use on behalf of the user without his participation. The idea is that DIRAC itself is not an identity provider and relies on third-party resources such as EGI Checkin or WLCG IAM. Initialize:: server = AuthServer() """ LOCATION = None def __init__(self): self.db = AuthDB() # place to store session information self.log = sLog self.idps = IdProviderFactory() self.proxyCli = ProxyManagerClient() # take care about proxies self.tokenCli = TokenManagerClient() # take care about tokens # The authorization server has its own settings, but they are standardized self.metadata = collectMetadata() self.metadata.validate() # Initialize AuthorizationServer _AuthorizationServer.__init__(self, scopes_supported=self.metadata["scopes_supported"]) # authlib requires the following methods: # The following `save_token` method is called when requesting a new access token to save it after it is generated. # Let's skip this step, because getting tokens and saving them if necessary has already taken place in `generate_token` method. self.save_token = lambda x, y: None # Framework integration can re-implement this method to support signal system. # But in this implementation, this system is not used. self.send_signal = lambda *x, **y: None # The main method that will return an access token to the user (this can be a proxy) self.generate_token = self.generateProxyOrToken # Register configured grants self.register_grant(RefreshTokenGrant) # Enable refreshing tokens # Enable device code flow self.register_grant(DeviceCodeGrant) self.register_endpoint(DeviceAuthorizationEndpoint) self.register_endpoint(RevocationEndpoint) # Enable revokation tokens self.register_grant(AuthorizationCodeGrant, [CodeChallenge(required=True)]) # Enable authorization code flow # pylint: disable=method-hidden def query_client(self, client_id): """Search authorization client. :param str clientID: client ID :return: client as object or None """ gLogger.debug("Try to query %s client" % client_id) clients = getDIRACClients() for cli in clients: if client_id == clients[cli]["client_id"]: gLogger.debug("Found %s client:\n" % cli, pprint.pformat(clients[cli])) # Authorization successful return Client(clients[cli]) # Authorization failed, client not found return None def _getScope(self, scope, param): """Get parameter scope :param str scope: scope :param str param: parameter scope :return: str or None """ try: return [s.split(":")[1] for s in scope_to_list(scope) if s.startswith("%s:" % param) and s.split(":")[1]][0] except Exception: return None def generateProxyOrToken( self, client, grant_type, user=None, scope=None, expires_in=None, include_refresh_token=True ): """Generate proxy or tokens after authorization :param client: instance of the IdP client :param grant_type: authorization grant type (unused) :param str user: user identificator :param str scope: requested scope :param expires_in: when the token should expire (unused) :param bool include_refresh_token: to include refresh token (unused) :return: dict or str -- will return tokens as dict or proxy as string """ # Read requested scopes group = self._getScope(scope, "g") lifetime = self._getScope(scope, "lifetime") # Found provider name for group provider = getIdPForGroup(group) # Search DIRAC username by user ID result = getUsernameForDN(wrapIDAsDN(user)) if not result["OK"]: raise OAuth2Error(result["Message"]) userName = result["Value"] # User request a proxy if "proxy" in scope_to_list(scope): # Try to return user proxy if proxy scope present in the authorization request if not isDownloadProxyAllowed(): raise OAuth2Error("You can't get proxy, configuration(allowProxyDownload) not allow to do that.") sLog.debug( "Try to query %s@%s proxy%s" % (user, group, ("with lifetime:%s" % lifetime) if lifetime else "") ) # Get user DNs result = getDNForUsername(userName) if not result["OK"]: raise OAuth2Error(result["Message"]) userDNs = result["Value"] err = [] # Try every DN to generate a proxy for dn in userDNs: sLog.debug("Try to get proxy for %s" % dn) params = {} if lifetime: params["requiredTimeLeft"] = int(lifetime) # if the configuration describes adding a VOMS extension, we will do so if getGroupOption(group, "AutoAddVOMS", False): result = self.proxyCli.downloadVOMSProxy(dn, group, **params) else: # otherwise we will return the usual proxy result = self.proxyCli.downloadProxy(dn, group, **params) if not result["OK"]: err.append(result["Message"]) else: sLog.info("Proxy was created.") result = result["Value"].dumpAllToString() if not result["OK"]: raise OAuth2Error(result["Message"]) # Proxy generated return { "proxy": result["Value"].decode() if isinstance(result["Value"], bytes) else result["Value"] } # Proxy cannot be generated or not found raise OAuth2Error("; ".join(err)) # User request a tokens else: # Ask TokenManager to generate new tokens for user result = self.tokenCli.getToken(userName, group) if not result["OK"]: raise OAuth2Error(result["Message"]) token = result["Value"] # Wrap the refresh token and register it to protect against reuse result = self.registerRefreshToken( dict(sub=user, scope=scope, provider=provider, azp=client.get_client_id()), token ) if not result["OK"]: raise OAuth2Error(result["Message"]) # Return tokens as dictionary return result["Value"] def __signToken(self, payload): """Sign token :param dict payload: payload :return: S_OK(str)/S_ERROR() """ result = self.db.getPrivateKey() if not result["OK"]: return result key = result["Value"] try: return S_OK(jwt.encode(dict(alg="RS256", kid=key.thumbprint()), payload, key).decode("utf-8")) except Exception as e: sLog.exception(e) return S_ERROR(repr(e)) def readToken(self, token): """Decode self token :param str token: token to decode :return: S_OK(dict)/S_ERROR() """ result = self.db.getKeySet() if not result["OK"]: return result try: return S_OK(jwt.decode(token, JsonWebKey.import_key_set(result["Value"].as_dict()))) except Exception as e: sLog.exception(e) return S_ERROR(repr(e)) def registerRefreshToken(self, payload, token): """Register refresh token to protect it from reuse :param dict payload: payload :param dict token: token as a dictionary :return: S_OK(dict)S_ERROR() """ result = self.db.storeRefreshToken(token, payload.get("jti")) if result["OK"]: payload.update(result["Value"]) result = self.__signToken(payload) if not result["OK"]: if token.get("refresh_token"): prov = self.idps.getIdProvider(payload["provider"]) if prov["OK"]: prov["Value"].revokeToken(token["refresh_token"]) prov["Value"].revokeToken(token["access_token"], "access_token") return result token["refresh_token"] = result["Value"] return S_OK(token) def getIdPAuthorization(self, provider, request): """Submit subsession to authorize with chosen provider and return dict with authorization url and session number :param str provider: provider name :param object request: main session request :return: S_OK(response)/S_ERROR() -- dictionary contain response generated by `handle_response` """ result = self.idps.getIdProvider(provider) if not result["OK"]: raise Exception(result["Message"]) idpObj = result["Value"] authURL, state, session = idpObj.submitNewSession() session["state"] = state session["Provider"] = provider session["firstRequest"] = request if isinstance(request, dict) else request.toDict() sLog.verbose("Redirect to", authURL) return self.handle_response(302, {}, [("Location", authURL)], session) def parseIdPAuthorizationResponse(self, response, session): """Fill session by user profile, tokens, comment, OIDC authorize status, etc. Prepare dict with user parameters, if DN is absent there try to get it. Create new or modify existing DIRAC user and store the session :param dict response: authorization response :param str session: session :return: S_OK(dict)/S_ERROR() """ providerName = session.pop("Provider") sLog.debug("Try to parse authentification response from %s:\n" % providerName, pprint.pformat(response)) # Parse response result = self.idps.getIdProvider(providerName) if not result["OK"]: return result idpObj = result["Value"] result = idpObj.parseAuthResponse(response, session) if not result["OK"]: return result # FINISHING with IdP # As a result of authentication we will receive user credential dictionary credDict, payload = result["Value"] sLog.debug("Read profile:", pprint.pformat(credDict)) # Is ID registred? result = getUsernameForDN(credDict["DN"]) if not result["OK"]: comment = f"ID {credDict['ID']} is not registred in DIRAC. " payload.update(idpObj.getUserProfile().get("Value", {})) result = self.__registerNewUser(providerName, payload) if result["OK"]: comment += "Administrators have been notified about you." else: comment += "Please, contact the DIRAC administrators." # Notify user about problem html = getHTML("unregistered user!", info=comment, theme="warning") return S_ERROR(html) credDict["username"] = result["Value"] # Update token for user. This token will be stored separately in the database and # updated from time to time. This token will never be transmitted, # it will be used to make exchange token requests. result = self.tokenCli.updateToken(idpObj.token, credDict["ID"], idpObj.name) return S_OK(credDict) if result["OK"] else result def create_oauth2_request(self, request, method_cls=OAuth2Request, use_json=False): """Parse request. Rewrite authlib method.""" self.log.debug("Create OAuth2 request", "with json" if use_json else "") return createOAuth2Request(request, method_cls, use_json) def create_json_request(self, request): """Parse request. Rewrite authlib method.""" return self.create_oauth2_request(request, HttpRequest, True) def validate_requested_scope(self, scope, state=None): """See :func:`authlib.oauth2.rfc6749.authorization_server.validate_requested_scope`""" # We also consider parametric scope containing ":" charter extended_scope = list_to_scope( [re.sub(r":.*$", ":", s) for s in scope_to_list((scope or "").replace("+", " "))] ) super(AuthServer, self).validate_requested_scope(extended_scope, state) def handle_response(self, status_code=None, payload=None, headers=None, newSession=None, delSession=None): """Handle response :param int status_code: http status code :param payload: response payload :param list headers: headers :param dict newSession: session data to store :return: TornadoResponse() """ resp = TornadoResponse(payload, status_code) if not isinstance(payload, dict): sLog.debug( f"Handle authorization response with {status_code} status code:", "HTML page" if payload.startswith("<!DOCTYPE html>") else payload, ) elif "error" in payload: resp.clear_cookie("auth_session") # pylint: disable=no-member sLog.error(f"{payload['error']}: {payload.get('error_description', 'unknown')}") if headers: sLog.debug("Headers:", headers) for key, value in headers: resp.set_header(key, value) # pylint: disable=no-member if newSession: sLog.debug("Initialize new session:", newSession) # pylint: disable=no-member resp.set_secure_cookie("auth_session", json.dumps(newSession), secure=True, httponly=True) if delSession: resp.clear_cookie("auth_session") # pylint: disable=no-member return resp def create_authorization_response(self, response, username): """Rewrite original Authlib method `authlib.authlib.oauth2.rfc6749.authorization_server.create_authorization_response` to catch errors and remove authorization session. :return: TornadoResponse object """ try: response = super().create_authorization_response(response, username) response.clear_cookie("auth_session") return response except Exception as e: sLog.exception(e) return self.handle_response( payload=getHTML("server error", theme="error", body="traceback", info=repr(e)), delSession=True ) def validate_consent_request(self, request, provider=None): """Validate current HTTP request for authorization page. This page is designed for resource owner to grant or deny the authorization:: :param object request: tornado request :param provider: provider :return: response generated by `handle_response` or S_ERROR or html """ try: request = self.create_oauth2_request(request) # Check Identity Provider req = self.validateIdentityProvider(request, provider) # If return HTML page with IdP selector if isinstance(req, str): return req sLog.info("Validate consent request for ", req.state) grant = self.get_authorization_grant(req) sLog.debug("Use grant:", grant.GRANT_TYPE) grant.validate_consent_request() if not hasattr(grant, "prompt"): grant.prompt = None # Submit second auth flow through IdP return self.getIdPAuthorization(req.provider, req) except OAuth2Error as error: self.db.removeSession(request.sessionID) code, body, _ = error(None) return self.handle_response( payload=getHTML(repr(error), state=code, body=body, info="OAuth2 error."), delSession=True ) except Exception as e: self.db.removeSession(request.sessionID) sLog.exception(e) return self.handle_response( payload=getHTML("server error", theme="error", body="traceback", info=repr(e)), delSession=True ) def validateIdentityProvider(self, request, provider): """Check if identity provider registred in DIRAC :param object request: request :param str provider: provider name :return: OAuth2Request object or HTML -- new request with provider name or provider selector """ if provider: request.provider = provider # Find identity provider for group groupProvider = getIdPForGroup(request.group) if request.groups else None # If requested access token for group that is not registred in any identity provider # or the requested provider does not match the group return error if request.group and not groupProvider and "proxy" not in request.scope: raise Exception("The %s group belongs to the VO that is not tied to any Identity Provider." % request.group) sLog.debug("Check if %s identity provider registred in DIRAC.." % request.provider) # Research supported IdPs result = getProvidersForInstance("Id") if not result["OK"]: raise Exception(result["Message"]) idPs = result["Value"] if not idPs: raise Exception("No identity providers found.") if request.provider: if request.provider not in idPs: raise Exception("%s identity provider is not registered." % request.provider) elif groupProvider and request.provider != groupProvider: raise Exception( 'The %s group Identity Provider is "%s" and not "%s".' % (request.group, groupProvider, request.provider) ) return request # If no identity provider is specified, it must be assigned if groupProvider: request.provider = groupProvider return request # If only one identity provider is registered, then choose it if len(idPs) == 1: request.provider = idPs[0] return request # Choose IdP HTML interface with dom.div(cls="row m-5 justify-content-md-center") as tag: for idP in idPs: result = getProviderInfo(idP) if result["OK"]: logo = result["Value"].get("logoURL") with dom.div(cls="col-md-6 p-2").add(dom.div(cls="card shadow-lg h-100 border-0")): with dom.div(cls="row m-2 justify-content-md-center align-items-center h-100"): with dom.div(cls="col-auto"): dom.h2(idP) dom.a( href="%s/authorization/%s?%s" % (self.LOCATION, idP, request.query), cls="stretched-link", ) if logo: dom.div(dom.img(src=logo, cls="card-img"), cls="col-auto") # Render into HTML return getHTML( "Identity Provider selection..", body=tag, icon="fingerprint", info="Dirac itself is not an Identity Provider. " "You will need to select one to continue.", ) def __registerNewUser(self, provider, payload): """Register new user :param str provider: provider :param dict payload: user information dictionary :return: S_OK()/S_ERROR() """ from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient username = payload["sub"] mail = {} mail["subject"] = "[DIRAC AS] User %s to be added." % username mail["body"] = "User %s was authenticated by %s." % (username, provider) mail["body"] += "\n\nNew user to be added with the following information:\n%s" % pprint.pformat(payload) mail["body"] += "\n\nPlease, add '%s' to /Register/Users/<username>/DN option.\n" % wrapIDAsDN(username) mail["body"] += "\n\n------" mail["body"] += "\n This is a notification from the DIRAC authorization service, please do not reply.\n" result = S_OK() for addresses in getEmailsForGroup("dirac_admin"): result = NotificationClient().sendMail(addresses, mail["subject"], mail["body"], localAttempt=False) if not result["OK"]: sLog.error(result["Message"]) if result["OK"]: sLog.info(result["Value"], "administrators have been notified about a new user.") return result
class DIRACBackend(GridBackend): """Grid backend using the GFAL command line tools `gfal-*`.""" def __init__(self, **kwargs): GridBackend.__init__(self, catalogue_prefix='', **kwargs) from DIRAC.Core.Base import Script Script.initialize() from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient self.pm = ProxyManagerClient() proxy = self.pm.getUserProxiesInfo() if not proxy['OK']: raise BackendException("Proxy error.") from DIRAC.Interfaces.API.Dirac import Dirac self.dirac = Dirac() from DIRAC.Resources.Catalog.FileCatalog import FileCatalog self.fc = FileCatalog() from DIRAC.DataManagementSystem.Client.DataManager import DataManager self.dm = DataManager() self._xattr_cmd = sh.Command('gfal-xattr').bake(_tty_out=False) self._replica_checksum_cmd = sh.Command('gfal-sum').bake(_tty_out=False) self._bringonline_cmd = sh.Command('gfal-legacy-bringonline').bake(_tty_out=False) self._cp_cmd = sh.Command('gfal-copy').bake(_tty_out=False) self._ls_se_cmd = sh.Command('gfal-ls').bake(color='never', _tty_out=False) self._move_cmd = sh.Command('gfal-rename').bake(_tty_out=False) self._mkdir_cmd = sh.Command('gfal-mkdir').bake(_tty_out=False) self._replicate_cmd = sh.Command('dirac-dms-replicate-lfn').bake(_tty_out=False) self._add_cmd = sh.Command('dirac-dms-add-file').bake(_tty_out=False) @staticmethod def _check_return_value(ret): if not ret['OK']: raise BackendException("Failed: %s", ret['Message']) for path, error in ret['Value']['Failed'].items(): if ('No such' in error) or ('Directory does not' in error): raise DoesNotExistException("No such file or directory.") else: raise BackendException(error) def _is_dir(self, lurl): isdir = self.fc.isDirectory(lurl) self._check_return_value(isdir) return isdir['Value']['Successful'][lurl] def _is_file(self, lurl): isfile = self.fc.isFile(lurl) self._check_return_value(isfile) return isfile['Value']['Successful'][lurl] def _get_dir_entry(self, lurl, infodict=None): """Take a lurl and return a DirEntry.""" # If no dctionary with the information is specified, get it from the catalogue try: md = infodict['MetaData'] except TypeError: md = self.fc.getFileMetadata(lurl) if not md['OK']: raise BackendException("Failed to list path '%s': %s", lurl, md['Message']) for path, error in md['Value']['Failed'].items(): if 'No such file' in error: # File does not exist, maybe a directory? md = self.fc.getDirectoryMetadata(lurl) for path, error in md['Value']['Failed'].items(): raise DoesNotExistException("No such file or directory.") else: raise BackendException(md['Value']['Failed'][lurl]) md = md['Value']['Successful'][lurl] return DirEntry(posixpath.basename(lurl), mode=oct(md.get('Mode', -1)), links=md.get('links', -1), gid=md['OwnerGroup'], uid=md['Owner'], size=md.get('Size', -1), modified=str(md.get('ModificationDate', '?'))) def _iter_directory(self, lurl): """Iterate over entries in a directory.""" ret = self.fc.listDirectory(lurl) if not ret['OK']: raise BackendException("Failed to list path '%s': %s", lurl, ret['Message']) for path, error in ret['Value']['Failed'].items(): if 'Directory does not' in error: # Dir does not exist, maybe a File? if self.fc.isFile(lurl): lst = [(lurl, None)] break else: raise DoesNotExistException("No such file or Directory.") else: raise BackendException(ret['Value']['Failed'][lurl]) else: # Sort items by keys, i.e. paths lst = sorted(ret['Value']['Successful'][lurl]['Files'].items() + ret['Value']['Successful'][lurl]['SubDirs'].items()) for item in lst: yield item # = path, dict def _ls(self, lurl, **kwargs): # Translate keyword arguments d = kwargs.pop('directory', False) if d: # Just the requested entry itself yield self._get_dir_entry(lurl) return for path, info in self._iter_directory(lurl): yield self._get_dir_entry(path, info) def _ls_se(self, surl, **kwargs): # Translate keyword arguments d = kwargs.pop('directory', False) args = [] if -d: args.append('-d') args.append('-l') args.append(surl) try: output = self._ls_se_cmd(*args, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or Directory.") else: raise BackendException(e.stderr) for line in output: fields = line.split() mode, links, gid, uid, size = fields[:5] name = fields[-1] modified = ' '.join(fields[5:-1]) yield DirEntry(name, mode=mode, links=int(links), gid=gid, uid=uid, size=int(size), modified=modified) def _replicas(self, lurl, **kwargs): # Check the lurl actually exists self._ls(lurl, directory=True) rep = self.dirac.getReplicas(lurl) self._check_return_value(rep) rep = rep['Value']['Successful'][lurl] return rep.values() def _exists(self, surl, **kwargs): try: ret = self._ls_se_cmd(surl, '-d', '-l', **kwargs).strip() except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: return False else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) else: return ret[0] != 'd' # Return `False` for directories def _register(self, surl, lurl, verbose=False, **kwargs): # Register an existing physical copy in the file catalogue se = storage.get_SE(surl).name # See if file already exists in DFC ret = self.fc.getFileMetadata(lurl) try: self._check_return_value(ret) except DoesNotExistException: # Add new file size = next(self._ls_se(surl, directory=True)).size checksum = self.checksum(surl) guid = str(uuid.uuid4()) # The guid does not seem to be important. Make it unique if possible. ret = self.dm.registerFile((lurl, surl, size, se, guid, checksum)) else: # Add new replica ret = self.dm.registerReplica((lurl, surl, se)) self._check_return_value(ret) if verbose: print_("Successfully registered replica %s of %s from %s."%(surl, lurl, se)) return True def _deregister(self, surl, lurl, verbose=False, **kwargs): # DIRAC only needs to know the SE name to deregister a replica se = storage.get_SE(surl).name ret = self.dm.removeReplicaFromCatalog(se, [lurl]) self._check_return_value(ret) if verbose: print_("Successfully deregistered replica of %s from %s."%(lurl, se)) return True def _state(self, surl, **kwargs): try: state = self._xattr_cmd(surl, 'user.status', **kwargs).strip() except sh.ErrorReturnCode as e: if "No such file" in e.stderr: raise DoesNotExistException("No such file or Directory.") state = '?' except sh.SignalException_SIGSEGV: state = '?' return state def _checksum(self, surl, **kwargs): try: checksum = self._replica_checksum_cmd(surl, 'ADLER32', **kwargs).split()[1] except sh.ErrorReturnCode: checksum = '?' except sh.SignalException_SIGSEGV: checksum = '?' except IndexError: checksum = '?' return checksum def _bringonline(self, surl, timeout, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None # gfal does not notice when files come online, it seems # Just send a single short request, then check regularly if verbose: out = sys.stdout else: out = None end = time.time() + timeout try: self._bringonline_cmd('-t', 10, surl, _out=out, **kwargs) except sh.ErrorReturnCode as e: # The command fails if the file is not online # To be expected after 10 seconds if "No such file" in e.stderr: # Except when the file does not actually exist on the tape storage raise DoesNotExistException("No such file or Directory.") wait = 5 while(True): if verbose: print_("Checking replica state...") if self.is_online(surl): if verbose: print_("Replica brought online.") return True time_left = end - time.time() if time_left <= 0: if verbose: print_("Could not bring replica online.") return False wait *= 2 if time_left < wait: wait = time_left if verbose: print_("Timeout remaining: %d s"%(time_left)) print_("Checking again in: %d s"%(wait)) time.sleep(wait) def _replicate(self, source_surl, destination_surl, lurl, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None source = storage.get_SE(source_surl).name destination = storage.get_SE(destination_surl).name try: self._replicate_cmd(lurl, destination, source, _out=out, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or directory.") else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) return True def _get(self, surl, localpath, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None try: self._cp_cmd('-f', '--checksum', 'ADLER32', surl, localpath, _out=out, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or directory.") else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) return os.path.isfile(localpath) def _put(self, localpath, surl, lurl, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None se = storage.get_SE(surl).name try: self._add_cmd(lurl, localpath, se, _out=out, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or directory.") else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) return True def _remove(self, surl, lurl, last=False, verbose=False, **kwargs): se = storage.get_SE(surl).name if last: # Delete lfn if verbose: print_("Removing all replicas of %s."%(lurl,)) ret = self.dm.removeFile([lurl]) else: if verbose: print_("Removing replica of %s from %s."%(lurl, se)) ret = self.dm.removeReplica(se, [lurl]) if not ret['OK']: raise BackendException('Failed: %s'%(ret['Message'])) for lurl, error in ret['Value']['Failed'].items(): if 'No such file' in error: raise DoesNotExistException("No such file or directory.") else: raise BackendException(error) return True def _rmdir(self, lurl, verbose=False): """Remove the an empty directory from the catalogue.""" rep = self.fc.removeDirectory(lurl) self._check_return_value(rep) return True def _move_replica(self, surl, new_surl, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None try: folder = posixpath.dirname(new_surl) self._mkdir_cmd(folder, '-p', _out=out, **kwargs) self._move_cmd(surl, new_surl, _out=out, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or directory.") else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) return True