def getNonFinishedOperations(self, limit=20, operationAssignmentTag="Assigned", **kwargs): """Get all the FTS3Operations that have files in New or Failed state (reminder: Failed is NOT terminal for files. Failed is when fts failed, but we can retry) :param limit: max number of jobs to retrieve :return: json list of FTS3Operation """ res = self._getRPC(**kwargs).getNonFinishedOperations( limit, operationAssignmentTag) if not res["OK"]: return res operationsJSON = res["Value"] try: operations, _size = decode(operationsJSON) return S_OK(operations) except Exception as e: return S_ERROR( 0, "Exception when decoding the non finished operations json %s" % e)
def _gatherPeerCredentials(self): """ Load client certchain in DIRAC and extract informations. The dictionary returned is designed to work with the AuthManager, already written for DISET and re-used for HTTPS. :returns: a dict containing the return of :py:meth:`DIRAC.Core.Security.X509Chain.X509Chain.getCredentials` (not a DIRAC structure !) """ chainAsText = self.request.get_ssl_certificate().as_pem() peerChain = X509Chain() # Here we read all certificate chain cert_chain = self.request.get_ssl_certificate_chain() for cert in cert_chain: chainAsText += cert.as_pem() peerChain.loadChainFromString(chainAsText) # Retrieve the credentials res = peerChain.getCredentials(withRegistryInfo=False) if not res['OK']: raise Exception(res['Message']) credDict = res['Value'] # We check if client sends extra credentials... if "extraCredentials" in self.request.arguments: extraCred = self.get_argument("extraCredentials") if extraCred: credDict['extraCredentials'] = decode(extraCred)[0] return credDict
def __executeMethod(self): """ Execute the method called, this method is ran in an executor We have several try except to catch the different problem which can occur - First, the method does not exist => Attribute error, return an error to client - second, anything happend during execution => General Exception, send error to client .. warning:: This method is called in an executor, and so cannot use methods like self.write See https://www.tornadoweb.org/en/branch5.1/web.html#thread-safety-notes """ # getting method try: # For compatibility reasons with DISET, the methods are still called ``export_*`` method = getattr(self, 'export_%s' % self.method) except AttributeError as e: sLog.error("Invalid method", self.method) raise HTTPError(status_code=http_client.NOT_IMPLEMENTED) # Decode args args_encoded = self.get_body_argument('args', default=encode([])) args = decode(args_encoded)[0] # Execute try: self.initializeRequest() retVal = method(*args) except Exception as e: # pylint: disable=broad-except sLog.exception("Exception serving request", "%s:%s" % (str(e), repr(e))) raise HTTPError(http_client.INTERNAL_SERVER_ERROR) return retVal
def export_persistOperation(self, opJSON): """update or insert request into db :param opJSON: json string representing the operation :return: OperationID """ opObj, _size = decode(opJSON) isAuthorized = self._isAllowed(opObj, self.getRemoteCredentials()) if not isAuthorized: return S_ERROR(DErrno.ENOAUTH, "Credentials in the requests are not allowed") return self.fts3db.persistOperation(opObj)
def getOperationsFromRMSOpID(self, rmsOpID, **kwargs): """Get the FTS3Operations matching a given RMS Operation :param rmsOpID: id of the operation in the RMS :return: list of FTS3Operation objects """ res = self._getRPC(**kwargs).getOperationsFromRMSOpID(rmsOpID) if not res["OK"]: return res operationsJSON = res["Value"] try: operations, _size = decode(operationsJSON) return S_OK(operations) except Exception as e: return S_ERROR(0, "Exception when decoding the operations json %s" % e)
def getActiveJobs(self, limit=20, lastMonitor=None, jobAssignmentTag="Assigned", **kwargs): """Get all the FTSJobs that are not in a final state :param limit: max number of jobs to retrieve :return: list of FTS3Jobs """ res = self._getRPC(**kwargs).getActiveJobs(limit, lastMonitor, jobAssignmentTag) if not res["OK"]: return res activeJobsJSON = res["Value"] try: activeJobs, _size = decode(activeJobsJSON) return S_OK(activeJobs) except Exception as e: return S_ERROR("Exception when decoding the active jobs json %s" % e)
def getOperation(self, operationID, **kwargs): """Get the FTS3Operation from the database :param operationID: id of the operation :return: FTS3Operation object """ res = self._getRPC(**kwargs).getOperation(operationID) if not res["OK"]: return res opJSON = res["Value"] try: opObj, _size = decode(opJSON) return S_OK(opObj) except Exception as e: return S_ERROR("Exception when decoding the FTS3Operation object %s" % e)
def prepareTransformationTasks(self, transBody, taskDict, owner="", ownerGroup="", ownerDN="", bulkSubmissionFlag=False): """Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB""" if not taskDict: return S_OK({}) if (not owner) or (not ownerGroup): res = getProxyInfo(False, False) if not res["OK"]: return res proxyInfo = res["Value"] owner = proxyInfo["username"] ownerGroup = proxyInfo["group"] if not ownerDN: res = getDNForUsername(owner) if not res["OK"]: return res ownerDN = res["Value"][0] try: transJson, _decLen = decode(transBody) if isinstance(transJson, BaseBody): self._bodyPlugins(transJson, taskDict, ownerDN, ownerGroup) else: self._multiOperationsBody(transJson, taskDict, ownerDN, ownerGroup) except ValueError: # #json couldn't load self._singleOperationsBody(transBody, taskDict, ownerDN, ownerGroup) return S_OK(taskDict)
def _request(self, retry=0, outputFile=None, **kwargs): """ Sends the request to server :param retry: internal parameters for recursive call. TODO: remove ? :param outputFile: (default None) path to a file where to store the received data. If set, the server response will be streamed for optimization purposes, and the response data will not go through the JDecode process :param **kwargs: Any argument there is used as a post parameter. They are detailed bellow. :param method: (mandatory) name of the distant method :param args: (mandatory) json serialized list of argument for the procedure :returns: The received data. If outputFile is set, return always S_OK """ # Adding some informations to send if self.__extraCredentials: kwargs[self.KW_EXTRA_CREDENTIALS] = encode(self.__extraCredentials) kwargs["clientVO"] = self.vo kwargs["clientSetup"] = self.setup # Getting URL url = self.__findServiceURL() if not url["OK"]: return url url = url["Value"] # Getting CA file (or skip verification) verify = not self.kwargs.get(self.KW_SKIP_CA_CHECK) if verify: if not self.__ca_location: self.__ca_location = Locations.getCAsLocation() if not self.__ca_location: gLogger.error("No CAs found!") return S_ERROR("No CAs found!") verify = self.__ca_location # getting certificate # Do we use the server certificate ? if self.kwargs[self.KW_USE_CERTIFICATES]: cert = Locations.getHostCertificateAndKeyLocation() elif self.kwargs.get(self.KW_PROXY_STRING): tmpHandle, cert = tempfile.mkstemp() fp = os.fdopen(tmpHandle, "wb") fp.write(self.kwargs[self.KW_PROXY_STRING]) fp.close() # CHRIS 04.02.21 # TODO: add proxyLocation check ? else: cert = Locations.getProxyLocation() if not cert: gLogger.error("No proxy found") return S_ERROR("No proxy found") # We have a try/except for all the exceptions # whose default behavior is to try again, # maybe to different server try: # And we have a second block to handle specific exceptions # which makes it not worth retrying try: rawText = None # Default case, just return the result if not outputFile: call = requests.post(url, data=kwargs, timeout=self.timeout, verify=verify, cert=cert) # raising the exception for status here # means essentialy that we are losing here the information of what is returned by the server # as error message, since it is not passed to the exception # However, we can store the text and return it raw as an error, # since there is no guarantee that it is any JEncoded text # Note that we would get an exception only if there is an exception on the server side which # is not handled. # Any standard S_ERROR will be transfered as an S_ERROR with a correct code. rawText = call.text call.raise_for_status() return decode(rawText)[0] else: # Instruct the server not to encode the response kwargs["rawContent"] = True rawText = None # Stream download # https://requests.readthedocs.io/en/latest/user/advanced/#body-content-workflow with requests.post( url, data=kwargs, timeout=self.timeout, verify=verify, cert=cert, stream=True ) as r: rawText = r.text r.raise_for_status() with open(outputFile, "wb") as f: for chunk in r.iter_content(4096): # if chunk: # filter out keep-alive new chuncks f.write(chunk) return S_OK() # Some HTTPError are not worth retrying except requests.exceptions.HTTPError as e: status_code = e.response.status_code if status_code == http_client.NOT_IMPLEMENTED: return S_ERROR(errno.ENOSYS, "%s is not implemented" % kwargs.get("method")) elif status_code in (http_client.FORBIDDEN, http_client.UNAUTHORIZED): return S_ERROR(errno.EACCES, "No access to %s" % url) # if it is something else, retry raise # Whatever exception we have here, we deem worth retrying except Exception as e: # CHRIS TODO review this part: retry logic is fishy # self.__bannedUrls is emptied in findServiceURLs if url not in self.__bannedUrls: self.__bannedUrls += [url] if retry < self.__nbOfUrls - 1: self._request(retry=retry + 1, outputFile=outputFile, **kwargs) errStr = "%s: %s" % (str(e), rawText) return S_ERROR(errStr)
def _getMethodArgs(self, args: tuple, kwargs: dict) -> tuple: """Decode target function arguments.""" args_encoded = self.get_body_argument("args", default=encode([])) return (decode(args_encoded)[0], {})
def getProxyWithToken(self, token): """ Get proxy with token :param str token: access token :return: S_OK()/S_ERROR() """ # Get REST endpoints from local CS confUrl = gConfig.getValue("/LocalInstallation/ConfigurationServerAPI") if not confUrl: return S_ERROR('Could not get configuration server API URL.') setup = gConfig.getValue("/DIRAC/Setup") if not setup: return S_ERROR('Could not get setup name.') # Get REST endpoints from ConfigurationService try: r = requests.get( '%s/option?path=/Systems/Framework/Production/URLs/ProxyAPI' % confUrl, verify=False) r.raise_for_status() proxyAPI = decode(r.text)[0] except requests.exceptions.Timeout: return S_ERROR('Time out') except requests.exceptions.RequestException as e: return S_ERROR(str(e)) except Exception as e: return S_ERROR('Cannot read response: %s' % e) # Fill the proxy request URL url = '%ss:%s/g:%s/proxy?lifetime=%s' % (proxyAPI, setup, self.group, self.lifetime) voms = self.voms or Registry.getGroupOption(self.group, "AutoAddVOMS", False) if voms: url += '&voms=%s' % voms # Get proxy from REST API try: r = requests.get(url, headers={'Authorization': 'Bearer ' + token}, verify=False) r.raise_for_status() proxy = decode(r.text)[0] except requests.exceptions.Timeout: return S_ERROR('Time out') except requests.exceptions.RequestException as e: return S_ERROR(str(e)) except Exception as e: return S_ERROR('Cannot read response: %s' % e) if not proxy: return S_ERROR("Result is empty.") self.log.notice('Saving proxy.. to %s..' % self.pPath) # Save proxy to file try: with open(self.pPath, 'w+') as fd: fd.write(proxy.encode("UTF-8")) os.chmod(self.pPath, stat.S_IRUSR | stat.S_IWUSR) except Exception as e: return S_ERROR("%s :%s" % (self.pPath, repr(e).replace(',)', ')'))) self.log.notice('Proxy is saved to %s.' % self.pPath) return S_OK()