def export_echo(data): """ This method used for testing the performance of a service """ return S_OK(data)
def systemCall(self, cmdSeq, callbackFunction=None, shell=False, env=None): """ system call (no shell) - execute :cmdSeq: """ if shell: self.log.verbose('shellCall:', cmdSeq) else: self.log.verbose('systemCall:', cmdSeq) self.cmdSeq = cmdSeq self.callback = callbackFunction if sys.platform.find("win") == 0: closefd = False else: closefd = True try: self.child = subprocess.Popen(self.cmdSeq, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=closefd, env=env) self.childPID = self.child.pid except OSError as v: retDict = S_ERROR(v) retDict['Value'] = (-1, '', str(v)) return retDict except Exception as x: try: self.child.stdout.close() self.child.stderr.close() except Exception: pass retDict = S_ERROR(x) retDict['Value'] = (-1, '', str(x)) return retDict try: self.bufferList = [["", 0], ["", 0]] initialTime = time.time() exitStatus = self.__poll(self.child.pid) while (0, 0) == exitStatus or exitStatus is None: retDict = self.__readFromCommand() if not retDict['OK']: return retDict if self.timeout and time.time() - initialTime > self.timeout: exitStatus = self.killChild() self.__readFromCommand() return self.__generateSystemCommandError(exitStatus, "Timeout (%d seconds) for '%s' call" % (self.timeout, cmdSeq)) time.sleep(0.01) exitStatus = self.__poll(self.child.pid) self.__readFromCommand() if exitStatus: exitStatus = exitStatus[1] if exitStatus >= 256: exitStatus /= 256 return S_OK((exitStatus, self.bufferList[0][0], self.bufferList[1][0])) finally: try: self.child.stdout.close() self.child.stderr.close() except Exception: pass
def getStorageElementName( self ): """ SE name getter """ self.log.getSubLogger( 'getStorageElementName' ).verbose( "The Storage Element name is %s." % self.name ) return S_OK( self.name )
def forceRefresh(self, fromMaster=False): if self.__refreshEnabled: return self.__refresh(fromMaster=fromMaster) return S_OK()
def __findServiceURL(self): """ Discovers the URL of a service, taking into account gateways, multiple URLs, banned URLs If the site on which we run is configured to use gateways (/DIRAC/Gateways/<siteName>), these URLs will be used. To ignore the gateway, it is possible to set KW_IGNORE_GATEWAYS to False in kwargs. If self._destinationSrv (given as constructor attribute) is a properly formed URL, we just return this one. If we have to use a gateway, we just replace the server name in the url. The list of URLs defined in the CS (<System>/URLs/<Component>) is randomized This method also sets some attributes: * self.__nbOfUrls = number of URLs * self.__nbOfRetry = 2 if we have more than 2 urls, otherwise 3 * self.__bannedUrls is reinitialized if all the URLs are banned :return: the selected URL """ if not self.__initStatus['OK']: return self.__initStatus # Load the Gateways URLs for the current site Name gatewayURL = False if self.KW_IGNORE_GATEWAYS not in self.kwargs or not self.kwargs[self.KW_IGNORE_GATEWAYS]: dRetVal = gConfig.getOption("/DIRAC/Gateways/%s" % DIRAC.siteName()) if dRetVal['OK']: rawGatewayURL = List.randomize(List.fromChar(dRetVal['Value'], ","))[0] gatewayURL = "/".join(rawGatewayURL.split("/")[:3]) # If what was given as constructor attribute is a properly formed URL, # we just return this one. # If we have to use a gateway, we just replace the server name in it for protocol in gProtocolDict: if self._destinationSrv.find("%s://" % protocol) == 0: gLogger.debug("Already given a valid url", self._destinationSrv) if not gatewayURL: return S_OK(self._destinationSrv) gLogger.debug("Reconstructing given URL to pass through gateway") path = "/".join(self._destinationSrv.split("/")[3:]) finalURL = "%s/%s" % (gatewayURL, path) gLogger.debug("Gateway URL conversion:\n %s -> %s" % (self._destinationSrv, finalURL)) return S_OK(finalURL) if gatewayURL: gLogger.debug("Using gateway", gatewayURL) return S_OK("%s/%s" % (gatewayURL, self._destinationSrv)) # We extract the list of URLs from the CS (System/URLs/Component) try: urls = getServiceURL(self._destinationSrv, setup=self.setup) except Exception as e: return S_ERROR("Cannot get URL for %s in setup %s: %s" % (self._destinationSrv, self.setup, repr(e))) if not urls: return S_ERROR("URL for service %s not found" % self._destinationSrv) failoverUrls = [] # Try if there are some failover URLs to use as last resort try: failoverUrlsStr = getServiceFailoverURL(self._destinationSrv, setup=self.setup) if failoverUrlsStr: failoverUrls = failoverUrlsStr.split(',') except Exception as e: pass # We randomize the list, and add at the end the failover URLs (System/FailoverURLs/Component) urlsList = List.randomize(List.fromChar(urls, ",")) + failoverUrls self.__nbOfUrls = len(urlsList) self.__nbOfRetry = 2 if self.__nbOfUrls > 2 else 3 # we retry 2 times all services, if we run more than 2 services if self.__nbOfUrls == len(self.__bannedUrls): self.__bannedUrls = [] # retry all urls gLogger.debug("Retrying again all URLs") if len(self.__bannedUrls) > 0 and len(urlsList) > 1: # we have host which is not accessible. We remove that host from the list. # We only remove if we have more than one instance for i in self.__bannedUrls: gLogger.debug("Removing banned URL", "%s" % i) urlsList.remove(i) # Take the first URL from the list #randUrls = List.randomize( urlsList ) + failoverUrls sURL = urlsList[0] # If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL # is not on a host which is banned. If it is, we take the next one in the list using __selectUrl # If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL # is not on a host which is banned. If it is, we take the next one in the list using __selectUrl if len(self.__bannedUrls) > 0 and self.__nbOfUrls > 2: # when we have multiple services then we can # have a situation when two services are running on the same machine with different ports... retVal = Network.splitURL(sURL) nexturl = None if retVal['OK']: nexturl = retVal['Value'] found = False for i in self.__bannedUrls: retVal = Network.splitURL(i) if retVal['OK']: bannedurl = retVal['Value'] else: break # We found a banned URL on the same host as the one we are running on if nexturl[1] == bannedurl[1]: found = True break if found: nexturl = self.__selectUrl(nexturl, urlsList[1:]) if nexturl: # an url found which is in different host sURL = nexturl gLogger.debug("Discovering URL for service", "%s -> %s" % (self._destinationSrv, sURL)) return S_OK(sURL)
def exec_taskFreeze(cls, taskId, taskObj, eType): return S_OK()
def handshake_singleStep(self): """ Used to perform SSL handshakes. These are now done automatically. """ # This isn't used any more, the handshake is done inside the M2Crypto library return S_OK()
osSocket.close() if retries: return self.__socketConnect( hostAddress, timeout, retries - 1 ) else: return S_ERROR( "Can't connect: %s" % str( e ) ) if e.args[0] not in ( 114, 115 ): return S_ERROR( "Can't connect: %s" % str( e ) ) #Connect in progress oL = select.select( [], [ osSocket ], [], timeout )[1] if len( oL ) == 0: osSocket.close() return S_ERROR( "Connection timeout" ) errno = osSocket.getsockopt( socket.SOL_SOCKET, socket.SO_ERROR ) if errno != 0: return S_ERROR( "Can't connect: %s" % str( ( errno, os.strerror( errno ) ) ) ) return S_OK( osSocket ) def __connect( self, socketInfo, hostAddress ): #Connect baby! result = self.__socketConnect( hostAddress, socketInfo.infoDict[ 'timeout' ] ) if not result[ 'OK' ]: return result osSocket = result[ 'Value' ] #SSL MAGIC sslSocket = GSI.SSL.Connection( socketInfo.getSSLContext(), osSocket ) #Generate sessionId sessionHash = md5.md5() sessionHash.update( str( hostAddress ) ) sessionHash.update( "|%s" % str( socketInfo.getLocalCredentialsLocation() ) ) for key in ( 'proxyLocation', 'proxyString' ): if key in socketInfo.infoDict:
def getConfigurationTree(self, root='', *filters): """ Create a dictionary with all sections, subsections and options starting from given root. Result can be filtered. :param str root: Starting point in the configuration tree. :param filters: Select results that contain given substrings (check full path, i.e. with option name) :type filters: str or python:list[str] :return: Return a dictionary where keys are paths taken from the configuration (e.g. /Systems/Configuration/...). Value is "None" when path points to a section or not "None" if path points to an option. """ log = DIRAC.gLogger.getSubLogger('getConfigurationTree') # check if root is an option (special case) option = self.getOption(root) if option['OK']: result = {root: option['Value']} else: result = {root: None} for substr in filters: if substr not in root: result = {} break # remove slashes at the end root = root.rstrip('/') # get options of current root options = self.getOptionsDict(root) if not options['OK']: log.error("getOptionsDict() failed with message: %s" % options['Message']) return S_ERROR('Invalid root path provided') for key, value in options['Value'].iteritems(): path = cfgPath(root, key) addOption = True for substr in filters: if substr not in path: addOption = False break if addOption: result[path] = value # get subsections of the root sections = self.getSections(root) if not sections['OK']: log.error("getSections() failed with message: %s" % sections['Message']) return S_ERROR('Invalid root path provided') # recursively go through subsections and get their subsections for section in sections['Value']: subtree = self.getConfigurationTree("%s/%s" % (root, section), *filters) if not subtree['OK']: log.error( "getConfigurationTree() failed with message: %s" % sections['Message']) return S_ERROR( 'Configuration was altered during the operation') result.update(subtree['Value']) return S_OK(result)
def ldapsearchBDII(filt=None, attr=None, host=None, base=None, selectionString="Glue"): """ Python wrapper for ldapserch at bdii. :param filt: Filter used to search ldap, default = '', means select all :param attr: Attributes returned by ldapsearch, default = '*', means return all :param host: Host used for ldapsearch, default = 'lcg-bdii.cern.ch:2170', can be changed by $LCG_GFAL_INFOSYS :return: standard DIRAC answer with Value equals to list of ldapsearch responses Each element of list is dictionary with keys: 'dn': Distinguished name of ldapsearch response 'objectClass': List of classes in response 'attr': Dictionary of attributes """ if filt is None: filt = '' if attr is None: attr = '' if host is None: host = 'lcg-bdii.cern.ch:2170' if base is None: base = 'Mds-Vo-name=local,o=grid' if isinstance(attr, list): attr = ' '.join(attr) cmd = 'ldapsearch -x -LLL -o ldif-wrap=no -h %s -b %s "%s" %s' % ( host, base, filt, attr) result = shellCall(0, cmd) response = [] if not result['OK']: return result status = result['Value'][0] stdout = result['Value'][1] stderr = result['Value'][2] if status != 0: return S_ERROR(stderr) lines = [] for line in stdout.split("\n"): if line.find(" ") == 0: lines[-1] += line.strip() else: lines.append(line.strip()) record = None for line in lines: if line.find('dn:') == 0: record = { 'dn': line.replace('dn:', '').strip(), 'objectClass': [], 'attr': { 'dn': line.replace('dn:', '').strip() } } response.append(record) continue if record: if line.find('objectClass:') == 0: record['objectClass'].append( line.replace('objectClass:', '').strip()) continue if line.find(selectionString) == 0: index = line.find(':') if index > 0: attr = line[:index] value = line[index + 1:].strip() if attr in record['attr']: if isinstance(record['attr'][attr], list): record['attr'][attr].append(value) else: record['attr'][attr] = [ record['attr'][attr], value ] else: record['attr'][attr] = value return S_OK(response)
def applyModifications(self, modList, parentSection=""): """ Apply modifications to a CFG :type modList: List :param modList: Modifications from a getModifications call :return: True/False """ for modAction in modList: action = modAction[0] key = modAction[1] iPos = modAction[2] value = modAction[3] if action == 'addSec': if key in self.listSections(): return S_ERROR("Section %s/%s already exists" % (parentSection, key)) # key, value, comment, beforeKey = "" value = CFG().loadFromBuffer(value) comment = modAction[4].strip() if iPos < len(self.__orderedList): beforeKey = self.__orderedList[iPos] else: beforeKey = "" self.addKey(key, value, comment, beforeKey) elif action == 'delSec': if key not in self.listSections(): return S_ERROR("Section %s/%s does not exist" % (parentSection, key)) self.deleteKey(key) elif action == 'modSec': if key not in self.listSections(): return S_ERROR("Section %s/%s does not exist" % (parentSection, key)) comment = modAction[4].strip() self.setComment(key, comment) if value: result = self[key].applyModifications( value, "%s/%s" % (parentSection, key)) if not result['OK']: return result if iPos >= len( self.__orderedList) or key != self.__orderedList[iPos]: prevPos = self.__orderedList.index(key) del self.__orderedList[prevPos] self.__orderedList.insert(iPos, key) elif action == "addOpt": if key in self.listOptions(): return S_ERROR("Option %s/%s exists already" % (parentSection, key)) # key, value, comment, beforeKey = "" comment = modAction[4].strip() if iPos < len(self.__orderedList): beforeKey = self.__orderedList[iPos] else: beforeKey = "" self.addKey(key, value, comment, beforeKey) elif action == "modOpt": if key not in self.listOptions(): return S_ERROR("Option %s/%s does not exist" % (parentSection, key)) comment = modAction[4].strip() self.setOption(key, value, comment) if iPos >= len( self.__orderedList) or key != self.__orderedList[iPos]: prevPos = self.__orderedList.index(key) del (self.__orderedList[prevPos]) self.__orderedList.insert(iPos, key) elif action == "delOpt": if key not in self.listOptions(): return S_ERROR("Option %s/%s does not exist" % (parentSection, key)) self.deleteKey(key) return S_OK()
def getBdiiCEInfo(vo, host=None, glue2=False): """ Get information for all the CEs/queues for a given VO :param str vo: BDII VO name :param str host: url to query for information :param bool glue2: if True query the GLUE2 information schema :return: result structure: result['Value'][siteID]['CEs'][ceID]['Queues'][queueName]. For each siteID, ceID, queueName all the BDII/Glue parameters are retrieved """ if glue2: return Glue2.getGlue2CEInfo(vo, host=host) result = ldapCEState('', vo, host=host) if not result['OK']: return result siteDict = {} ceDict = {} queueDict = {} for queue in result['Value']: queue = dict(queue) clusterID = queue.get('GlueForeignKey', '').replace('GlueClusterUniqueID=', '') ceID = queue.get('GlueCEUniqueID', '').split(':')[0] queueDict[queue['GlueCEUniqueID']] = queue queueDict[queue['GlueCEUniqueID']]['CE'] = ceID if ceID not in ceDict: result = ldapCluster(clusterID, host=host) if not result['OK']: continue if not result['Value']: continue ce = result['Value'][0] ceDict[ceID] = ce fKey = ce['GlueForeignKey'] # pylint: disable=unsubscriptable-object siteID = '' for key in fKey: if key.startswith('GlueSiteUniqueID'): siteID = key.replace('GlueSiteUniqueID=', '') ceDict[ceID]['Site'] = siteID result = ldapCE(clusterID, host=host) ce = {} if result['OK'] and result['Value']: ce = result['Value'][0] ceDict[ceID].update(ce) if siteID not in siteDict: site = {} result = ldapSite(siteID, host=host) if result['OK'] and result['Value']: site = result['Value'][0] siteDict[siteID] = site for ceID in ceDict: siteID = ceDict[ceID]['Site'] if siteID in siteDict: siteDict[siteID].setdefault('CEs', {}) siteDict[siteID]['CEs'][ceID] = ceDict[ceID] for queueID in queueDict: ceID = queueDict[queueID]['CE'] siteID = ceDict[ceID]['Site'] siteDict[siteID]['CEs'][ceID].setdefault('Queues', {}) queueName = re.split(r':\d+/', queueDict[queueID]['GlueCEUniqueID'])[1] siteDict[siteID]['CEs'][ceID]['Queues'][queueName] = queueDict[queueID] return S_OK(siteDict)
def __executeMethod(self, lfn, *args, **kwargs): """ Forward the call to each storage in turn until one works. The method to be executed is stored in self.methodName :param lfn : string, list or dictionary :param *args : variable amount of non-keyword arguments. SHOULD BE EMPTY :param **kwargs : keyword arguments :returns S_OK( { 'Failed': {lfn : reason} , 'Successful': {lfn : value} } ) The Failed dict contains the lfn only if the operation failed on all the storages The Successful dict contains the value returned by the successful storages. A special kwargs is 'inputProtocol', which can be specified for putFile. It describes the protocol used as source protocol, since there is in principle only one. """ removedArgs = {} log = self.log.getSubLogger('__executeMethod') log.verbose("preparing the execution of %s" % (self.methodName)) # args should normaly be empty to avoid problem... if len(args): log.verbose("args should be empty!%s" % args) # because there is normally only one kw argument, I can move it from args to kwargs methDefaultArgs = StorageElementItem.__defaultsArguments.get( self.methodName, {}).keys() if len(methDefaultArgs): kwargs[methDefaultArgs[0]] = args[0] args = args[1:] log.verbose( "put it in kwargs, but dirty and might be dangerous!args %s kwargs %s" % (args, kwargs)) # We check the deprecated arguments for depArg in StorageElementItem.__deprecatedArguments: if depArg in kwargs: log.verbose( "%s is not an allowed argument anymore. Please change your code!" % depArg) removedArgs[depArg] = kwargs[depArg] del kwargs[depArg] # Set default argument if any methDefaultArgs = StorageElementItem.__defaultsArguments.get( self.methodName, {}) for argName in methDefaultArgs: if argName not in kwargs: log.debug("default argument %s for %s not present.\ Setting value %s." % (argName, self.methodName, methDefaultArgs[argName])) kwargs[argName] = methDefaultArgs[argName] res = checkArgumentFormat(lfn) if not res['OK']: errStr = "Supplied lfns must be string, list of strings or a dictionary." log.debug(errStr) return res lfnDict = res['Value'] log.verbose("Attempting to perform '%s' operation with %s lfns." % (self.methodName, len(lfnDict))) res = self.isValid(operation=self.methodName) if not res['OK']: return res else: if not self.valid: return S_ERROR(self.errorReason) # In case executing putFile, we can assume that all the source urls # are from the same protocol. This optional parameter, if defined # can be used to ignore some storage plugins and thus save time # and avoid fake failures showing in the accounting inputProtocol = kwargs.pop('inputProtocol', None) successful = {} failed = {} filteredPlugins = self.__filterPlugins(self.methodName, kwargs.get('protocols'), inputProtocol) if not filteredPlugins: return S_ERROR( errno.EPROTONOSUPPORT, "No storage plugins matching the requirements\ (operation %s protocols %s inputProtocol %s)" \ % ( self.methodName, kwargs.get( 'protocols' ), inputProtocol ) ) # Try all of the storages one by one for storage in filteredPlugins: # Determine whether to use this storage object storageParameters = storage.getParameters() pluginName = storageParameters['PluginName'] if not lfnDict: log.debug("No lfns to be attempted for %s protocol." % pluginName) continue log.verbose("Generating %s protocol URLs for %s." % (len(lfnDict), pluginName)) replicaDict = kwargs.pop('replicaDict', {}) if storage.pluginName != "Proxy": res = self.__generateURLDict(lfnDict, storage, replicaDict=replicaDict) urlDict = res['Value']['Successful'] # url : lfn failed.update(res['Value']['Failed']) else: urlDict = dict([(lfn, lfn) for lfn in lfnDict]) if not len(urlDict): log.verbose( "__executeMethod No urls generated for protocol %s." % pluginName) else: log.verbose( "Attempting to perform '%s' for %s physical files" % (self.methodName, len(urlDict))) fcn = None if hasattr(storage, self.methodName) and callable( getattr(storage, self.methodName)): fcn = getattr(storage, self.methodName) if not fcn: return S_ERROR( DErrno.ENOMETH, "SE.__executeMethod: unable to invoke %s, it isn't a member function of storage" ) urlsToUse = { } # url : the value of the lfn dictionary for the lfn of this url for url in urlDict: urlsToUse[url] = lfnDict[urlDict[url]] startDate = datetime.datetime.utcnow() startTime = time.time() res = fcn(urlsToUse, *args, **kwargs) elapsedTime = time.time() - startTime self.addAccountingOperation(urlsToUse, startDate, elapsedTime, storageParameters, res) if not res['OK']: errStr = "Completely failed to perform %s." % self.methodName log.debug( errStr, 'with plugin %s: %s' % (pluginName, res['Message'])) for lfn in urlDict.values(): if lfn not in failed: failed[lfn] = '' failed[lfn] = "%s %s" % ( failed[lfn], res['Message']) if failed[lfn] else res['Message'] else: for url, lfn in urlDict.items(): if url not in res['Value']['Successful']: if lfn not in failed: failed[lfn] = '' if url in res['Value']['Failed']: self.log.debug(res['Value']['Failed'][url]) failed[lfn] = "%s %s" % ( failed[lfn], res['Value']['Failed'][url]) if failed[ lfn] else res['Value']['Failed'][url] else: errStr = 'No error returned from plug-in' failed[lfn] = "%s %s" % ( failed[lfn], errStr) if failed[lfn] else errStr else: successful[lfn] = res['Value']['Successful'][url] if lfn in failed: failed.pop(lfn) lfnDict.pop(lfn) gDataStoreClient.commit() return S_OK({'Failed': failed, 'Successful': successful})
def getStatus(self): """ Return Status of the SE, a dictionary with: - Read: True (is allowed), False (it is not allowed) - Write: True (is allowed), False (it is not allowed) - Remove: True (is allowed), False (it is not allowed) - Check: True (is allowed), False (it is not allowed). NB: Check always allowed IF Read is allowed (regardless of what set in the Check option of the configuration) - DiskSE: True if TXDY with Y > 0 (defaults to True) - TapeSE: True if TXDY with X > 0 (defaults to False) - TotalCapacityTB: float (-1 if not defined) - DiskCacheTB: float (-1 if not defined) """ self.log.getSubLogger('getStatus').verbose( "determining status of %s." % self.name) retDict = {} if not self.valid: retDict['Read'] = False retDict['Write'] = False retDict['Remove'] = False retDict['Check'] = False retDict['DiskSE'] = False retDict['TapeSE'] = False retDict['TotalCapacityTB'] = -1 retDict['DiskCacheTB'] = -1 return S_OK(retDict) # If nothing is defined in the CS Access is allowed # If something is defined, then it must be set to Active retDict['Read'] = not ('ReadAccess' in self.options and self.options['ReadAccess'] not in ('Active', 'Degraded')) retDict['Write'] = not ('WriteAccess' in self.options and self.options['WriteAccess'] not in ('Active', 'Degraded')) retDict['Remove'] = not ('RemoveAccess' in self.options and self.options['RemoveAccess'] not in ('Active', 'Degraded')) if retDict['Read']: retDict['Check'] = True else: retDict['Check'] = not ('CheckAccess' in self.options and self.options['CheckAccess'] not in ('Active', 'Degraded')) diskSE = True tapeSE = False if 'SEType' in self.options: # Type should follow the convention TXDY seType = self.options['SEType'] diskSE = re.search('D[1-9]', seType) != None tapeSE = re.search('T[1-9]', seType) != None retDict['DiskSE'] = diskSE retDict['TapeSE'] = tapeSE try: retDict['TotalCapacityTB'] = float(self.options['TotalCapacityTB']) except Exception: retDict['TotalCapacityTB'] = -1 try: retDict['DiskCacheTB'] = float(self.options['DiskCacheTB']) except Exception: retDict['DiskCacheTB'] = -1 return S_OK(retDict)
def exec_executorConnected(cls, execName, trid): return S_OK()
def initializeConfigurationHandler(serviceInfo): global gServiceInterface gServiceInterface = ServiceInterface(serviceInfo["URL"]) return S_OK()
def exec_prepareToSend(cls, taskId, taskObj, eId): return S_OK()
def export_getVersion(cls): return S_OK(gServiceInterface.getVersion())
def close(self): # pylint: disable=line-too-long """ Close this socket. """ if self.oSocket: # TL;DR: # Do NOT touch that method # # Surprisingly (to me at least), M2Crypto does not close # the underlying socket when calling SSL.Connection.close # It only does it when the garbage collector kicks in (see ~M2Crypto.SSL.Connection.Connection.__del__) # If the socket is not closed, the connection may hang forever. # # Thus, we are setting self.oSocket to None to allow the GC to do the work, but since we are not sure # that it will run, we anyway force the connection to be closed # # However, we should close the underlying socket only after SSL was shutdown properly. # This is because OpenSSL `ssl3_shutdown` (see callstack below) may still read some data # (see https://github.com/openssl/openssl/blob/master/ssl/s3_lib.c#L4509):: # # # 1 0x00007fffe9d48fc0 in sock_read () from /lib/libcrypto.so.1.0.0 # 2 0x00007fffe9d46e83 in BIO_read () from /lib/libcrypto.so.1.0.0 # 3 0x00007fffe9eab9dd in ssl3_read_n () from /lib/libssl.so.1.0.0 # 4 0x00007fffe9ead216 in ssl3_read_bytes () from /lib/libssl.so.1.0.0 # 5 0x00007fffe9ea999c in ssl3_shutdown () from /lib/libssl.so.1.0.0 # 6 0x00007fffe9ed4f93 in ssl_free () from /lib/libssl.so.1.0.0 # 7 0x00007fffe9d46d5b in BIO_free () from /lib/libcrypto.so.1.0.0 # 8 0x00007fffe9f30a96 in bio_free (bio=0x5555556f3200) at SWIG/_m2crypto_wrap.c:5008 # 9 0x00007fffe9f30b1e in _wrap_bio_free (self=<optimized out>, args=<optimized out>) at SWIG/_m2crypto_wrap.c # # We unfortunately have no way to force that order, and there is a risk of deadlock # when running in a multi threaded environment like the agents:: # # Thread A opens socket, gets FD = 111 # Thread A works on it # Thread A closes FD 111 (underlying socket.close()) # Thread B opens socket, gets FD = 111 # Thread A calls read on FD=111 from ssl3_shutdown # # This is illustrated on the strace below:: # # 26461 14:25:15.266692 write(111<TCPv6:[[<srcAddressV6>]:42688->[<dstAddressV6>]:9140]>, # "blabla", 37 <unfinished ...> # 26464 14:25:15.266857 <... connect resumed>) = 0 <0.000195> # 26464 14:25:15.267023 getsockname(120<UDP:[<srcAddress>:44252->188.185.84.86:9140]>, <unfinished ...> # 26461 14:25:15.267176 <... write resumed>) = 37 <0.000453> # 26464 14:25:15.267425 <... getsockname resumed>{sa_family=AF_INET, sin_port=htons(44252), # sin_addr=inet_addr("<srcAddress>")}, [28->16]) = 0 <0.000292> # 26461 14:25:15.267466 close(111<TCPv6:[[<srcAddressV6>]:42688->[<dstAddressV6>]:9140]> <unfinished ...> # 26464 14:25:15.267637 close(120<UDP:[<srcAddress>:44252->188.185.84.86:9140]> <unfinished ...> # 26464 14:25:15.267738 <... close resumed>) = 0 <0.000086> # 26461 14:25:15.267768 <... close resumed>) = 0 <0.000285> # 26464 14:25:15.267827 socket(AF_INET6, SOCK_DGRAM|SOCK_CLOEXEC, IPPROTO_IP <unfinished ...> # 26461 14:25:15.267888 futex(0x21f8620, FUTEX_WAKE_PRIVATE, 1 <unfinished ...> # 26464 14:25:15.267976 <... socket resumed>) = 111<UDPv6:[1207802822]> <0.000138> # 26461 14:25:15.268092 <... futex resumed>) = 1 <0.000196> # 26464 14:25:15.268195 connect(111<UDPv6:[1207802822]>, # {sa_family=AF_INET6, sin6_port=htons(9140), # inet_pton(AF_INET6, "<dstAddressV6>", &sin6_addr), # sin6_flowinfo=htonl(0), sin6_scope_id=0 # }, 28 <unfinished ...> # 26461 14:25:15.268294 read(111<UDPv6:[[<srcAddressV6>]:42480->[<dstAddressV6>]:9140]>, <unfinished ...> # 26464 14:25:15.268503 <... connect resumed>) = 0 <0.000217> # 26464 14:25:15.268673 getsockname(111<UDPv6:[[<srcAddressV6>]:42480->[<dstAddressV6>]:9140]>, <unfinished ...> # 26464 14:25:15.268862 <... getsockname resumed>{sa_family=AF_INET6, sin6_port=htons(42480), # inet_pton(AF_INET6, "<srcAddressV6>", &sin6_addr), sin6_flowinfo=htonl(0), sin6_scope_id= # 0}, [28]) = 0 <0.000168> # 26464 14:25:15.269048 # close(111<UDPv6:[[<srcAddressV6>]:42480->[<dstAddressV6>]:9140]> # <unfinished ...> # # # Update 16.07.20: # M2Crypto 0.36 contains the bug fix https://gitlab.com/m2crypto/m2crypto/-/merge_requests/247 # that allows proper closing. So manual closing of the underlying socket should not be needed anymore # Update 16.07.20 # I add this shutdown call without being 100% sure # it solves some hanging connections issues, but it seems # to work. it does not appear in any M2Crypto doc, but comparing # some internals of M2Crypto and official python SSL library, # it seems to make sense self.oSocket.shutdown(socket.SHUT_RDWR) # Update 16.07.20 # With freeBio=True, we force the # closing of the socket before the GC runs self.oSocket.close(freeBio=True) # underlyingSocket = self.oSocket.socket self.oSocket = None # underlyingSocket.close() return S_OK()
def export_getCompressedData(cls): sData = gServiceInterface.getCompressedConfigurationData() return S_OK(sData)
def enable(self): self.__refreshEnabled = True if self.__lastRefreshExpired(): return self.forceRefresh() return S_OK()
def export_publishSlaveServer(cls, sURL): gServiceInterface.publishSlaveServer(sURL) return S_OK()
def __doFileTransfer(self, sDirection): """ Execute a file transfer action :type sDirection: string :param sDirection: Direction of the transfer :return: S_OK/S_ERROR """ retVal = self.__trPool.receive(self.__trid) if not retVal["OK"]: raise ConnectionError( "Error while receiving file description %s %s" % (self.srv_getFormattedRemoteCredentials(), retVal["Message"])) # Reconvert to tuple fileInfo = tuple(retVal["Value"]) sDirection = "%s%s" % (sDirection[0].lower(), sDirection[1:]) if "transfer_%s" % sDirection not in dir(self): self.__trPool.send( self.__trid, S_ERROR("Service can't transfer files %s" % sDirection)) return retVal = self.__trPool.send(self.__trid, S_OK("Accepted")) if not retVal["OK"]: return retVal self.__logRemoteQuery("FileTransfer/%s" % sDirection, fileInfo) self.__lockManager.lock("FileTransfer/%s" % sDirection) try: try: fileHelper = FileHelper(self.__trPool.get(self.__trid)) if sDirection == "fromClient": fileHelper.setDirection("fromClient") uRetVal = self.transfer_fromClient(fileInfo[0], fileInfo[1], fileInfo[2], fileHelper) elif sDirection == "toClient": fileHelper.setDirection("toClient") uRetVal = self.transfer_toClient(fileInfo[0], fileInfo[1], fileHelper) elif sDirection == "bulkFromClient": fileHelper.setDirection("fromClient") uRetVal = self.transfer_bulkFromClient( fileInfo[0], fileInfo[1], fileInfo[2], fileHelper) elif sDirection == "bulkToClient": fileHelper.setDirection("toClient") uRetVal = self.transfer_bulkToClient( fileInfo[0], fileInfo[1], fileHelper) elif sDirection == "listBulk": fileHelper.setDirection("toClient") uRetVal = self.transfer_listBulk(fileInfo[0], fileInfo[1], fileHelper) else: return S_ERROR("Direction %s does not exist!!!" % sDirection) if uRetVal["OK"] and not fileHelper.finishedTransmission(): gLogger.error( "You haven't finished receiving/sending the file", str(fileInfo)) return S_ERROR("Incomplete transfer") del fileHelper return uRetVal finally: self.__lockManager.unlock("FileTransfer/%s" % sDirection) except Exception as e: # pylint: disable=broad-except gLogger.exception("Uncaught exception when serving Transfer", "%s" % sDirection, lException=e) return S_ERROR("Server error while serving %s: %s" % (sDirection, repr(e)))
def export_writeEnabled(cls): return S_OK(gServiceInterface.isMaster())
def _connect(self): """ Establish the connection. It uses the URL discovered in __discoverURL. In case the connection cannot be established, __discoverURL is called again, and _connect calls itself. We stop after trying self.__nbOfRetry * self.__nbOfUrls """ # Check if the useServerCertificate configuration changed # Note: I am not really sure that all this block makes # any sense at all since all these variables are # evaluated in __discoverCredentialsToUse if gConfig.useServerCertificate() != self.__useCertificates: if self.__forceUseCertificates is None: self.__useCertificates = gConfig.useServerCertificate() self.kwargs[self.KW_USE_CERTIFICATES] = self.__useCertificates # The server certificate use context changed, rechecking the transport sanity result = self.__checkTransportSanity() if not result['OK']: return result # Take all the extra credentials self.__discoverExtraCredentials() if not self.__initStatus['OK']: return self.__initStatus if self.__enableThreadCheck: self.__checkThreadID() gLogger.debug("Connecting to: %s" % self.serviceURL) try: # Calls the transport method of the apropriate protocol. # self.__URLTuple[1:3] = [server name, port, System/Component] transport = gProtocolDict[self.__URLTuple[0]]['transport'](self.__URLTuple[1:3], **self.kwargs) # the socket timeout is the default value which is 1. # later we increase to 5 retVal = transport.initAsClient() # If we have an issue connecting if not retVal['OK']: # We try at most __nbOfRetry each URLs if self.__retry < self.__nbOfRetry * self.__nbOfUrls - 1: # Recompose the URL (why not using self.serviceURL ? ) url = "%s://%s:%d/%s" % (self.__URLTuple[0], self.__URLTuple[1], int(self.__URLTuple[2]), self.__URLTuple[3]) # Add the url to the list of banned URLs if it is not already there. (Can it happen ? I don't think so) if url not in self.__bannedUrls: self.__bannedUrls += [url] # Why only printing in this case ? if len(self.__bannedUrls) < self.__nbOfUrls: gLogger.notice("Non-responding URL temporarily banned", "%s" % url) # Increment the retry couunter self.__retry += 1 # If it is our last attempt for each URL, we increase the timeout if self.__retryCounter == self.__nbOfRetry - 1: transport.setSocketTimeout(5) # we increase the socket timeout in case the network is not good gLogger.info("Retry connection", ": %d to %s" % (self.__retry, self.serviceURL)) # If we tried all the URL, we increase the global counter (__retryCounter), and sleep if len(self.__bannedUrls) == self.__nbOfUrls: self.__retryCounter += 1 # we run only one service! In that case we increase the retry delay. self.__retryDelay = 3. / self.__nbOfUrls if self.__nbOfUrls > 1 else 2 gLogger.info("Waiting %f seconds before retry all service(s)" % self.__retryDelay) time.sleep(self.__retryDelay) # rediscover the URL self.__discoverURL() # try to reconnect return self._connect() else: return retVal except Exception as e: gLogger.exception(lException=True, lExcInfo=True) return S_ERROR("Can't connect to %s: %s" % (self.serviceURL, repr(e))) # We add the connection to the transport pool trid = getGlobalTransportPool().add(transport) return S_OK((trid, transport))
def conn_drop(self, trid): self.__eDispatch.removeExecutor(trid) return S_OK()
class SocketInfoFactory: def __init__(self): self.__timeout = 1 def setSocketTimeout(self, timeout): self.__timeout = timeout def getSocketTimeout(self): return self.__timeout def generateClientInfo(self, destinationHostname, kwargs): infoDict = { 'clientMode': True, 'hostname': destinationHostname, 'timeout': 600, 'enableSessions': True } for key in kwargs.keys(): infoDict[key] = kwargs[key] try: return S_OK(SocketInfo(infoDict)) except Exception as e: return S_ERROR("Error while creating SSL context: %s" % str(e)) def generateServerInfo(self, kwargs): infoDict = {'clientMode': False, 'timeout': 30} for key in kwargs.keys(): infoDict[key] = kwargs[key] try: return S_OK(SocketInfo(infoDict)) except Exception as e: return S_ERROR(str(e)) def __socketConnect(self, hostAddress, timeout, retries=2): addrs = socket.getaddrinfo(hostAddress[0], hostAddress[1], 0, socket.SOCK_STREAM) errs = [] for addr in addrs: res = self.__sockConnect(addr[4], addr[0], timeout, retries) if res['OK']: return res else: errs.append(res['Message']) return S_ERROR(", ".join(errs)) def __sockConnect(self, hostAddress, sockType, timeout, retries): try: osSocket = socket.socket(sockType, socket.SOCK_STREAM) except socket.error as e: gLogger.warn("Exception while creating a socket:", str(e)) return S_ERROR("Exception while creating a socket:%s" % str(e)) # osSocket.setblocking( 0 ) if timeout: tsocket = self.getSocketTimeout() gLogger.verbose("Connection timeout set to: ", tsocket) osSocket.settimeout( tsocket) # we try to connect 3 times with 1 second timeout try: osSocket.connect(hostAddress) except socket.error, e: if e.args[0] == "timed out": osSocket.close() if retries: return self.__sockConnect(hostAddress, sockType, timeout, retries - 1) else: return S_ERROR("Can't connect: %s" % str(e)) if e.args[0] not in (114, 115): return S_ERROR("Can't connect: %s" % str(e)) #Connect in progress oL = select.select([], [osSocket], [], timeout)[1] if len(oL) == 0: osSocket.close() return S_ERROR("Connection timeout") errno = osSocket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) if errno != 0: return S_ERROR("Can't connect: %s" % str( (errno, os.strerror(errno)))) return S_OK(osSocket)
def exec_executorDisconnected(cls, trid): return S_OK()
def isValid( self, operation = '' ): """ check CS/RSS statuses for :operation: :param str operation: operation name """ log = self.log.getSubLogger( 'isValid', True ) log.verbose( "Determining if the StorageElement %s is valid for VO %s" % ( self.name, self.vo ) ) if not self.valid: log.debug( "Failed to create StorageElement plugins.", self.errorReason ) return S_ERROR( "SE.isValid: Failed to create StorageElement plugins: %s" % self.errorReason ) # Check if the Storage Element is eligible for the user's VO if 'VO' in self.options and not self.vo in self.options['VO']: log.debug( "StorageElement is not allowed for VO", self.vo ) return S_ERROR( errno.EACCES, "StorageElement.isValid: StorageElement is not allowed for VO" ) log.verbose( "Determining if the StorageElement %s is valid for %s" % ( self.name, operation ) ) if ( not operation ) or ( operation in self.okMethods ): return S_OK() # Determine whether the StorageElement is valid for checking, reading, writing res = self.getStatus() if not res[ 'OK' ]: log.debug( "Could not call getStatus", res['Message'] ) return S_ERROR( "SE.isValid could not call the getStatus method" ) checking = res[ 'Value' ][ 'Check' ] reading = res[ 'Value' ][ 'Read' ] writing = res[ 'Value' ][ 'Write' ] removing = res[ 'Value' ][ 'Remove' ] # Determine whether the requested operation can be fulfilled if ( not operation ) and ( not reading ) and ( not writing ) and ( not checking ): log.debug( "Read, write and check access not permitted." ) return S_ERROR( errno.EACCES, "SE.isValid: Read, write and check access not permitted." ) # The supplied operation can be 'Read','Write' or any of the possible StorageElement methods. if ( operation in self.readMethods ) or ( operation.lower() in ( 'read', 'readaccess' ) ): operation = 'ReadAccess' elif operation in self.writeMethods or ( operation.lower() in ( 'write', 'writeaccess' ) ): operation = 'WriteAccess' elif operation in self.removeMethods or ( operation.lower() in ( 'remove', 'removeaccess' ) ): operation = 'RemoveAccess' elif operation in self.checkMethods or ( operation.lower() in ( 'check', 'checkaccess' ) ): operation = 'CheckAccess' else: log.debug( "The supplied operation is not known.", operation ) return S_ERROR( DErrno.ENOMETH , "SE.isValid: The supplied operation is not known." ) log.debug( "check the operation: %s " % operation ) # Check if the operation is valid if operation == 'CheckAccess': if not reading: if not checking: log.debug( "Check access not currently permitted." ) return S_ERROR( errno.EACCES, "SE.isValid: Check access not currently permitted." ) if operation == 'ReadAccess': if not reading: log.debug( "Read access not currently permitted." ) return S_ERROR( errno.EACCES, "SE.isValid: Read access not currently permitted." ) if operation == 'WriteAccess': if not writing: log.debug( "Write access not currently permitted." ) return S_ERROR( errno.EACCES, "SE.isValid: Write access not currently permitted." ) if operation == 'RemoveAccess': if not removing: log.debug( "Remove access not currently permitted." ) return S_ERROR( errno.EACCES, "SE.isValid: Remove access not currently permitted." ) return S_OK()
# else check the parameter elif not isinstance(args[iIndex], oTypesList[iIndex]): mismatch = True # Has there been a mismatch? if mismatch: sError = "Type mismatch in parameter %d (starting with param 0) Received %s, expected %s" % ( iIndex, type(args[iIndex]), str(oTypesList[iIndex])) return S_ERROR(sError) if len(args) < len(oTypesList): return S_ERROR("Function %s expects at least %s arguments" % (method, len(oTypesList))) except Exception, v: sError = "Error in parameter check: %s" % str(v) gLogger.exception(sError) return S_ERROR(sError) return S_OK() #### # # Connection methods # #### __connectionCallbackTypes = { 'new': [types.StringTypes, types.DictType], 'connected': [], 'drop': [] } def __doConnection(self, methodName): """