def export_echo(data):
"""
This method used for testing the performance of a service
"""
return S_OK(data)
def systemCall(self, cmdSeq, callbackFunction=None, shell=False, env=None):
""" system call (no shell) - execute :cmdSeq: """
if shell:
self.log.verbose('shellCall:', cmdSeq)
else:
self.log.verbose('systemCall:', cmdSeq)
self.cmdSeq = cmdSeq
self.callback = callbackFunction
if sys.platform.find("win") == 0:
closefd = False
else:
closefd = True
try:
self.child = subprocess.Popen(self.cmdSeq,
shell=shell,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=closefd,
env=env)
self.childPID = self.child.pid
except OSError as v:
retDict = S_ERROR(v)
retDict['Value'] = (-1, '', str(v))
return retDict
except Exception as x:
try:
self.child.stdout.close()
self.child.stderr.close()
except Exception:
pass
retDict = S_ERROR(x)
retDict['Value'] = (-1, '', str(x))
return retDict
try:
self.bufferList = [["", 0], ["", 0]]
initialTime = time.time()
exitStatus = self.__poll(self.child.pid)
while (0, 0) == exitStatus or exitStatus is None:
retDict = self.__readFromCommand()
if not retDict['OK']:
return retDict
if self.timeout and time.time() - initialTime > self.timeout:
exitStatus = self.killChild()
self.__readFromCommand()
return self.__generateSystemCommandError(exitStatus,
"Timeout (%d seconds) for '%s' call" %
(self.timeout, cmdSeq))
time.sleep(0.01)
exitStatus = self.__poll(self.child.pid)
self.__readFromCommand()
if exitStatus:
exitStatus = exitStatus[1]
if exitStatus >= 256:
exitStatus /= 256
return S_OK((exitStatus, self.bufferList[0][0], self.bufferList[1][0]))
finally:
try:
self.child.stdout.close()
self.child.stderr.close()
except Exception:
pass
def getStorageElementName( self ): """ SE name getter """ self.log.getSubLogger( 'getStorageElementName' ).verbose( "The Storage Element name is %s." % self.name ) return S_OK( self.name )
def forceRefresh(self, fromMaster=False):
if self.__refreshEnabled:
return self.__refresh(fromMaster=fromMaster)
return S_OK()
def __findServiceURL(self):
"""
Discovers the URL of a service, taking into account gateways, multiple URLs, banned URLs
If the site on which we run is configured to use gateways (/DIRAC/Gateways/<siteName>),
these URLs will be used. To ignore the gateway, it is possible to set KW_IGNORE_GATEWAYS
to False in kwargs.
If self._destinationSrv (given as constructor attribute) is a properly formed URL,
we just return this one. If we have to use a gateway, we just replace the server name in the url.
The list of URLs defined in the CS (<System>/URLs/<Component>) is randomized
This method also sets some attributes:
* self.__nbOfUrls = number of URLs
* self.__nbOfRetry = 2 if we have more than 2 urls, otherwise 3
* self.__bannedUrls is reinitialized if all the URLs are banned
:return: the selected URL
"""
if not self.__initStatus['OK']:
return self.__initStatus
# Load the Gateways URLs for the current site Name
gatewayURL = False
if self.KW_IGNORE_GATEWAYS not in self.kwargs or not self.kwargs[self.KW_IGNORE_GATEWAYS]:
dRetVal = gConfig.getOption("/DIRAC/Gateways/%s" % DIRAC.siteName())
if dRetVal['OK']:
rawGatewayURL = List.randomize(List.fromChar(dRetVal['Value'], ","))[0]
gatewayURL = "/".join(rawGatewayURL.split("/")[:3])
# If what was given as constructor attribute is a properly formed URL,
# we just return this one.
# If we have to use a gateway, we just replace the server name in it
for protocol in gProtocolDict:
if self._destinationSrv.find("%s://" % protocol) == 0:
gLogger.debug("Already given a valid url", self._destinationSrv)
if not gatewayURL:
return S_OK(self._destinationSrv)
gLogger.debug("Reconstructing given URL to pass through gateway")
path = "/".join(self._destinationSrv.split("/")[3:])
finalURL = "%s/%s" % (gatewayURL, path)
gLogger.debug("Gateway URL conversion:\n %s -> %s" % (self._destinationSrv, finalURL))
return S_OK(finalURL)
if gatewayURL:
gLogger.debug("Using gateway", gatewayURL)
return S_OK("%s/%s" % (gatewayURL, self._destinationSrv))
# We extract the list of URLs from the CS (System/URLs/Component)
try:
urls = getServiceURL(self._destinationSrv, setup=self.setup)
except Exception as e:
return S_ERROR("Cannot get URL for %s in setup %s: %s" % (self._destinationSrv, self.setup, repr(e)))
if not urls:
return S_ERROR("URL for service %s not found" % self._destinationSrv)
failoverUrls = []
# Try if there are some failover URLs to use as last resort
try:
failoverUrlsStr = getServiceFailoverURL(self._destinationSrv, setup=self.setup)
if failoverUrlsStr:
failoverUrls = failoverUrlsStr.split(',')
except Exception as e:
pass
# We randomize the list, and add at the end the failover URLs (System/FailoverURLs/Component)
urlsList = List.randomize(List.fromChar(urls, ",")) + failoverUrls
self.__nbOfUrls = len(urlsList)
self.__nbOfRetry = 2 if self.__nbOfUrls > 2 else 3 # we retry 2 times all services, if we run more than 2 services
if self.__nbOfUrls == len(self.__bannedUrls):
self.__bannedUrls = [] # retry all urls
gLogger.debug("Retrying again all URLs")
if len(self.__bannedUrls) > 0 and len(urlsList) > 1:
# we have host which is not accessible. We remove that host from the list.
# We only remove if we have more than one instance
for i in self.__bannedUrls:
gLogger.debug("Removing banned URL", "%s" % i)
urlsList.remove(i)
# Take the first URL from the list
#randUrls = List.randomize( urlsList ) + failoverUrls
sURL = urlsList[0]
# If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL
# is not on a host which is banned. If it is, we take the next one in the list using __selectUrl
# If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL
# is not on a host which is banned. If it is, we take the next one in the list using __selectUrl
if len(self.__bannedUrls) > 0 and self.__nbOfUrls > 2: # when we have multiple services then we can
# have a situation when two services are running on the same machine with different ports...
retVal = Network.splitURL(sURL)
nexturl = None
if retVal['OK']:
nexturl = retVal['Value']
found = False
for i in self.__bannedUrls:
retVal = Network.splitURL(i)
if retVal['OK']:
bannedurl = retVal['Value']
else:
break
# We found a banned URL on the same host as the one we are running on
if nexturl[1] == bannedurl[1]:
found = True
break
if found:
nexturl = self.__selectUrl(nexturl, urlsList[1:])
if nexturl: # an url found which is in different host
sURL = nexturl
gLogger.debug("Discovering URL for service", "%s -> %s" % (self._destinationSrv, sURL))
return S_OK(sURL)
def exec_taskFreeze(cls, taskId, taskObj, eType):
return S_OK()
def handshake_singleStep(self):
""" Used to perform SSL handshakes.
These are now done automatically.
"""
# This isn't used any more, the handshake is done inside the M2Crypto library
return S_OK()
osSocket.close()
if retries:
return self.__socketConnect( hostAddress, timeout, retries - 1 )
else:
return S_ERROR( "Can't connect: %s" % str( e ) )
if e.args[0] not in ( 114, 115 ):
return S_ERROR( "Can't connect: %s" % str( e ) )
#Connect in progress
oL = select.select( [], [ osSocket ], [], timeout )[1]
if len( oL ) == 0:
osSocket.close()
return S_ERROR( "Connection timeout" )
errno = osSocket.getsockopt( socket.SOL_SOCKET, socket.SO_ERROR )
if errno != 0:
return S_ERROR( "Can't connect: %s" % str( ( errno, os.strerror( errno ) ) ) )
return S_OK( osSocket )
def __connect( self, socketInfo, hostAddress ):
#Connect baby!
result = self.__socketConnect( hostAddress, socketInfo.infoDict[ 'timeout' ] )
if not result[ 'OK' ]:
return result
osSocket = result[ 'Value' ]
#SSL MAGIC
sslSocket = GSI.SSL.Connection( socketInfo.getSSLContext(), osSocket )
#Generate sessionId
sessionHash = md5.md5()
sessionHash.update( str( hostAddress ) )
sessionHash.update( "|%s" % str( socketInfo.getLocalCredentialsLocation() ) )
for key in ( 'proxyLocation', 'proxyString' ):
if key in socketInfo.infoDict:
def getConfigurationTree(self, root='', *filters):
"""
Create a dictionary with all sections, subsections and options
starting from given root. Result can be filtered.
:param str root: Starting point in the configuration tree.
:param filters: Select results that contain given substrings (check full path, i.e. with option name)
:type filters: str or python:list[str]
:return: Return a dictionary where keys are paths taken from
the configuration (e.g. /Systems/Configuration/...).
Value is "None" when path points to a section
or not "None" if path points to an option.
"""
log = DIRAC.gLogger.getSubLogger('getConfigurationTree')
# check if root is an option (special case)
option = self.getOption(root)
if option['OK']:
result = {root: option['Value']}
else:
result = {root: None}
for substr in filters:
if substr not in root:
result = {}
break
# remove slashes at the end
root = root.rstrip('/')
# get options of current root
options = self.getOptionsDict(root)
if not options['OK']:
log.error("getOptionsDict() failed with message: %s" %
options['Message'])
return S_ERROR('Invalid root path provided')
for key, value in options['Value'].iteritems():
path = cfgPath(root, key)
addOption = True
for substr in filters:
if substr not in path:
addOption = False
break
if addOption:
result[path] = value
# get subsections of the root
sections = self.getSections(root)
if not sections['OK']:
log.error("getSections() failed with message: %s" %
sections['Message'])
return S_ERROR('Invalid root path provided')
# recursively go through subsections and get their subsections
for section in sections['Value']:
subtree = self.getConfigurationTree("%s/%s" % (root, section),
*filters)
if not subtree['OK']:
log.error(
"getConfigurationTree() failed with message: %s" %
sections['Message'])
return S_ERROR(
'Configuration was altered during the operation')
result.update(subtree['Value'])
return S_OK(result)
def ldapsearchBDII(filt=None,
attr=None,
host=None,
base=None,
selectionString="Glue"):
""" Python wrapper for ldapserch at bdii.
:param filt: Filter used to search ldap, default = '', means select all
:param attr: Attributes returned by ldapsearch, default = '*', means return all
:param host: Host used for ldapsearch, default = 'lcg-bdii.cern.ch:2170', can be changed by $LCG_GFAL_INFOSYS
:return: standard DIRAC answer with Value equals to list of ldapsearch responses
Each element of list is dictionary with keys:
'dn': Distinguished name of ldapsearch response
'objectClass': List of classes in response
'attr': Dictionary of attributes
"""
if filt is None:
filt = ''
if attr is None:
attr = ''
if host is None:
host = 'lcg-bdii.cern.ch:2170'
if base is None:
base = 'Mds-Vo-name=local,o=grid'
if isinstance(attr, list):
attr = ' '.join(attr)
cmd = 'ldapsearch -x -LLL -o ldif-wrap=no -h %s -b %s "%s" %s' % (
host, base, filt, attr)
result = shellCall(0, cmd)
response = []
if not result['OK']:
return result
status = result['Value'][0]
stdout = result['Value'][1]
stderr = result['Value'][2]
if status != 0:
return S_ERROR(stderr)
lines = []
for line in stdout.split("\n"):
if line.find(" ") == 0:
lines[-1] += line.strip()
else:
lines.append(line.strip())
record = None
for line in lines:
if line.find('dn:') == 0:
record = {
'dn': line.replace('dn:', '').strip(),
'objectClass': [],
'attr': {
'dn': line.replace('dn:', '').strip()
}
}
response.append(record)
continue
if record:
if line.find('objectClass:') == 0:
record['objectClass'].append(
line.replace('objectClass:', '').strip())
continue
if line.find(selectionString) == 0:
index = line.find(':')
if index > 0:
attr = line[:index]
value = line[index + 1:].strip()
if attr in record['attr']:
if isinstance(record['attr'][attr], list):
record['attr'][attr].append(value)
else:
record['attr'][attr] = [
record['attr'][attr], value
]
else:
record['attr'][attr] = value
return S_OK(response)
def applyModifications(self, modList, parentSection=""):
"""
Apply modifications to a CFG
:type modList: List
:param modList: Modifications from a getModifications call
:return: True/False
"""
for modAction in modList:
action = modAction[0]
key = modAction[1]
iPos = modAction[2]
value = modAction[3]
if action == 'addSec':
if key in self.listSections():
return S_ERROR("Section %s/%s already exists" %
(parentSection, key))
# key, value, comment, beforeKey = ""
value = CFG().loadFromBuffer(value)
comment = modAction[4].strip()
if iPos < len(self.__orderedList):
beforeKey = self.__orderedList[iPos]
else:
beforeKey = ""
self.addKey(key, value, comment, beforeKey)
elif action == 'delSec':
if key not in self.listSections():
return S_ERROR("Section %s/%s does not exist" %
(parentSection, key))
self.deleteKey(key)
elif action == 'modSec':
if key not in self.listSections():
return S_ERROR("Section %s/%s does not exist" %
(parentSection, key))
comment = modAction[4].strip()
self.setComment(key, comment)
if value:
result = self[key].applyModifications(
value, "%s/%s" % (parentSection, key))
if not result['OK']:
return result
if iPos >= len(
self.__orderedList) or key != self.__orderedList[iPos]:
prevPos = self.__orderedList.index(key)
del self.__orderedList[prevPos]
self.__orderedList.insert(iPos, key)
elif action == "addOpt":
if key in self.listOptions():
return S_ERROR("Option %s/%s exists already" %
(parentSection, key))
# key, value, comment, beforeKey = ""
comment = modAction[4].strip()
if iPos < len(self.__orderedList):
beforeKey = self.__orderedList[iPos]
else:
beforeKey = ""
self.addKey(key, value, comment, beforeKey)
elif action == "modOpt":
if key not in self.listOptions():
return S_ERROR("Option %s/%s does not exist" %
(parentSection, key))
comment = modAction[4].strip()
self.setOption(key, value, comment)
if iPos >= len(
self.__orderedList) or key != self.__orderedList[iPos]:
prevPos = self.__orderedList.index(key)
del (self.__orderedList[prevPos])
self.__orderedList.insert(iPos, key)
elif action == "delOpt":
if key not in self.listOptions():
return S_ERROR("Option %s/%s does not exist" %
(parentSection, key))
self.deleteKey(key)
return S_OK()
def getBdiiCEInfo(vo, host=None, glue2=False):
""" Get information for all the CEs/queues for a given VO
:param str vo: BDII VO name
:param str host: url to query for information
:param bool glue2: if True query the GLUE2 information schema
:return: result structure: result['Value'][siteID]['CEs'][ceID]['Queues'][queueName]. For
each siteID, ceID, queueName all the BDII/Glue parameters are retrieved
"""
if glue2:
return Glue2.getGlue2CEInfo(vo, host=host)
result = ldapCEState('', vo, host=host)
if not result['OK']:
return result
siteDict = {}
ceDict = {}
queueDict = {}
for queue in result['Value']:
queue = dict(queue)
clusterID = queue.get('GlueForeignKey',
'').replace('GlueClusterUniqueID=', '')
ceID = queue.get('GlueCEUniqueID', '').split(':')[0]
queueDict[queue['GlueCEUniqueID']] = queue
queueDict[queue['GlueCEUniqueID']]['CE'] = ceID
if ceID not in ceDict:
result = ldapCluster(clusterID, host=host)
if not result['OK']:
continue
if not result['Value']:
continue
ce = result['Value'][0]
ceDict[ceID] = ce
fKey = ce['GlueForeignKey'] # pylint: disable=unsubscriptable-object
siteID = ''
for key in fKey:
if key.startswith('GlueSiteUniqueID'):
siteID = key.replace('GlueSiteUniqueID=', '')
ceDict[ceID]['Site'] = siteID
result = ldapCE(clusterID, host=host)
ce = {}
if result['OK'] and result['Value']:
ce = result['Value'][0]
ceDict[ceID].update(ce)
if siteID not in siteDict:
site = {}
result = ldapSite(siteID, host=host)
if result['OK'] and result['Value']:
site = result['Value'][0]
siteDict[siteID] = site
for ceID in ceDict:
siteID = ceDict[ceID]['Site']
if siteID in siteDict:
siteDict[siteID].setdefault('CEs', {})
siteDict[siteID]['CEs'][ceID] = ceDict[ceID]
for queueID in queueDict:
ceID = queueDict[queueID]['CE']
siteID = ceDict[ceID]['Site']
siteDict[siteID]['CEs'][ceID].setdefault('Queues', {})
queueName = re.split(r':\d+/', queueDict[queueID]['GlueCEUniqueID'])[1]
siteDict[siteID]['CEs'][ceID]['Queues'][queueName] = queueDict[queueID]
return S_OK(siteDict)
def __executeMethod(self, lfn, *args, **kwargs):
""" Forward the call to each storage in turn until one works.
The method to be executed is stored in self.methodName
:param lfn : string, list or dictionary
:param *args : variable amount of non-keyword arguments. SHOULD BE EMPTY
:param **kwargs : keyword arguments
:returns S_OK( { 'Failed': {lfn : reason} , 'Successful': {lfn : value} } )
The Failed dict contains the lfn only if the operation failed on all the storages
The Successful dict contains the value returned by the successful storages.
A special kwargs is 'inputProtocol', which can be specified for putFile. It describes
the protocol used as source protocol, since there is in principle only one.
"""
removedArgs = {}
log = self.log.getSubLogger('__executeMethod')
log.verbose("preparing the execution of %s" % (self.methodName))
# args should normaly be empty to avoid problem...
if len(args):
log.verbose("args should be empty!%s" % args)
# because there is normally only one kw argument, I can move it from args to kwargs
methDefaultArgs = StorageElementItem.__defaultsArguments.get(
self.methodName, {}).keys()
if len(methDefaultArgs):
kwargs[methDefaultArgs[0]] = args[0]
args = args[1:]
log.verbose(
"put it in kwargs, but dirty and might be dangerous!args %s kwargs %s"
% (args, kwargs))
# We check the deprecated arguments
for depArg in StorageElementItem.__deprecatedArguments:
if depArg in kwargs:
log.verbose(
"%s is not an allowed argument anymore. Please change your code!"
% depArg)
removedArgs[depArg] = kwargs[depArg]
del kwargs[depArg]
# Set default argument if any
methDefaultArgs = StorageElementItem.__defaultsArguments.get(
self.methodName, {})
for argName in methDefaultArgs:
if argName not in kwargs:
log.debug("default argument %s for %s not present.\
Setting value %s." %
(argName, self.methodName, methDefaultArgs[argName]))
kwargs[argName] = methDefaultArgs[argName]
res = checkArgumentFormat(lfn)
if not res['OK']:
errStr = "Supplied lfns must be string, list of strings or a dictionary."
log.debug(errStr)
return res
lfnDict = res['Value']
log.verbose("Attempting to perform '%s' operation with %s lfns." %
(self.methodName, len(lfnDict)))
res = self.isValid(operation=self.methodName)
if not res['OK']:
return res
else:
if not self.valid:
return S_ERROR(self.errorReason)
# In case executing putFile, we can assume that all the source urls
# are from the same protocol. This optional parameter, if defined
# can be used to ignore some storage plugins and thus save time
# and avoid fake failures showing in the accounting
inputProtocol = kwargs.pop('inputProtocol', None)
successful = {}
failed = {}
filteredPlugins = self.__filterPlugins(self.methodName,
kwargs.get('protocols'),
inputProtocol)
if not filteredPlugins:
return S_ERROR( errno.EPROTONOSUPPORT, "No storage plugins matching the requirements\
(operation %s protocols %s inputProtocol %s)" \
% ( self.methodName, kwargs.get( 'protocols' ), inputProtocol ) )
# Try all of the storages one by one
for storage in filteredPlugins:
# Determine whether to use this storage object
storageParameters = storage.getParameters()
pluginName = storageParameters['PluginName']
if not lfnDict:
log.debug("No lfns to be attempted for %s protocol." %
pluginName)
continue
log.verbose("Generating %s protocol URLs for %s." %
(len(lfnDict), pluginName))
replicaDict = kwargs.pop('replicaDict', {})
if storage.pluginName != "Proxy":
res = self.__generateURLDict(lfnDict,
storage,
replicaDict=replicaDict)
urlDict = res['Value']['Successful'] # url : lfn
failed.update(res['Value']['Failed'])
else:
urlDict = dict([(lfn, lfn) for lfn in lfnDict])
if not len(urlDict):
log.verbose(
"__executeMethod No urls generated for protocol %s." %
pluginName)
else:
log.verbose(
"Attempting to perform '%s' for %s physical files" %
(self.methodName, len(urlDict)))
fcn = None
if hasattr(storage, self.methodName) and callable(
getattr(storage, self.methodName)):
fcn = getattr(storage, self.methodName)
if not fcn:
return S_ERROR(
DErrno.ENOMETH,
"SE.__executeMethod: unable to invoke %s, it isn't a member function of storage"
)
urlsToUse = {
} # url : the value of the lfn dictionary for the lfn of this url
for url in urlDict:
urlsToUse[url] = lfnDict[urlDict[url]]
startDate = datetime.datetime.utcnow()
startTime = time.time()
res = fcn(urlsToUse, *args, **kwargs)
elapsedTime = time.time() - startTime
self.addAccountingOperation(urlsToUse, startDate, elapsedTime,
storageParameters, res)
if not res['OK']:
errStr = "Completely failed to perform %s." % self.methodName
log.debug(
errStr,
'with plugin %s: %s' % (pluginName, res['Message']))
for lfn in urlDict.values():
if lfn not in failed:
failed[lfn] = ''
failed[lfn] = "%s %s" % (
failed[lfn],
res['Message']) if failed[lfn] else res['Message']
else:
for url, lfn in urlDict.items():
if url not in res['Value']['Successful']:
if lfn not in failed:
failed[lfn] = ''
if url in res['Value']['Failed']:
self.log.debug(res['Value']['Failed'][url])
failed[lfn] = "%s %s" % (
failed[lfn],
res['Value']['Failed'][url]) if failed[
lfn] else res['Value']['Failed'][url]
else:
errStr = 'No error returned from plug-in'
failed[lfn] = "%s %s" % (
failed[lfn],
errStr) if failed[lfn] else errStr
else:
successful[lfn] = res['Value']['Successful'][url]
if lfn in failed:
failed.pop(lfn)
lfnDict.pop(lfn)
gDataStoreClient.commit()
return S_OK({'Failed': failed, 'Successful': successful})
def getStatus(self):
"""
Return Status of the SE, a dictionary with:
- Read: True (is allowed), False (it is not allowed)
- Write: True (is allowed), False (it is not allowed)
- Remove: True (is allowed), False (it is not allowed)
- Check: True (is allowed), False (it is not allowed).
NB: Check always allowed IF Read is allowed (regardless of what set in the Check option of the configuration)
- DiskSE: True if TXDY with Y > 0 (defaults to True)
- TapeSE: True if TXDY with X > 0 (defaults to False)
- TotalCapacityTB: float (-1 if not defined)
- DiskCacheTB: float (-1 if not defined)
"""
self.log.getSubLogger('getStatus').verbose(
"determining status of %s." % self.name)
retDict = {}
if not self.valid:
retDict['Read'] = False
retDict['Write'] = False
retDict['Remove'] = False
retDict['Check'] = False
retDict['DiskSE'] = False
retDict['TapeSE'] = False
retDict['TotalCapacityTB'] = -1
retDict['DiskCacheTB'] = -1
return S_OK(retDict)
# If nothing is defined in the CS Access is allowed
# If something is defined, then it must be set to Active
retDict['Read'] = not ('ReadAccess' in self.options
and self.options['ReadAccess']
not in ('Active', 'Degraded'))
retDict['Write'] = not ('WriteAccess' in self.options
and self.options['WriteAccess']
not in ('Active', 'Degraded'))
retDict['Remove'] = not ('RemoveAccess' in self.options
and self.options['RemoveAccess']
not in ('Active', 'Degraded'))
if retDict['Read']:
retDict['Check'] = True
else:
retDict['Check'] = not ('CheckAccess' in self.options
and self.options['CheckAccess']
not in ('Active', 'Degraded'))
diskSE = True
tapeSE = False
if 'SEType' in self.options:
# Type should follow the convention TXDY
seType = self.options['SEType']
diskSE = re.search('D[1-9]', seType) != None
tapeSE = re.search('T[1-9]', seType) != None
retDict['DiskSE'] = diskSE
retDict['TapeSE'] = tapeSE
try:
retDict['TotalCapacityTB'] = float(self.options['TotalCapacityTB'])
except Exception:
retDict['TotalCapacityTB'] = -1
try:
retDict['DiskCacheTB'] = float(self.options['DiskCacheTB'])
except Exception:
retDict['DiskCacheTB'] = -1
return S_OK(retDict)
def exec_executorConnected(cls, execName, trid):
return S_OK()
def initializeConfigurationHandler(serviceInfo):
global gServiceInterface
gServiceInterface = ServiceInterface(serviceInfo["URL"])
return S_OK()
def exec_prepareToSend(cls, taskId, taskObj, eId):
return S_OK()
def export_getVersion(cls):
return S_OK(gServiceInterface.getVersion())
def close(self):
# pylint: disable=line-too-long
""" Close this socket. """
if self.oSocket:
# TL;DR:
# Do NOT touch that method
#
# Surprisingly (to me at least), M2Crypto does not close
# the underlying socket when calling SSL.Connection.close
# It only does it when the garbage collector kicks in (see ~M2Crypto.SSL.Connection.Connection.__del__)
# If the socket is not closed, the connection may hang forever.
#
# Thus, we are setting self.oSocket to None to allow the GC to do the work, but since we are not sure
# that it will run, we anyway force the connection to be closed
#
# However, we should close the underlying socket only after SSL was shutdown properly.
# This is because OpenSSL `ssl3_shutdown` (see callstack below) may still read some data
# (see https://github.com/openssl/openssl/blob/master/ssl/s3_lib.c#L4509)::
#
#
# 1 0x00007fffe9d48fc0 in sock_read () from /lib/libcrypto.so.1.0.0
# 2 0x00007fffe9d46e83 in BIO_read () from /lib/libcrypto.so.1.0.0
# 3 0x00007fffe9eab9dd in ssl3_read_n () from /lib/libssl.so.1.0.0
# 4 0x00007fffe9ead216 in ssl3_read_bytes () from /lib/libssl.so.1.0.0
# 5 0x00007fffe9ea999c in ssl3_shutdown () from /lib/libssl.so.1.0.0
# 6 0x00007fffe9ed4f93 in ssl_free () from /lib/libssl.so.1.0.0
# 7 0x00007fffe9d46d5b in BIO_free () from /lib/libcrypto.so.1.0.0
# 8 0x00007fffe9f30a96 in bio_free (bio=0x5555556f3200) at SWIG/_m2crypto_wrap.c:5008
# 9 0x00007fffe9f30b1e in _wrap_bio_free (self=<optimized out>, args=<optimized out>) at SWIG/_m2crypto_wrap.c
#
# We unfortunately have no way to force that order, and there is a risk of deadlock
# when running in a multi threaded environment like the agents::
#
# Thread A opens socket, gets FD = 111
# Thread A works on it
# Thread A closes FD 111 (underlying socket.close())
# Thread B opens socket, gets FD = 111
# Thread A calls read on FD=111 from ssl3_shutdown
#
# This is illustrated on the strace below::
#
# 26461 14:25:15.266692 write(111<TCPv6:[[<srcAddressV6>]:42688->[<dstAddressV6>]:9140]>,
# "blabla", 37 <unfinished ...>
# 26464 14:25:15.266857 <... connect resumed>) = 0 <0.000195>
# 26464 14:25:15.267023 getsockname(120<UDP:[<srcAddress>:44252->188.185.84.86:9140]>, <unfinished ...>
# 26461 14:25:15.267176 <... write resumed>) = 37 <0.000453>
# 26464 14:25:15.267425 <... getsockname resumed>{sa_family=AF_INET, sin_port=htons(44252),
# sin_addr=inet_addr("<srcAddress>")}, [28->16]) = 0 <0.000292>
# 26461 14:25:15.267466 close(111<TCPv6:[[<srcAddressV6>]:42688->[<dstAddressV6>]:9140]> <unfinished ...>
# 26464 14:25:15.267637 close(120<UDP:[<srcAddress>:44252->188.185.84.86:9140]> <unfinished ...>
# 26464 14:25:15.267738 <... close resumed>) = 0 <0.000086>
# 26461 14:25:15.267768 <... close resumed>) = 0 <0.000285>
# 26464 14:25:15.267827 socket(AF_INET6, SOCK_DGRAM|SOCK_CLOEXEC, IPPROTO_IP <unfinished ...>
# 26461 14:25:15.267888 futex(0x21f8620, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
# 26464 14:25:15.267976 <... socket resumed>) = 111<UDPv6:[1207802822]> <0.000138>
# 26461 14:25:15.268092 <... futex resumed>) = 1 <0.000196>
# 26464 14:25:15.268195 connect(111<UDPv6:[1207802822]>,
# {sa_family=AF_INET6, sin6_port=htons(9140),
# inet_pton(AF_INET6, "<dstAddressV6>", &sin6_addr),
# sin6_flowinfo=htonl(0), sin6_scope_id=0
# }, 28 <unfinished ...>
# 26461 14:25:15.268294 read(111<UDPv6:[[<srcAddressV6>]:42480->[<dstAddressV6>]:9140]>, <unfinished ...>
# 26464 14:25:15.268503 <... connect resumed>) = 0 <0.000217>
# 26464 14:25:15.268673 getsockname(111<UDPv6:[[<srcAddressV6>]:42480->[<dstAddressV6>]:9140]>, <unfinished ...>
# 26464 14:25:15.268862 <... getsockname resumed>{sa_family=AF_INET6, sin6_port=htons(42480),
# inet_pton(AF_INET6, "<srcAddressV6>", &sin6_addr), sin6_flowinfo=htonl(0), sin6_scope_id=
# 0}, [28]) = 0 <0.000168>
# 26464 14:25:15.269048
# close(111<UDPv6:[[<srcAddressV6>]:42480->[<dstAddressV6>]:9140]>
# <unfinished ...>
#
#
# Update 16.07.20:
# M2Crypto 0.36 contains the bug fix https://gitlab.com/m2crypto/m2crypto/-/merge_requests/247
# that allows proper closing. So manual closing of the underlying socket should not be needed anymore
# Update 16.07.20
# I add this shutdown call without being 100% sure
# it solves some hanging connections issues, but it seems
# to work. it does not appear in any M2Crypto doc, but comparing
# some internals of M2Crypto and official python SSL library,
# it seems to make sense
self.oSocket.shutdown(socket.SHUT_RDWR)
# Update 16.07.20
# With freeBio=True, we force the
# closing of the socket before the GC runs
self.oSocket.close(freeBio=True)
# underlyingSocket = self.oSocket.socket
self.oSocket = None
# underlyingSocket.close()
return S_OK()
def export_getCompressedData(cls):
sData = gServiceInterface.getCompressedConfigurationData()
return S_OK(sData)
def enable(self):
self.__refreshEnabled = True
if self.__lastRefreshExpired():
return self.forceRefresh()
return S_OK()
def export_publishSlaveServer(cls, sURL):
gServiceInterface.publishSlaveServer(sURL)
return S_OK()
def __doFileTransfer(self, sDirection):
"""
Execute a file transfer action
:type sDirection: string
:param sDirection: Direction of the transfer
:return: S_OK/S_ERROR
"""
retVal = self.__trPool.receive(self.__trid)
if not retVal["OK"]:
raise ConnectionError(
"Error while receiving file description %s %s" %
(self.srv_getFormattedRemoteCredentials(), retVal["Message"]))
# Reconvert to tuple
fileInfo = tuple(retVal["Value"])
sDirection = "%s%s" % (sDirection[0].lower(), sDirection[1:])
if "transfer_%s" % sDirection not in dir(self):
self.__trPool.send(
self.__trid,
S_ERROR("Service can't transfer files %s" % sDirection))
return
retVal = self.__trPool.send(self.__trid, S_OK("Accepted"))
if not retVal["OK"]:
return retVal
self.__logRemoteQuery("FileTransfer/%s" % sDirection, fileInfo)
self.__lockManager.lock("FileTransfer/%s" % sDirection)
try:
try:
fileHelper = FileHelper(self.__trPool.get(self.__trid))
if sDirection == "fromClient":
fileHelper.setDirection("fromClient")
uRetVal = self.transfer_fromClient(fileInfo[0],
fileInfo[1],
fileInfo[2], fileHelper)
elif sDirection == "toClient":
fileHelper.setDirection("toClient")
uRetVal = self.transfer_toClient(fileInfo[0], fileInfo[1],
fileHelper)
elif sDirection == "bulkFromClient":
fileHelper.setDirection("fromClient")
uRetVal = self.transfer_bulkFromClient(
fileInfo[0], fileInfo[1], fileInfo[2], fileHelper)
elif sDirection == "bulkToClient":
fileHelper.setDirection("toClient")
uRetVal = self.transfer_bulkToClient(
fileInfo[0], fileInfo[1], fileHelper)
elif sDirection == "listBulk":
fileHelper.setDirection("toClient")
uRetVal = self.transfer_listBulk(fileInfo[0], fileInfo[1],
fileHelper)
else:
return S_ERROR("Direction %s does not exist!!!" %
sDirection)
if uRetVal["OK"] and not fileHelper.finishedTransmission():
gLogger.error(
"You haven't finished receiving/sending the file",
str(fileInfo))
return S_ERROR("Incomplete transfer")
del fileHelper
return uRetVal
finally:
self.__lockManager.unlock("FileTransfer/%s" % sDirection)
except Exception as e: # pylint: disable=broad-except
gLogger.exception("Uncaught exception when serving Transfer",
"%s" % sDirection,
lException=e)
return S_ERROR("Server error while serving %s: %s" %
(sDirection, repr(e)))
def export_writeEnabled(cls):
return S_OK(gServiceInterface.isMaster())
def _connect(self):
""" Establish the connection.
It uses the URL discovered in __discoverURL.
In case the connection cannot be established, __discoverURL
is called again, and _connect calls itself.
We stop after trying self.__nbOfRetry * self.__nbOfUrls
"""
# Check if the useServerCertificate configuration changed
# Note: I am not really sure that all this block makes
# any sense at all since all these variables are
# evaluated in __discoverCredentialsToUse
if gConfig.useServerCertificate() != self.__useCertificates:
if self.__forceUseCertificates is None:
self.__useCertificates = gConfig.useServerCertificate()
self.kwargs[self.KW_USE_CERTIFICATES] = self.__useCertificates
# The server certificate use context changed, rechecking the transport sanity
result = self.__checkTransportSanity()
if not result['OK']:
return result
# Take all the extra credentials
self.__discoverExtraCredentials()
if not self.__initStatus['OK']:
return self.__initStatus
if self.__enableThreadCheck:
self.__checkThreadID()
gLogger.debug("Connecting to: %s" % self.serviceURL)
try:
# Calls the transport method of the apropriate protocol.
# self.__URLTuple[1:3] = [server name, port, System/Component]
transport = gProtocolDict[self.__URLTuple[0]]['transport'](self.__URLTuple[1:3], **self.kwargs)
# the socket timeout is the default value which is 1.
# later we increase to 5
retVal = transport.initAsClient()
# If we have an issue connecting
if not retVal['OK']:
# We try at most __nbOfRetry each URLs
if self.__retry < self.__nbOfRetry * self.__nbOfUrls - 1:
# Recompose the URL (why not using self.serviceURL ? )
url = "%s://%s:%d/%s" % (self.__URLTuple[0], self.__URLTuple[1], int(self.__URLTuple[2]), self.__URLTuple[3])
# Add the url to the list of banned URLs if it is not already there. (Can it happen ? I don't think so)
if url not in self.__bannedUrls:
self.__bannedUrls += [url]
# Why only printing in this case ?
if len(self.__bannedUrls) < self.__nbOfUrls:
gLogger.notice("Non-responding URL temporarily banned", "%s" % url)
# Increment the retry couunter
self.__retry += 1
# If it is our last attempt for each URL, we increase the timeout
if self.__retryCounter == self.__nbOfRetry - 1:
transport.setSocketTimeout(5) # we increase the socket timeout in case the network is not good
gLogger.info("Retry connection", ": %d to %s" % (self.__retry, self.serviceURL))
# If we tried all the URL, we increase the global counter (__retryCounter), and sleep
if len(self.__bannedUrls) == self.__nbOfUrls:
self.__retryCounter += 1
# we run only one service! In that case we increase the retry delay.
self.__retryDelay = 3. / self.__nbOfUrls if self.__nbOfUrls > 1 else 2
gLogger.info("Waiting %f seconds before retry all service(s)" % self.__retryDelay)
time.sleep(self.__retryDelay)
# rediscover the URL
self.__discoverURL()
# try to reconnect
return self._connect()
else:
return retVal
except Exception as e:
gLogger.exception(lException=True, lExcInfo=True)
return S_ERROR("Can't connect to %s: %s" % (self.serviceURL, repr(e)))
# We add the connection to the transport pool
trid = getGlobalTransportPool().add(transport)
return S_OK((trid, transport))
def conn_drop(self, trid):
self.__eDispatch.removeExecutor(trid)
return S_OK()
class SocketInfoFactory:
def __init__(self):
self.__timeout = 1
def setSocketTimeout(self, timeout):
self.__timeout = timeout
def getSocketTimeout(self):
return self.__timeout
def generateClientInfo(self, destinationHostname, kwargs):
infoDict = {
'clientMode': True,
'hostname': destinationHostname,
'timeout': 600,
'enableSessions': True
}
for key in kwargs.keys():
infoDict[key] = kwargs[key]
try:
return S_OK(SocketInfo(infoDict))
except Exception as e:
return S_ERROR("Error while creating SSL context: %s" % str(e))
def generateServerInfo(self, kwargs):
infoDict = {'clientMode': False, 'timeout': 30}
for key in kwargs.keys():
infoDict[key] = kwargs[key]
try:
return S_OK(SocketInfo(infoDict))
except Exception as e:
return S_ERROR(str(e))
def __socketConnect(self, hostAddress, timeout, retries=2):
addrs = socket.getaddrinfo(hostAddress[0], hostAddress[1], 0,
socket.SOCK_STREAM)
errs = []
for addr in addrs:
res = self.__sockConnect(addr[4], addr[0], timeout, retries)
if res['OK']:
return res
else:
errs.append(res['Message'])
return S_ERROR(", ".join(errs))
def __sockConnect(self, hostAddress, sockType, timeout, retries):
try:
osSocket = socket.socket(sockType, socket.SOCK_STREAM)
except socket.error as e:
gLogger.warn("Exception while creating a socket:", str(e))
return S_ERROR("Exception while creating a socket:%s" % str(e))
# osSocket.setblocking( 0 )
if timeout:
tsocket = self.getSocketTimeout()
gLogger.verbose("Connection timeout set to: ", tsocket)
osSocket.settimeout(
tsocket) # we try to connect 3 times with 1 second timeout
try:
osSocket.connect(hostAddress)
except socket.error, e:
if e.args[0] == "timed out":
osSocket.close()
if retries:
return self.__sockConnect(hostAddress, sockType, timeout,
retries - 1)
else:
return S_ERROR("Can't connect: %s" % str(e))
if e.args[0] not in (114, 115):
return S_ERROR("Can't connect: %s" % str(e))
#Connect in progress
oL = select.select([], [osSocket], [], timeout)[1]
if len(oL) == 0:
osSocket.close()
return S_ERROR("Connection timeout")
errno = osSocket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
if errno != 0:
return S_ERROR("Can't connect: %s" % str(
(errno, os.strerror(errno))))
return S_OK(osSocket)
def exec_executorDisconnected(cls, trid):
return S_OK()
def isValid( self, operation = '' ):
""" check CS/RSS statuses for :operation:
:param str operation: operation name
"""
log = self.log.getSubLogger( 'isValid', True )
log.verbose( "Determining if the StorageElement %s is valid for VO %s" % ( self.name, self.vo ) )
if not self.valid:
log.debug( "Failed to create StorageElement plugins.", self.errorReason )
return S_ERROR( "SE.isValid: Failed to create StorageElement plugins: %s" % self.errorReason )
# Check if the Storage Element is eligible for the user's VO
if 'VO' in self.options and not self.vo in self.options['VO']:
log.debug( "StorageElement is not allowed for VO", self.vo )
return S_ERROR( errno.EACCES, "StorageElement.isValid: StorageElement is not allowed for VO" )
log.verbose( "Determining if the StorageElement %s is valid for %s" % ( self.name, operation ) )
if ( not operation ) or ( operation in self.okMethods ):
return S_OK()
# Determine whether the StorageElement is valid for checking, reading, writing
res = self.getStatus()
if not res[ 'OK' ]:
log.debug( "Could not call getStatus", res['Message'] )
return S_ERROR( "SE.isValid could not call the getStatus method" )
checking = res[ 'Value' ][ 'Check' ]
reading = res[ 'Value' ][ 'Read' ]
writing = res[ 'Value' ][ 'Write' ]
removing = res[ 'Value' ][ 'Remove' ]
# Determine whether the requested operation can be fulfilled
if ( not operation ) and ( not reading ) and ( not writing ) and ( not checking ):
log.debug( "Read, write and check access not permitted." )
return S_ERROR( errno.EACCES, "SE.isValid: Read, write and check access not permitted." )
# The supplied operation can be 'Read','Write' or any of the possible StorageElement methods.
if ( operation in self.readMethods ) or ( operation.lower() in ( 'read', 'readaccess' ) ):
operation = 'ReadAccess'
elif operation in self.writeMethods or ( operation.lower() in ( 'write', 'writeaccess' ) ):
operation = 'WriteAccess'
elif operation in self.removeMethods or ( operation.lower() in ( 'remove', 'removeaccess' ) ):
operation = 'RemoveAccess'
elif operation in self.checkMethods or ( operation.lower() in ( 'check', 'checkaccess' ) ):
operation = 'CheckAccess'
else:
log.debug( "The supplied operation is not known.", operation )
return S_ERROR( DErrno.ENOMETH , "SE.isValid: The supplied operation is not known." )
log.debug( "check the operation: %s " % operation )
# Check if the operation is valid
if operation == 'CheckAccess':
if not reading:
if not checking:
log.debug( "Check access not currently permitted." )
return S_ERROR( errno.EACCES, "SE.isValid: Check access not currently permitted." )
if operation == 'ReadAccess':
if not reading:
log.debug( "Read access not currently permitted." )
return S_ERROR( errno.EACCES, "SE.isValid: Read access not currently permitted." )
if operation == 'WriteAccess':
if not writing:
log.debug( "Write access not currently permitted." )
return S_ERROR( errno.EACCES, "SE.isValid: Write access not currently permitted." )
if operation == 'RemoveAccess':
if not removing:
log.debug( "Remove access not currently permitted." )
return S_ERROR( errno.EACCES, "SE.isValid: Remove access not currently permitted." )
return S_OK()
# else check the parameter
elif not isinstance(args[iIndex], oTypesList[iIndex]):
mismatch = True
# Has there been a mismatch?
if mismatch:
sError = "Type mismatch in parameter %d (starting with param 0) Received %s, expected %s" % (
iIndex, type(args[iIndex]), str(oTypesList[iIndex]))
return S_ERROR(sError)
if len(args) < len(oTypesList):
return S_ERROR("Function %s expects at least %s arguments" %
(method, len(oTypesList)))
except Exception, v:
sError = "Error in parameter check: %s" % str(v)
gLogger.exception(sError)
return S_ERROR(sError)
return S_OK()
####
#
# Connection methods
#
####
__connectionCallbackTypes = {
'new': [types.StringTypes, types.DictType],
'connected': [],
'drop': []
}
def __doConnection(self, methodName):
"""
