def setParameters( self, parameters ):
""" Applying extra storage parameters
"""
StorageBase.setParameters( self, parameters )
if "CheckSum" in parameters and parameters['CheckSum'].lower() in ['0', 'no', 'false', 'off']:
self.checkSum = "NoCheckSum"
return S_OK()
def setParameters( self, parameters ):
""" Applying extra storage parameters
"""
StorageBase.setParameters( self, parameters )
if "CheckSum" in parameters and parameters['CheckSum'].lower() in ['0', 'no', 'false', 'off']:
self.checkSum = "NoCheckSum"
return S_OK()
def __init__( self, storageName, parameters ):
StorageBase.__init__( self, storageName, parameters )
self.pluginName = 'Proxy'
self.isok = True
self.url = PathFinder.getServiceURL( "DataManagement/StorageElementProxy" )
if not self.url:
self.isok = False
def __init__( self, storageName, parameters ):
StorageBase.__init__( self, storageName, parameters )
self.pluginName = 'Proxy'
self.isok = True
self.url = PathFinder.getServiceURL( "DataManagement/StorageElementProxy" )
if not self.url:
self.isok = False
def mock_StorageFactory_generateStorageObject(self,
storageName,
pluginName,
parameters,
hideExceptions=False): # pylint: disable=unused-argument
"""Generate fake storage object"""
storageObj = StorageBase(storageName, parameters)
storageObj.pluginName = pluginName
return S_OK(storageObj)
def mock_StorageFactory_generateStorageObject(storageName,
pluginName,
parameters,
hideExceptions=False):
""" Generate fake storage object"""
storageObj = StorageBase(storageName, parameters)
if pluginName == "SRM2":
storageObj = fake_SRM2Plugin(storageName, parameters)
storageObj.protocolParameters['InputProtocols'] = [
'file', 'root', 'srm'
]
storageObj.protocolParameters['OutputProtocols'] = [
'file', 'root', 'dcap', 'gsidcap', 'rfio', 'srm'
]
elif pluginName == "File":
# Not needed to do anything, StorageBase should do it :)
pass
elif pluginName == 'XROOT':
storageObj = fake_XROOTPlugin(storageName, parameters)
storageObj.protocolParameters['InputProtocols'] = ['file', 'root']
storageObj.protocolParameters['OutputProtocols'] = ['root']
elif pluginName == 'GSIFTP':
storageObj = fake_GSIFTPPlugin(storageName, parameters)
storageObj.protocolParameters['InputProtocols'] = ['file', 'gsiftp']
storageObj.protocolParameters['OutputProtocols'] = ['gsiftp']
storageObj.pluginName = pluginName
return S_OK(storageObj)
def mock_StorageFactory_generateStorageObject(storageName,
pluginName,
parameters,
hideExceptions=False):
"""Generate fake storage object"""
storageObj = StorageBase(storageName, parameters)
if pluginName == "SRM2":
storageObj = fake_SRM2Plugin(storageName, parameters)
storageObj.protocolParameters["InputProtocols"] = [
"file", "root", "srm"
]
storageObj.protocolParameters["OutputProtocols"] = [
"file", "root", "dcap", "gsidcap", "rfio", "srm"
]
elif pluginName == "File":
# Not needed to do anything, StorageBase should do it :)
pass
elif pluginName == "XROOT":
storageObj = fake_XROOTPlugin(storageName, parameters)
storageObj.protocolParameters["InputProtocols"] = ["file", "root"]
storageObj.protocolParameters["OutputProtocols"] = ["root"]
elif pluginName == "GSIFTP":
storageObj = fake_GSIFTPPlugin(storageName, parameters)
storageObj.protocolParameters["InputProtocols"] = ["file", "gsiftp"]
storageObj.protocolParameters["OutputProtocols"] = ["gsiftp"]
storageObj.pluginName = pluginName
return S_OK(storageObj)
def __init__(self, storageName, parameters):
""" c'tor
:param self: self reference
:param str storageName: SE name
"""
StorageBase.__init__(self, storageName, parameters)
self.log = gLogger.getSubLogger(__name__, True)
self.pluginName = 'FCOnly'
self.protocol = []
def __init__( self, storageName, parameters ):
StorageBase.__init__( self, storageName, parameters )
self.spaceToken = self.protocolParameters['SpaceToken']
self.isok = True
self.pluginName = 'RFIO'
self.timeout = 100
self.long_timeout = 600
def __init__(self, storageName, parameters):
StorageBase.__init__(self, storageName, parameters)
self.spaceToken = self.protocolParameters['SpaceToken']
self.isok = True
self.pluginName = 'RFIO'
self.timeout = 100
self.long_timeout = 600
def __getFileStat(self, path):
"""
Get the file stat information
"""
conn, error, userDict = self.__irodsClient()
if not conn:
return S_ERROR(error)
file_path = self.__resolveFileID(path, userDict)
irodsHome = userDict.get('iRodsHome', IRODS_HOME)
file_path = irodsHome + file_path
gLogger.debug("file_path to read: %s" % file_path)
resultDict = {}
irodsFile = irodsOpen(conn, file_path, "r")
if irodsFile:
resultDict['Exists'] = True
resultDict['File'] = True
resultDict['Directory'] = False
resultDict['Type'] = "File"
resultDict['Size'] = irodsFile.getSize()
resultDict['TimeStamps'] = (irodsFile.getCreateTs(),
irodsFile.getModifyTs(),
irodsFile.getCreateTs())
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = 0o755
resultDict = StorageBase._addCommonMetadata(resultDict)
return S_OK(resultDict)
else:
coll = irodsCollection(conn, file_path)
if coll:
resultDict['Exists'] = True
resultDict['Type'] = "Directory"
resultDict['File'] = False
resultDict['Directory'] = True
resultDict['Size'] = 0
resultDict['TimeStamps'] = (0, 0, 0)
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = 0o755
resultDict = StorageBase._addCommonMetadata(resultDict)
return S_OK(resultDict)
else:
return S_ERROR('Path does not exist')
def __getFileStat( self, path ):
"""
Get the file stat information
"""
conn , error, userDict = self.__irodsClient( )
if not conn:
return S_ERROR( error )
file_path = self.__resolveFileID( path, userDict )
irodsHome = userDict.get( 'iRodsHome', IRODS_HOME )
file_path = irodsHome + file_path
gLogger.debug( "file_path to read: %s" % file_path )
resultDict = {}
irodsFile = irodsOpen( conn , file_path , "r" )
if irodsFile:
resultDict['Exists'] = True
resultDict['File'] = True
resultDict['Directory'] = False
resultDict['Type'] = "File"
resultDict['Size'] = irodsFile.getSize()
resultDict['TimeStamps'] = ( irodsFile.getCreateTs(), irodsFile.getModifyTs(), irodsFile.getCreateTs() )
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = 0o755
resultDict = StorageBase._addCommonMetadata( resultDict )
return S_OK( resultDict )
else:
coll = irodsCollection( conn, file_path )
if coll:
resultDict['Exists'] = True
resultDict['Type'] = "Directory"
resultDict['File'] = False
resultDict['Directory'] = True
resultDict['Size'] = 0
resultDict['TimeStamps'] = ( 0,0,0 )
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = 0o755
resultDict = StorageBase._addCommonMetadata( resultDict )
return S_OK( resultDict )
else:
return S_ERROR( 'Path does not exist' )
def __init__(self, storageName, parameters):
""" c'tor
:param self: self reference
:param str storageName: SE name
:param str protocol: protocol to use
:param str rootdir: base path for vo files
"""
# # init base class
StorageBase.__init__(self, storageName, parameters)
self.log = gLogger.getSubLogger("FileStorage", True)
# self.log.setLevel( "DEBUG" )
self.pluginName = 'File'
self.protocol = self.protocolParameters['Protocol']
def __init__( self, storageName, parameters ):
""" c'tor
:param self: self reference
:param str storageName: SE name
:param str protocol: protocol to use
:param str rootdir: base path for vo files
"""
# # init base class
StorageBase.__init__( self, storageName, parameters )
self.log = gLogger.getSubLogger( "FileStorage", True )
# self.log.setLevel( "DEBUG" )
self.pluginName = 'File'
self.protocol = self.protocolParameters['Protocol']
def __init__(self, storageName, parameters):
""" """
StorageBase.__init__(self, storageName, parameters)
self.pluginName = "DIP"
self.log = gLogger.getSubLogger(self.__class__.__name__)
# Several ports can be specified as comma separated list, choose
# randomly one of those ports
ports = self.protocolParameters["Port"].split(",")
random.shuffle(ports)
self.protocolParameters["Port"] = ports[0]
pathDict = dict(self.protocolParameters)
pathDict["Path"] = self.basePath
result = pfnunparse(pathDict)
if result["OK"]:
self.url = result["Value"]
self.checkSum = "CheckSum"
self.isok = True
def __init__(self, storageName, parameters):
"""
"""
StorageBase.__init__(self, storageName, parameters)
self.pluginName = 'DIP'
self.log = gLogger.getSubLogger("DIPStorage", True)
# Several ports can be specified as comma separated list, choose
# randomly one of those ports
ports = self.protocolParameters['Port'].split(',')
random.shuffle(ports)
self.protocolParameters['Port'] = ports[0]
pathDict = dict(self.protocolParameters)
pathDict['Path'] = self.basePath
result = pfnunparse(pathDict)
if result['OK']:
self.url = result['Value']
self.checkSum = "CheckSum"
self.isok = True
def __init__( self, storageName, parameters ):
"""
"""
StorageBase.__init__( self, storageName, parameters )
self.pluginName = 'DIP'
self.log = gLogger.getSubLogger( "DIPStorage", True )
# Several ports can be specified as comma separated list, choose
# randomly one of those ports
ports = self.protocolParameters['Port'].split( ',' )
random.shuffle( ports )
self.protocolParameters['Port'] = ports[0]
pathDict = dict( self.protocolParameters )
pathDict['Path'] = self.basePath
result = pfnunparse( pathDict )
if result['OK']:
self.url = result['Value']
self.checkSum = "CheckSum"
self.isok = True
def mock_StorageFactory_generateStorageObject( storageName, pluginName, parameters, hideExceptions = False ):
""" Generate fake storage object"""
storageObj = StorageBase( storageName, parameters )
if pluginName == "SRM2":
storageObj = fake_SRM2Plugin( storageName, parameters )
storageObj.protocolParameters['InputProtocols'] = ['file', 'root', 'srm']
storageObj.protocolParameters['OutputProtocols'] = ['file', 'root', 'dcap', 'gsidcap', 'rfio', 'srm']
elif pluginName == "File":
# Not needed to do anything, StorageBase should do it :)
pass
elif pluginName == 'XROOT':
storageObj = fake_XROOTPlugin( storageName, parameters )
storageObj.protocolParameters['InputProtocols'] = ['file', 'root']
storageObj.protocolParameters['OutputProtocols'] = ['root']
elif pluginName == 'GSIFTP':
storageObj = fake_GSIFTPPlugin( storageName, parameters )
storageObj.protocolParameters['InputProtocols'] = ['file', 'gsiftp']
storageObj.protocolParameters['OutputProtocols'] = ['gsiftp']
storageObj.pluginName = pluginName
return S_OK( storageObj )
def __getFileStat(path):
""" Get the file stat information
"""
resultDict = {}
try:
statTuple = os.stat(path)
except OSError as x:
if str(x).find('No such file') >= 0:
resultDict['Exists'] = False
return S_OK(resultDict)
return S_ERROR('Failed to get metadata for %s' % path)
resultDict['Exists'] = True
mode = statTuple[stat.ST_MODE]
resultDict['Type'] = "File"
resultDict['File'] = True
resultDict['Directory'] = False
if stat.S_ISDIR(mode):
resultDict['Type'] = "Directory"
resultDict['File'] = False
resultDict['Directory'] = True
resultDict['Size'] = statTuple[stat.ST_SIZE]
resultDict['TimeStamps'] = (statTuple[stat.ST_ATIME],
statTuple[stat.ST_MTIME],
statTuple[stat.ST_CTIME])
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = stat.S_IMODE(mode)
if resultDict['File']:
cks = fileAdler(path)
resultDict['Checksum'] = cks
resultDict = StorageBase._addCommonMetadata(resultDict)
return S_OK(resultDict)
def __getFileStat(path):
"""Get the file stat information"""
resultDict = {}
try:
statTuple = os.stat(path)
except OSError as x:
if str(x).find("No such file") >= 0:
resultDict["Exists"] = False
return S_OK(resultDict)
return S_ERROR("Failed to get metadata for %s" % path)
resultDict["Exists"] = True
mode = statTuple[stat.ST_MODE]
resultDict["Type"] = "File"
resultDict["File"] = True
resultDict["Directory"] = False
if stat.S_ISDIR(mode):
resultDict["Type"] = "Directory"
resultDict["File"] = False
resultDict["Directory"] = True
resultDict["Size"] = statTuple[stat.ST_SIZE]
resultDict["TimeStamps"] = (statTuple[stat.ST_ATIME],
statTuple[stat.ST_MTIME],
statTuple[stat.ST_CTIME])
resultDict["Cached"] = 1
resultDict["Migrated"] = 0
resultDict["Lost"] = 0
resultDict["Unavailable"] = 0
resultDict["Mode"] = stat.S_IMODE(mode)
if resultDict["File"]:
cks = fileAdler(path)
resultDict["Checksum"] = cks
resultDict = StorageBase._addCommonMetadata(resultDict)
return S_OK(resultDict)
def __getFileStat(path):
""" Get the file stat information
"""
resultDict = {}
try:
statTuple = os.stat(path)
except OSError as x:
if str(x).find('No such file') >= 0:
resultDict['Exists'] = False
return S_OK(resultDict)
return S_ERROR('Failed to get metadata for %s' % path)
resultDict['Exists'] = True
mode = statTuple[stat.ST_MODE]
resultDict['Type'] = "File"
resultDict['File'] = True
resultDict['Directory'] = False
if stat.S_ISDIR(mode):
resultDict['Type'] = "Directory"
resultDict['File'] = False
resultDict['Directory'] = True
resultDict['Size'] = statTuple[stat.ST_SIZE]
resultDict['TimeStamps'] = (statTuple[stat.ST_ATIME], statTuple[stat.ST_MTIME], statTuple[stat.ST_CTIME])
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = stat.S_IMODE(mode)
if resultDict['File']:
cks = fileAdler(path)
resultDict['Checksum'] = cks
resultDict = StorageBase._addCommonMetadata(resultDict)
return S_OK(resultDict)
class StorageElementHandler( RequestHandler ):
"""
.. class:: StorageElementHandler
"""
def __confirmToken( self, token, path, mode ):
""" Confirm the access rights for the path in a given mode
"""
# Not yet implemented
return True
@staticmethod
def __checkForDiskSpace( dpath, size ):
""" Check if the directory dpath can accommodate 'size' volume of data
"""
stats = os.statvfs( dpath )
dsize = stats.f_bsize * stats.f_bavail
maxStorageSizeBytes = MAX_STORAGE_SIZE * 1024 * 1024
return min( dsize, maxStorageSizeBytes ) > size
def __resolveFileID( self, fileID ):
""" get path to file for a given :fileID: """
port = self.getCSOption( 'Port', '' )
if not port:
return ''
if ":%s" % port in fileID:
loc = fileID.find( ":%s" % port )
if loc >= 0:
fileID = fileID[loc + len( ":%s" % port ):]
serviceName = self.serviceInfoDict['serviceName']
loc = fileID.find( serviceName )
if loc >= 0:
fileID = fileID[loc + len( serviceName ):]
loc = fileID.find( '?=' )
if loc >= 0:
fileID = fileID[loc + 2:]
if fileID.find( BASE_PATH ) == 0:
return fileID
while fileID and fileID[0] == '/':
fileID = fileID[1:]
return os.path.join( BASE_PATH, fileID )
@staticmethod
def __getFileStat( path ):
""" Get the file stat information
"""
resultDict = {}
try:
statTuple = os.stat( path )
except OSError, x:
if str( x ).find( 'No such file' ) >= 0:
resultDict['Exists'] = False
return S_OK( resultDict )
else:
return S_ERROR( 'Failed to get metadata for %s' % path )
resultDict['Exists'] = True
mode = statTuple[ST_MODE]
resultDict['Type'] = "File"
resultDict['File'] = True
resultDict['Directory'] = False
if S_ISDIR( mode ):
resultDict['Type'] = "Directory"
resultDict['File'] = False
resultDict['Directory'] = True
resultDict['Size'] = statTuple[ST_SIZE]
resultDict['TimeStamps'] = ( statTuple[ST_ATIME], statTuple[ST_MTIME], statTuple[ST_CTIME] )
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = S_IMODE( mode )
if resultDict['File']:
cks = fileAdler( path )
resultDict['Checksum'] = cks
resultDict = StorageBase._addCommonMetadata( resultDict )
return S_OK( resultDict )
def __init__(self, storageName, parameters):
StorageBase.__init__(self, storageName, parameters)
self.pluginName = 'Proxy'
self.isok = True
self.url = 'DataManagement/StorageElementProxy'
def setParameters(self, parameters):
"""Applying extra storage parameters"""
StorageBase.setParameters(self, parameters)
if "CheckSum" in parameters and parameters["CheckSum"].lower() in ["0", "no", "false", "off"]:
self.checkSum = "NoCheckSum"
return S_OK()
def __init__( self, storageName, parameters ):
StorageBase.__init__( self, storageName, parameters )
self.pluginName = 'Proxy'
self.isok = True
self.url = 'DataManagement/StorageElementProxy'