def transfer_toClient( self, fileID, token, fileHelper ):
""" Method to send files to clients.
fileID is the local file name in the SE.
token is used for access rights confirmation.
"""
conn , error = self.__irodsClient( "r" )
if not conn:
return S_ERROR( error )
file_path = self.__resolveFileID( fileID )
file_path = IRODS_HOME + file_path
gLogger.debug( "file_path to read: %s" % file_path )
fd = iRodsOpen( conn , file_path , "r" )
if not fd:
rcDisconnect( conn )
gLogger.error( "Failed to get file object" )
return S_ERROR( "Failed to get file object" )
result = fileHelper.FileToNetwork( fd )
fd.close()
rcDisconnect( conn )
if not result[ "OK" ]:
gLogger.error( "Failed to get file " + fileID )
return S_ERROR( "Failed to get file " + fileID )
else:
return result
def export_getResourceInfo( self , resource = None ):
""" Send the storage element resource information
"""
conn , error = self.__irodsClient( "r" )
if not conn:
return S_ERROR( error )
storageDict = {}
for resource in getResources( conn ):
name = resource.getName()
if resource and resource != name:
continue
storageDict[ name ] = {}
storageDict[ name ][ "Id" ] = resource.getId()
storageDict[ name ][ "AvailableSpace" ] = resource.getFreeSpace()
storageDict[ name ][ "Zone" ] = resource.getZone()
storageDict[ name ][ "Type" ] = resource.getTypeName()
storageDict[ name ][ "Class" ] = resource.getClassName()
storageDict[ name ][ "Host" ] = resource.getHost()
storageDict[ name ][ "Path" ] = resource.getPath()
storageDict[ name ][ "Free Space TS" ] = resource.getFreeSpaceTs()
storageDict[ name ][ "Info" ] = resource.getInfo()
storageDict[ name ][ "Comment" ] = resource.getComment()
rcDisconnect( conn )
return S_OK( storageDict )
def __removeFile(self, fileID, token):
""" Remove one file with fileID name from the storage
"""
conn, error, userDict = self.__irodsClient()
if not conn:
return S_ERROR(error)
file_path = self.__resolveFileID(fileID, userDict)
irodsHome = userDict.get('iRodsHome', IRODS_HOME)
file_path = irodsHome + file_path
gLogger.debug("file_path to read: %s" % file_path)
irodsFile = irodsOpen(conn, file_path, "r")
if not irodsFile:
rcDisconnect(conn)
gLogger.error("Failed to get file object")
return S_ERROR("Failed to get file object")
if self.__confirmToken(token, fileID, 'x'):
try:
status = irodsFile.delete()
if status == 0:
return S_OK()
else:
return S_ERROR(
'Failed to delete file with status code %d' % status)
except OSError, error:
if str(error).find('No such file') >= 0:
# File does not exist anyway
return S_OK()
else:
return S_ERROR(
'Failed to remove file %s with exception %s' %
(fileID, error))
def transfer_fromClient( self, fileID, token, fileSize, fileHelper ):
""" Method to receive file from clients.
fileID is the local file name in the SE.
fileSize can be Xbytes or -1 if unknown.
token is used for access rights confirmation.
"""
conn , error = self.__irodsClient( "w" )
if not conn:
return S_ERROR( error )
coll = irodsCollection( conn, IRODS_HOME )
if not self.__checkForDiskSpace( IRODS_HOME, fileSize ):
rcDisconnect( conn )
return S_ERROR( 'Not enough disk space' )
file_path = self.__resolveFileID( fileID )
path = file_path.split( "/" )
file_ = path.pop()
if len( path ) > 0:
coll = self.__changeCollection( coll , path )
file_path = IRODS_HOME + file_path
try:
if IRODS_RESOURCE:
fd = coll.create( file_ , IRODS_RESOURCE )
else:
fd = coll.create( file_ , "w" )
except Exception, error:
rcDisconnect( conn )
return S_ERROR( "Cannot open to write destination file %s: %s" % ( file_path, str(error) ) )
def export_getResourceInfo(self, resource=None):
""" Send the storage element resource information
"""
conn, error, _userDict = self.__irodsClient()
if not conn:
return S_ERROR(error)
storageDict = {}
for resource in getResources(conn):
name = resource.getName()
if resource and resource != name:
continue
storageDict[name] = {}
storageDict[name]["Id"] = resource.getId()
storageDict[name]["AvailableSpace"] = resource.getFreeSpace()
storageDict[name]["Zone"] = resource.getZone()
storageDict[name]["Type"] = resource.getTypeName()
storageDict[name]["Class"] = resource.getClassName()
storageDict[name]["Host"] = resource.getHost()
storageDict[name]["Path"] = resource.getPath()
storageDict[name]["Free Space TS"] = resource.getFreeSpaceTs()
storageDict[name]["Info"] = resource.getInfo()
storageDict[name]["Comment"] = resource.getComment()
rcDisconnect(conn)
return S_OK(storageDict)
def transfer_toClient( self, fileID, token, fileHelper ):
""" Method to send files to clients.
fileID is the local file name in the SE.
token is used for access rights confirmation.
"""
conn , error, userDict = self.__irodsClient( )
if not conn:
return S_ERROR( error )
file_path = self.__resolveFileID( fileID, userDict )
irodsHome = userDict.get( 'iRodsHome', IRODS_HOME )
file_path = irodsHome + file_path
gLogger.debug( "file_path to read: %s" % file_path )
irodsFile = irodsOpen( conn , file_path , "r" )
if not irodsFile:
rcDisconnect( conn )
gLogger.error( "Failed to get file object" )
return S_ERROR( "Failed to get file object" )
result = fileHelper.DataSourceToNetwork( irodsFile )
irodsFile.close()
rcDisconnect( conn )
if not result[ "OK" ]:
gLogger.error( "Failed to get file " + fileID )
return S_ERROR( "Failed to get file " + fileID )
else:
return result
def transfer_toClient(self, fileID, token, fileHelper):
""" Method to send files to clients.
fileID is the local file name in the SE.
token is used for access rights confirmation.
"""
conn, error, userDict = self.__irodsClient()
if not conn:
return S_ERROR(error)
file_path = self.__resolveFileID(fileID, userDict)
irodsHome = userDict.get('iRodsHome', IRODS_HOME)
file_path = irodsHome + file_path
gLogger.debug("file_path to read: %s" % file_path)
irodsFile = irodsOpen(conn, file_path, "r")
if not irodsFile:
rcDisconnect(conn)
gLogger.error("Failed to get file object")
return S_ERROR("Failed to get file object")
result = fileHelper.DataSourceToNetwork(irodsFile)
irodsFile.close()
rcDisconnect(conn)
if not result["OK"]:
gLogger.error("Failed to get file " + fileID)
return S_ERROR("Failed to get file " + fileID)
else:
return result
def __removeFile( self, fileID, token ):
""" Remove one file with fileID name from the storage
"""
conn , error, userDict = self.__irodsClient( )
if not conn:
return S_ERROR( error )
file_path = self.__resolveFileID( fileID, userDict )
irodsHome = userDict.get( 'iRodsHome', IRODS_HOME )
file_path = irodsHome + file_path
gLogger.debug( "file_path to read: %s" % file_path )
irodsFile = irodsOpen( conn , file_path , "r" )
if not irodsFile:
rcDisconnect( conn )
gLogger.error( "Failed to get file object" )
return S_ERROR( "Failed to get file object" )
if self.__confirmToken( token, fileID, 'x' ):
try:
status = irodsFile.delete()
if status == 0:
return S_OK()
else:
return S_ERROR( 'Failed to delete file with status code %d' % status )
except OSError, error:
if str( error ).find( 'No such file' ) >= 0:
# File does not exist anyway
return S_OK()
else:
return S_ERROR( 'Failed to remove file %s with exception %s' % ( fileID, error ) )
def transfer_fromClient( self, fileID, token, fileSize, fileHelper ):
""" Method to receive file from clients.
fileID is the local file name in the SE.
fileSize can be Xbytes or -1 if unknown.
token is used for access rights confirmation.
"""
conn , error, userDict = self.__irodsClient( )
if not conn:
return S_ERROR( error )
irodsHome = userDict.get( 'iRodsHome', IRODS_HOME )
coll = irodsCollection( conn, irodsHome )
if not self.__checkForDiskSpace( IRODS_HOME, fileSize ):
rcDisconnect( conn )
return S_ERROR( 'Not enough disk space' )
file_path = self.__resolveFileID( fileID, userDict )
path = file_path.split( "/" )
file_ = path.pop()
if len( path ) > 0:
coll = self.__changeCollection( coll , path )
file_path = irodsHome + file_path
try:
if IRODS_RESOURCE:
irodsFile = coll.create( file_ , IRODS_RESOURCE )
else:
irodsFile = coll.create( file_ , "w" )
except Exception, error:
rcDisconnect( conn )
return S_ERROR( "Cannot open to write destination file %s: %s" % ( file_path, str(error) ) )
def transfer_toClient(self, fileID, token, fileHelper):
""" Method to send files to clients.
fileID is the local file name in the SE.
token is used for access rights confirmation.
"""
conn, error = self.__irodsClient("r")
if not conn:
return S_ERROR(error)
file_path = self.__resolveFileID(fileID)
file_path = IRODS_HOME + file_path
gLogger.debug("file_path to read: %s" % file_path)
fd = iRodsOpen(conn, file_path, "r")
if not fd:
rcDisconnect(conn)
gLogger.error("Failed to get file object")
return S_ERROR("Failed to get file object")
result = fileHelper.FileToNetwork(fd)
fd.close()
rcDisconnect(conn)
if not result["OK"]:
gLogger.error("Failed to get file " + fileID)
return S_ERROR("Failed to get file " + fileID)
else:
return result
coll = self.__changeCollection( coll , path )
file_path = IRODS_HOME + file_path
try:
if IRODS_RESOURCE:
fd = coll.create( file_ , IRODS_RESOURCE )
else:
fd = coll.create( file_ , "w" )
except Exception, error:
rcDisconnect( conn )
return S_ERROR( "Cannot open to write destination file %s: %s" % ( file_path, str(error) ) )
result = fileHelper.networkToDataSink( fd, maxFileSize=( MAX_STORAGE_SIZE * 1024 * 1024 ) )
fd.close()
rcDisconnect( conn )
if not result[ 'OK' ]:
return result
return result
def transfer_toClient( self, fileID, token, fileHelper ):
""" Method to send files to clients.
fileID is the local file name in the SE.
token is used for access rights confirmation.
"""
conn , error = self.__irodsClient( "r" )
if not conn:
return S_ERROR( error )
file_path = self.__resolveFileID( fileID )
class IRODSStorageElementHandler(RequestHandler):
"""
.. class:: StorageElementHandler
"""
def __checkForDiskSpace(self, path, size):
""" Check if iRods resource has enough space
"""
## dsize = ( getDiskSpace( dpath ) - 1 ) * 1024 * 1024
## maxStorageSizeBytes = MAX_STORAGE_SIZE * 1024 * 1024
## return ( min( dsize, maxStorageSizeBytes ) > size )
return True
def __resolveFileID(self, fileID, userDict):
""" get path to file for a given :fileID: """
port = self.getCSOption('Port', '')
if not port:
return ''
if ":%s" % port in fileID:
loc = fileID.find(":%s" % port)
if loc >= 0:
fileID = fileID[loc + len(":%s" % port):]
serviceName = self.serviceInfoDict['serviceName']
loc = fileID.find(serviceName)
if loc >= 0:
fileID = fileID[loc + len(serviceName):]
loc = fileID.find('?=')
if loc >= 0:
fileID = fileID[loc + 2:]
# Leave only one / in front
while fileID and fileID[0] == '/':
fileID = fileID[1:]
fileID = '/' + fileID
# For user data strip off the HOME part as iRods has its own user home
if fileID.startswith(userDict['DIRACHome']):
fileID = fileID.replace(userDict['DIRACHome'], '')
return fileID
def __getFileStat(self, path):
"""
Get the file stat information
"""
conn, error, userDict = self.__irodsClient()
if not conn:
return S_ERROR(error)
file_path = self.__resolveFileID(path, userDict)
irodsHome = userDict.get('iRodsHome', IRODS_HOME)
file_path = irodsHome + file_path
gLogger.debug("file_path to read: %s" % file_path)
resultDict = {}
irodsFile = irodsOpen(conn, file_path, "r")
if irodsFile:
resultDict['Exists'] = True
resultDict['File'] = True
resultDict['Directory'] = False
resultDict['Type'] = "File"
resultDict['Size'] = irodsFile.getSize()
resultDict['TimeStamps'] = (irodsFile.getCreateTs(),
irodsFile.getModifyTs(),
irodsFile.getCreateTs())
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = 0o755
resultDict = StorageBase._addCommonMetadata(resultDict)
return S_OK(resultDict)
else:
coll = irodsCollection(conn, file_path)
if coll:
resultDict['Exists'] = True
resultDict['Type'] = "Directory"
resultDict['File'] = False
resultDict['Directory'] = True
resultDict['Size'] = 0
resultDict['TimeStamps'] = (0, 0, 0)
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = 0o755
resultDict = StorageBase._addCommonMetadata(resultDict)
return S_OK(resultDict)
else:
return S_ERROR('Path does not exist')
types_exists = [StringTypes]
def export_exists(self, fileID):
""" Check existence of the fileID """
conn, error, userDict = self.__irodsClient()
if not conn:
return S_ERROR(error)
file_path = self.__resolveFileID(fileID, userDict)
irodsHome = userDict.get('iRodsHome', IRODS_HOME)
file_path = irodsHome + file_path
gLogger.debug("file_path to read: %s" % file_path)
irodsFile = irodsOpen(conn, file_path, "r")
if irodsFile:
return S_OK(True)
else:
coll = irodsCollection(conn, file_path)
if coll:
return S_OK(True)
return S_OK(False)
types_getMetadata = [StringTypes]
def export_getMetadata(self, fileID):
"""
Get metadata for the file or directory specified by fileID
"""
return self.__getFileStat(fileID)
types_createDirectory = [StringTypes]
def export_createDirectory(self, dir_path):
"""
Creates the directory on the storage
"""
conn, error, userDict = self.__irodsClient()
if not conn:
return S_ERROR(error)
irodsHome = userDict.get('iRodsHome', IRODS_HOME)
coll = irodsCollection(conn, irodsHome)
dir_path = self.__resolveFileID(dir_path, userDict)
path = dir_path.split("/")
if len(path) > 0:
coll = self.__changeCollection(coll, path)
if not coll:
return S_ERROR('Failed to create directory')
return S_OK()
types_listDirectory = [StringTypes, StringTypes]
def export_listDirectory(self, dir_path, mode):
"""
Return the dir_path directory listing
"""
conn, error, userDict = self.__irodsClient()
if not conn:
return S_ERROR(error)
file_path = self.__resolveFileID(dir_path, userDict)
irodsHome = userDict.get('iRodsHome', IRODS_HOME)
irodsPath = irodsHome + file_path
gLogger.debug("file_path to read: %s" % irodsPath)
is_file = False
irodsFile = irodsOpen(conn, irodsPath, "r")
if not irodsFile:
is_file = True
else:
irodsDir = os.path.dirname(irodsPath)
coll = irodsCollection(conn, irodsDir)
if not coll:
return S_ERROR('Directory not found')
objects = coll.getObjects()
fileList = [x[0] for x in objects]
dirList = coll.getSubCollections()
resultDict = {}
if mode == 'l':
if is_file:
result = self.__getFileStat(dir_path)
if result['OK']:
resultDict[dir_path] = result['Value']
return S_OK(resultDict)
else:
return S_ERROR('Failed to get the file stat info')
else:
failed_list = []
one_OK = False
for fname in dirList + fileList:
result = self.__getFileStat(dir_path + '/' + fname)
if result['OK']:
resultDict[fname] = result['Value']
one_OK = True
else:
failed_list.append(fname)
if failed_list:
if one_OK:
result = S_ERROR(
'Failed partially to get the file stat info')
else:
result = S_ERROR('Failed to get the file stat info')
result['FailedList'] = failed_list
result['Value'] = resultDict
else:
result = S_OK(resultDict)
return result
else:
return S_OK(dirList)
def __changeCollection(self, coll, path):
if not len(path) > 0:
return coll
name = path.pop(0)
if not len(name) > 0:
return self.__changeCollection(coll, path)
gLogger.info('Check subcollection: %s' % name)
subs = coll.getSubCollections()
if not name in subs:
gLogger.info('Create subcollection: %s' % name)
coll.createCollection(name)
gLogger.info('Open subcollection: %s' % name)
coll.openCollection(name)
return self.__changeCollection(coll, path)
def transfer_fromClient(self, fileID, token, fileSize, fileHelper):
"""
Method to receive file from clients.
fileID is the local file name in the SE.
fileSize can be Xbytes or -1 if unknown.
token is used for access rights confirmation.
"""
conn, error, userDict = self.__irodsClient()
if not conn:
return S_ERROR(error)
irodsHome = userDict.get('iRodsHome', IRODS_HOME)
coll = irodsCollection(conn, irodsHome)
if not self.__checkForDiskSpace(IRODS_HOME, fileSize):
rcDisconnect(conn)
return S_ERROR('Not enough disk space')
file_path = self.__resolveFileID(fileID, userDict)
path = file_path.split("/")
file_ = path.pop()
if len(path) > 0:
coll = self.__changeCollection(coll, path)
file_path = irodsHome + file_path
try:
if IRODS_RESOURCE:
irodsFile = coll.create(file_, IRODS_RESOURCE)
else:
irodsFile = coll.create(file_, "w")
except Exception, error:
rcDisconnect(conn)
return S_ERROR("Cannot open to write destination file %s: %s" %
(file_path, str(error)))
if "NoCheckSum" in token:
fileHelper.disableCheckSum()
result = fileHelper.networkToDataSink(irodsFile,
maxFileSize=(MAX_STORAGE_SIZE *
1024 * 1024))
irodsFile.close()
rcDisconnect(conn)
if not result['OK']:
return result
return result
def _close(self, session):
irods.rcDisconnect(self.cxn)
self.cxn = None
self.coll = None
self.env = None
def disconnect(self): if self.conn is not None: irods.rcDisconnect(self.conn)
try:
if IRODS_RESOURCE:
fd = coll.create(file_, IRODS_RESOURCE)
else:
fd = coll.create(file_, "w")
except Exception, error:
rcDisconnect(conn)
return S_ERROR("Cannot open to write destination file %s: %s" %
(file_path, str(error)))
result = fileHelper.networkToDataSink(fd,
maxFileSize=(MAX_STORAGE_SIZE *
1024 * 1024))
fd.close()
rcDisconnect(conn)
if not result['OK']:
return result
return result
def transfer_toClient(self, fileID, token, fileHelper):
""" Method to send files to clients.
fileID is the local file name in the SE.
token is used for access rights confirmation.
"""
conn, error = self.__irodsClient("r")
if not conn:
return S_ERROR(error)
file_path = self.__resolveFileID(fileID)
