def save_file(self, irods_path, txt, resc=None):
"""Save textual content to an iRODS file"""
if resc:
f = irods.irodsOpen(self.conn, irods_path, "w", resc)
else:
f = irods.irodsOpen(self.conn, irods_path, "w")
if not f:
return
f.write(txt)
f.close()
def create_file(self, irods_path, content='', resc=None):
"""Create a new text file in iRODS"""
if resc:
f = irods.irodsOpen(self.conn, irods_path, "w", resc)
else:
f = irods.irodsOpen(self.conn, irods_path, "w")
if not f:
return False
f.write(content)
f.close()
return True
def read_file(self, irods_path, resc=None):
"""Read textual content from an iRODS file"""
if resc:
f = irods.irodsOpen(self.conn, irods_path, "r", resc)
else:
f = irods.irodsOpen(self.conn, irods_path, "r")
if not f:
return ""
txt = f.read()
f.close()
return txt
def __removeFile( self, fileID, token ):
""" Remove one file with fileID name from the storage
"""
conn , error, userDict = self.__irodsClient( )
if not conn:
return S_ERROR( error )
file_path = self.__resolveFileID( fileID, userDict )
irodsHome = userDict.get( 'iRodsHome', IRODS_HOME )
file_path = irodsHome + file_path
gLogger.debug( "file_path to read: %s" % file_path )
irodsFile = irodsOpen( conn , file_path , "r" )
if not irodsFile:
rcDisconnect( conn )
gLogger.error( "Failed to get file object" )
return S_ERROR( "Failed to get file object" )
if self.__confirmToken( token, fileID, 'x' ):
try:
status = irodsFile.delete()
if status == 0:
return S_OK()
else:
return S_ERROR( 'Failed to delete file with status code %d' % status )
except OSError, error:
if str( error ).find( 'No such file' ) >= 0:
# File does not exist anyway
return S_OK()
else:
return S_ERROR( 'Failed to remove file %s with exception %s' % ( fileID, error ) )
def __get_rods_handle(self, obj, mode='r', **kwargs):
if kwargs.get('dir_only', False):
return irods.irodsCollection(self.rods_conn,
self.__get_rods_path(obj, **kwargs))
else:
return irods.irodsOpen(self.rods_conn,
self.__get_rods_path(obj, **kwargs), mode)
def transfer_toClient(self, fileID, token, fileHelper):
""" Method to send files to clients.
fileID is the local file name in the SE.
token is used for access rights confirmation.
"""
conn, error, userDict = self.__irodsClient()
if not conn:
return S_ERROR(error)
file_path = self.__resolveFileID(fileID, userDict)
irodsHome = userDict.get('iRodsHome', IRODS_HOME)
file_path = irodsHome + file_path
gLogger.debug("file_path to read: %s" % file_path)
irodsFile = irodsOpen(conn, file_path, "r")
if not irodsFile:
rcDisconnect(conn)
gLogger.error("Failed to get file object")
return S_ERROR("Failed to get file object")
result = fileHelper.DataSourceToNetwork(irodsFile)
irodsFile.close()
rcDisconnect(conn)
if not result["OK"]:
gLogger.error("Failed to get file " + fileID)
return S_ERROR("Failed to get file " + fileID)
else:
return result
def transfer_toClient( self, fileID, token, fileHelper ):
""" Method to send files to clients.
fileID is the local file name in the SE.
token is used for access rights confirmation.
"""
conn , error, userDict = self.__irodsClient( )
if not conn:
return S_ERROR( error )
file_path = self.__resolveFileID( fileID, userDict )
irodsHome = userDict.get( 'iRodsHome', IRODS_HOME )
file_path = irodsHome + file_path
gLogger.debug( "file_path to read: %s" % file_path )
irodsFile = irodsOpen( conn , file_path , "r" )
if not irodsFile:
rcDisconnect( conn )
gLogger.error( "Failed to get file object" )
return S_ERROR( "Failed to get file object" )
result = fileHelper.DataSourceToNetwork( irodsFile )
irodsFile.close()
rcDisconnect( conn )
if not result[ "OK" ]:
gLogger.error( "Failed to get file " + fileID )
return S_ERROR( "Failed to get file " + fileID )
else:
return result
def __removeFile(self, fileID, token):
""" Remove one file with fileID name from the storage
"""
conn, error, userDict = self.__irodsClient()
if not conn:
return S_ERROR(error)
file_path = self.__resolveFileID(fileID, userDict)
irodsHome = userDict.get('iRodsHome', IRODS_HOME)
file_path = irodsHome + file_path
gLogger.debug("file_path to read: %s" % file_path)
irodsFile = irodsOpen(conn, file_path, "r")
if not irodsFile:
rcDisconnect(conn)
gLogger.error("Failed to get file object")
return S_ERROR("Failed to get file object")
if self.__confirmToken(token, fileID, 'x'):
try:
status = irodsFile.delete()
if status == 0:
return S_OK()
else:
return S_ERROR(
'Failed to delete file with status code %d' % status)
except OSError, error:
if str(error).find('No such file') >= 0:
# File does not exist anyway
return S_OK()
else:
return S_ERROR(
'Failed to remove file %s with exception %s' %
(fileID, error))
def export_listDirectory(self, dir_path, mode):
"""
Return the dir_path directory listing
"""
conn, error, userDict = self.__irodsClient()
if not conn:
return S_ERROR(error)
file_path = self.__resolveFileID(dir_path, userDict)
irodsHome = userDict.get('iRodsHome', IRODS_HOME)
irodsPath = irodsHome + file_path
gLogger.debug("file_path to read: %s" % irodsPath)
is_file = False
irodsFile = irodsOpen(conn, irodsPath, "r")
if not irodsFile:
is_file = True
else:
irodsDir = os.path.dirname(irodsPath)
coll = irodsCollection(conn, irodsDir)
if not coll:
return S_ERROR('Directory not found')
objects = coll.getObjects()
fileList = [x[0] for x in objects]
dirList = coll.getSubCollections()
resultDict = {}
if mode == 'l':
if is_file:
result = self.__getFileStat(dir_path)
if result['OK']:
resultDict[dir_path] = result['Value']
return S_OK(resultDict)
else:
return S_ERROR('Failed to get the file stat info')
else:
failed_list = []
one_OK = False
for fname in dirList + fileList:
result = self.__getFileStat(dir_path + '/' + fname)
if result['OK']:
resultDict[fname] = result['Value']
one_OK = True
else:
failed_list.append(fname)
if failed_list:
if one_OK:
result = S_ERROR(
'Failed partially to get the file stat info')
else:
result = S_ERROR('Failed to get the file stat info')
result['FailedList'] = failed_list
result['Value'] = resultDict
else:
result = S_OK(resultDict)
return result
else:
return S_OK(dirList)
def export_listDirectory( self, dir_path, mode ):
"""
Return the dir_path directory listing
"""
conn , error, userDict = self.__irodsClient( )
if not conn:
return S_ERROR( error )
file_path = self.__resolveFileID( dir_path, userDict )
irodsHome = userDict.get( 'iRodsHome', IRODS_HOME )
irodsPath = irodsHome + file_path
gLogger.debug( "file_path to read: %s" % irodsPath )
is_file = False
irodsFile = irodsOpen( conn , irodsPath , "r" )
if not irodsFile:
is_file = True
else:
irodsDir = os.path.dirname( irodsPath )
coll = irodsCollection( conn, irodsDir )
if not coll:
return S_ERROR( 'Directory not found' )
objects = coll.getObjects()
fileList = [ x[0] for x in objects ]
dirList = coll.getSubCollections()
resultDict = {}
if mode == 'l':
if is_file:
result = self.__getFileStat( dir_path )
if result['OK']:
resultDict[dir_path] = result['Value']
return S_OK( resultDict )
else:
return S_ERROR( 'Failed to get the file stat info' )
else:
failed_list = []
one_OK = False
for fname in dirList+fileList:
result = self.__getFileStat( dir_path + '/' + fname )
if result['OK']:
resultDict[fname] = result['Value']
one_OK = True
else:
failed_list.append( fname )
if failed_list:
if one_OK:
result = S_ERROR( 'Failed partially to get the file stat info' )
else:
result = S_ERROR( 'Failed to get the file stat info' )
result['FailedList'] = failed_list
result['Value'] = resultDict
else:
result = S_OK( resultDict )
return result
else:
return S_OK( dirList )
def open_irodsUrl(url, mode='r'):
"""Open and return the file specified by an iRODS URL.
Returns a file-like object - ``irods.IrodsFile``
"""
if irods is None:
raise MissingDependencyException("open_irodsUrl()", 'irods (PyRods)')
parsed = parse_irodsUrl(url)
conn, errMsg = irods.rcConnect(parsed.rodsHost, parsed.rodsPort,
parsed.rodsUserName, parsed.rodsZone)
status = irods.clientLogin(conn)
return irods.irodsOpen(conn, parsed.path, mode)
def __getFileStat(self, path):
"""
Get the file stat information
"""
conn, error, userDict = self.__irodsClient()
if not conn:
return S_ERROR(error)
file_path = self.__resolveFileID(path, userDict)
irodsHome = userDict.get('iRodsHome', IRODS_HOME)
file_path = irodsHome + file_path
gLogger.debug("file_path to read: %s" % file_path)
resultDict = {}
irodsFile = irodsOpen(conn, file_path, "r")
if irodsFile:
resultDict['Exists'] = True
resultDict['File'] = True
resultDict['Directory'] = False
resultDict['Type'] = "File"
resultDict['Size'] = irodsFile.getSize()
resultDict['TimeStamps'] = (irodsFile.getCreateTs(),
irodsFile.getModifyTs(),
irodsFile.getCreateTs())
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = 0o755
resultDict = StorageBase._addCommonMetadata(resultDict)
return S_OK(resultDict)
else:
coll = irodsCollection(conn, file_path)
if coll:
resultDict['Exists'] = True
resultDict['Type'] = "Directory"
resultDict['File'] = False
resultDict['Directory'] = True
resultDict['Size'] = 0
resultDict['TimeStamps'] = (0, 0, 0)
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = 0o755
resultDict = StorageBase._addCommonMetadata(resultDict)
return S_OK(resultDict)
else:
return S_ERROR('Path does not exist')
def __getFileStat( self, path ):
"""
Get the file stat information
"""
conn , error, userDict = self.__irodsClient( )
if not conn:
return S_ERROR( error )
file_path = self.__resolveFileID( path, userDict )
irodsHome = userDict.get( 'iRodsHome', IRODS_HOME )
file_path = irodsHome + file_path
gLogger.debug( "file_path to read: %s" % file_path )
resultDict = {}
irodsFile = irodsOpen( conn , file_path , "r" )
if irodsFile:
resultDict['Exists'] = True
resultDict['File'] = True
resultDict['Directory'] = False
resultDict['Type'] = "File"
resultDict['Size'] = irodsFile.getSize()
resultDict['TimeStamps'] = ( irodsFile.getCreateTs(), irodsFile.getModifyTs(), irodsFile.getCreateTs() )
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = 0o755
resultDict = StorageBase._addCommonMetadata( resultDict )
return S_OK( resultDict )
else:
coll = irodsCollection( conn, file_path )
if coll:
resultDict['Exists'] = True
resultDict['Type'] = "Directory"
resultDict['File'] = False
resultDict['Directory'] = True
resultDict['Size'] = 0
resultDict['TimeStamps'] = ( 0,0,0 )
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = 0o755
resultDict = StorageBase._addCommonMetadata( resultDict )
return S_OK( resultDict )
else:
return S_ERROR( 'Path does not exist' )
def open_irodsUrl(url, mode='r'):
"""Open and return the file specified by an iRODS URL.
Returns a file-like object - ``irods.IrodsFile``
"""
if irods is None:
raise MissingDependencyException("open_irodsUrl()", 'irods (PyRods)')
parsed = parse_irodsUrl(url)
conn, errMsg = irods.rcConnect(parsed.rodsHost,
parsed.rodsPort,
parsed.rodsUserName,
parsed.rodsZone
)
status = irods.clientLogin(conn)
return irods.irodsOpen(conn, parsed.path, mode)
def erase_irods_urls(url_list, rodsconfig):
from irods import irodsOpen, irodsCollection
for user in [u for u in get_user_list() if u.valid]:
conn = create_irods_connection(user.name, user.password, rodsconfig)
for obj in url_list:
file_handle = irodsOpen(conn, obj.path, "r")
if file_handle is not None:
file_handle.close()
file_handle.delete(force=True)
base, name = os.path.split(obj.path)
coll = irodsCollection(conn, base)
coll.deleteCollection(name)
conn.disconnect()
def create_irods_urls(url_list, rodsconfig):
from irods import irodsCollection, irodsOpen
for user in [u for u in get_user_list() if u.valid]:
conn = create_irods_connection(user.name, user.password, rodsconfig)
for obj in [o for o in url_list if o.exists]:
if obj.objtype == obj.ContainerType:
base, name = os.path.split(obj.path)
coll = irodsCollection(conn, base)
coll.createCollection(name)
elif obj.objtype == obj.FileType:
coll = irodsCollection(conn)
coll.createCollection(os.path.split(obj.path)[0])
file_handle = irodsOpen(conn, obj.path, "w")
if file_handle is not None:
file_handle.write(obj.objinfo["content"])
file_handle.close()
conn.disconnect()
def export_exists(self, fileID):
""" Check existence of the fileID """
conn, error, userDict = self.__irodsClient()
if not conn:
return S_ERROR(error)
file_path = self.__resolveFileID(fileID, userDict)
irodsHome = userDict.get('iRodsHome', IRODS_HOME)
file_path = irodsHome + file_path
gLogger.debug("file_path to read: %s" % file_path)
irodsFile = irodsOpen(conn, file_path, "r")
if irodsFile:
return S_OK(True)
else:
coll = irodsCollection(conn, file_path)
if coll:
return S_OK(True)
return S_OK(False)
def erase_irods_urls(url_list, rodsconfig):
from irods import irodsOpen, irodsCollection
for user in [u for u in get_user_list() if u.valid]:
conn = create_irods_connection(user.name, user.password, rodsconfig)
urls = url_list + [get_irods_copy_target_url(
rodsconfig[2], # rodszone
user
)]
for obj in urls:
file_handle = irodsOpen(conn, obj.path, 'w')
if file_handle is not None:
file_handle.close()
file_handle.delete(force=True)
else:
base, name = os.path.split(obj.path)
coll = irodsCollection(conn, base)
coll.deleteCollection(name)
conn.disconnect()
def export_exists( self, fileID ):
""" Check existence of the fileID """
conn , error, userDict = self.__irodsClient( )
if not conn:
return S_ERROR( error )
file_path = self.__resolveFileID( fileID, userDict )
irodsHome = userDict.get( 'iRodsHome', IRODS_HOME )
file_path = irodsHome + file_path
gLogger.debug( "file_path to read: %s" % file_path )
irodsFile = irodsOpen( conn , file_path , "r" )
if irodsFile:
return S_OK( True )
else:
coll = irodsCollection( conn, file_path )
if coll:
return S_OK( True )
return S_OK( False )
def create_irods_urls(url_list, rodsconfig):
from irods import irodsCollection, irodsOpen
for user in [u for u in get_user_list() if u.valid]:
conn = create_irods_connection(user.name, user.password, rodsconfig)
for obj in [o for o in url_list if o.exists]:
if obj.objtype == obj.ContainerType:
base, name = os.path.split(obj.path)
coll = irodsCollection(conn, base)
coll.createCollection(name)
dir_handle = irodsCollection(conn, obj.path)
if dir_handle is not None:
dir_handle.addUserMetadata('objectID', obj.objectid)
elif obj.objtype == obj.FileType:
coll = irodsCollection(conn)
coll.createCollection(os.path.split(obj.path)[0])
file_handle = irodsOpen(conn, obj.path, 'w')
if file_handle is not None:
file_handle.write(obj.objinfo['content'])
file_handle.addUserMetadata('objectID', obj.objectid)
file_handle.close()
conn.disconnect()
def file_exist(self, irods_path):
"""Return True if the file is present in iRODS"""
f = irods.irodsOpen(self.conn, irods_path)
return f != None
def open(self, path, resource, mode="r"):
"""Open a file in iRods, return an iRodsFile"""
return irods.irodsOpen(self.conn, path, resource, mode)
def __get_rods_handle( self, obj, mode='r', **kwargs ):
if kwargs.get( 'dir_only', False ):
return irods.irodsCollection( self.rods_conn, self.__get_rods_path( obj, **kwargs ) )
else:
return irods.irodsOpen( self.rods_conn, self.__get_rods_path( obj, **kwargs ), mode )
def get_irods_file_handle(connection, filename):
return irodsOpen(connection, filename, mode='w')
