def __init__( self, config, file_path=None, extra_dirs=None ):
super( IRODSObjectStore, self ).__init__( config, file_path=file_path, extra_dirs=extra_dirs )
assert irods is not None, IRODS_IMPORT_MESSAGE
self.cache_path = config.object_store_cache_path
self.default_resource = config.irods_default_resource or None
# Connect to iRODS (AssertionErrors will be raised if anything goes wrong)
self.rods_env, self.rods_conn = rods_connect()
# if the root collection path in the config is unset or relative, try to use a sensible default
if config.irods_root_collection_path is None or ( config.irods_root_collection_path is not None and not config.irods_root_collection_path.startswith( '/' ) ):
rods_home = self.rods_env.rodsHome
assert rods_home != '', "Unable to initialize iRODS Object Store: rodsHome cannot be determined and irods_root_collection_path in Galaxy config is unset or not absolute."
if config.irods_root_collection_path is None:
self.root_collection_path = path_join( rods_home, 'galaxy_data' )
else:
self.root_collection_path = path_join( rods_home, config.irods_root_collection_path )
else:
self.root_collection_path = config.irods_root_collection_path
# will return a collection object regardless of whether it exists
self.root_collection = irods.irodsCollection( self.rods_conn, self.root_collection_path )
if self.root_collection.getId() == -1:
log.warning( "iRODS root collection does not exist, will attempt to create: %s", self.root_collection_path )
self.root_collection.upCollection()
assert self.root_collection.createCollection( os.path.basename( self.root_collection_path ) ) == 0, "iRODS root collection creation failed: %s" % self.root_collection_path
self.root_collection = irods.irodsCollection( self.rods_conn, self.root_collection_path )
assert self.root_collection.getId() != -1, "iRODS root collection creation claimed success but still does not exist"
if self.default_resource is None:
self.default_resource = self.rods_env.rodsDefResource
log.info( "iRODS data for this instance will be stored in collection: %s, resource: %s", self.root_collection_path, self.default_resource )
def __init__( self, config, file_path=None, extra_dirs=None ):
super( IRODSObjectStore, self ).__init__( config, file_path=file_path, extra_dirs=extra_dirs )
assert irods is not None, IRODS_IMPORT_MESSAGE
self.cache_path = config.object_store_cache_path
self.default_resource = config.irods_default_resource or None
# Connect to iRODS (AssertionErrors will be raised if anything goes wrong)
self.rods_env, self.rods_conn = rods_connect()
# if the root collection path in the config is unset or relative, try to use a sensible default
if config.irods_root_collection_path is None or ( config.irods_root_collection_path is not None and not config.irods_root_collection_path.startswith( '/' ) ):
rods_home = self.rods_env.rodsHome
assert rods_home != '', "Unable to initialize iRODS Object Store: rodsHome cannot be determined and irods_root_collection_path in Galaxy config is unset or not absolute."
if config.irods_root_collection_path is None:
self.root_collection_path = path_join( rods_home, 'galaxy_data' )
else:
self.root_collection_path = path_join( rods_home, config.irods_root_collection_path )
else:
self.root_collection_path = config.irods_root_collection_path
# will return a collection object regardless of whether it exists
self.root_collection = irods.irodsCollection( self.rods_conn, self.root_collection_path )
if self.root_collection.getId() == -1:
log.warning( "iRODS root collection does not exist, will attempt to create: %s", self.root_collection_path )
self.root_collection.upCollection()
assert self.root_collection.createCollection( os.path.basename( self.root_collection_path ) ) == 0, "iRODS root collection creation failed: %s" % self.root_collection_path
self.root_collection = irods.irodsCollection( self.rods_conn, self.root_collection_path )
assert self.root_collection.getId() != -1, "iRODS root collection creation claimed success but still does not exist"
if self.default_resource is None:
self.default_resource = self.rods_env.rodsDefResource
log.info( "iRODS data for this instance will be stored in collection: %s, resource: %s", self.root_collection_path, self.default_resource )
def transfer_fromClient( self, fileID, token, fileSize, fileHelper ):
""" Method to receive file from clients.
fileID is the local file name in the SE.
fileSize can be Xbytes or -1 if unknown.
token is used for access rights confirmation.
"""
conn , error, userDict = self.__irodsClient( )
if not conn:
return S_ERROR( error )
irodsHome = userDict.get( 'iRodsHome', IRODS_HOME )
coll = irodsCollection( conn, irodsHome )
if not self.__checkForDiskSpace( IRODS_HOME, fileSize ):
rcDisconnect( conn )
return S_ERROR( 'Not enough disk space' )
file_path = self.__resolveFileID( fileID, userDict )
path = file_path.split( "/" )
file_ = path.pop()
if len( path ) > 0:
coll = self.__changeCollection( coll , path )
file_path = irodsHome + file_path
try:
if IRODS_RESOURCE:
irodsFile = coll.create( file_ , IRODS_RESOURCE )
else:
irodsFile = coll.create( file_ , "w" )
except Exception, error:
rcDisconnect( conn )
return S_ERROR( "Cannot open to write destination file %s: %s" % ( file_path, str(error) ) )
def transfer_fromClient( self, fileID, token, fileSize, fileHelper ):
""" Method to receive file from clients.
fileID is the local file name in the SE.
fileSize can be Xbytes or -1 if unknown.
token is used for access rights confirmation.
"""
conn , error = self.__irodsClient( "w" )
if not conn:
return S_ERROR( error )
coll = irodsCollection( conn, IRODS_HOME )
if not self.__checkForDiskSpace( IRODS_HOME, fileSize ):
rcDisconnect( conn )
return S_ERROR( 'Not enough disk space' )
file_path = self.__resolveFileID( fileID )
path = file_path.split( "/" )
file_ = path.pop()
if len( path ) > 0:
coll = self.__changeCollection( coll , path )
file_path = IRODS_HOME + file_path
try:
if IRODS_RESOURCE:
fd = coll.create( file_ , IRODS_RESOURCE )
else:
fd = coll.create( file_ , "w" )
except Exception, error:
rcDisconnect( conn )
return S_ERROR( "Cannot open to write destination file %s: %s" % ( file_path, str(error) ) )
def __init__(self, session, stream):
myEnv, status = irods.getRodsEnv()
conn, errMsg = irods.rcConnect(myEnv.getRodsHost(), myEnv.getRodsPort(),
myEnv.getRodsUserName(), myEnv.getRodsZone())
status = irods.clientLogin(conn)
if status:
raise ConfigFileException("Cannot connect to iRODS: (%s) %s" % (status, errMsg))
home = myEnv.getRodsHome()
c = irods.irodsCollection(conn, home)
self.cxn = conn
self.coll = c
instream = stream
# check if abs path to home dir
if stream.startswith(home):
stream = stream[len(home):]
if stream[0] == "/":
stream = stream[1:]
colls = stream.split('/')
for i, cln in enumerate(colls):
exit_status = c.openCollection(cln)
if exit_status < 0:
if (i < len(colls) - 1) or \
(cln not in [obj[0] for obj in c.getObjects()]):
raise IOError("When opening {0}: {1} does not exists in collection {2}".format(instream, cln, c.getCollName()))
def __get_rods_handle(self, obj, mode='r', **kwargs):
if kwargs.get('dir_only', False):
return irods.irodsCollection(self.rods_conn,
self.__get_rods_path(obj, **kwargs))
else:
return irods.irodsOpen(self.rods_conn,
self.__get_rods_path(obj, **kwargs), mode)
def createGuinanCollection(self):
conn = self.mRods.getConnection()
path = '/' + self.mRods.getZone()
if (conn is not None):
coll = irods.irodsCollection(conn, path)
coll.createCollection(self.mRods.getCollectionName())
def export_listDirectory(self, dir_path, mode):
"""
Return the dir_path directory listing
"""
conn, error, userDict = self.__irodsClient()
if not conn:
return S_ERROR(error)
file_path = self.__resolveFileID(dir_path, userDict)
irodsHome = userDict.get('iRodsHome', IRODS_HOME)
irodsPath = irodsHome + file_path
gLogger.debug("file_path to read: %s" % irodsPath)
is_file = False
irodsFile = irodsOpen(conn, irodsPath, "r")
if not irodsFile:
is_file = True
else:
irodsDir = os.path.dirname(irodsPath)
coll = irodsCollection(conn, irodsDir)
if not coll:
return S_ERROR('Directory not found')
objects = coll.getObjects()
fileList = [x[0] for x in objects]
dirList = coll.getSubCollections()
resultDict = {}
if mode == 'l':
if is_file:
result = self.__getFileStat(dir_path)
if result['OK']:
resultDict[dir_path] = result['Value']
return S_OK(resultDict)
else:
return S_ERROR('Failed to get the file stat info')
else:
failed_list = []
one_OK = False
for fname in dirList + fileList:
result = self.__getFileStat(dir_path + '/' + fname)
if result['OK']:
resultDict[fname] = result['Value']
one_OK = True
else:
failed_list.append(fname)
if failed_list:
if one_OK:
result = S_ERROR(
'Failed partially to get the file stat info')
else:
result = S_ERROR('Failed to get the file stat info')
result['FailedList'] = failed_list
result['Value'] = resultDict
else:
result = S_OK(resultDict)
return result
else:
return S_OK(dirList)
def export_listDirectory( self, dir_path, mode ):
"""
Return the dir_path directory listing
"""
conn , error, userDict = self.__irodsClient( )
if not conn:
return S_ERROR( error )
file_path = self.__resolveFileID( dir_path, userDict )
irodsHome = userDict.get( 'iRodsHome', IRODS_HOME )
irodsPath = irodsHome + file_path
gLogger.debug( "file_path to read: %s" % irodsPath )
is_file = False
irodsFile = irodsOpen( conn , irodsPath , "r" )
if not irodsFile:
is_file = True
else:
irodsDir = os.path.dirname( irodsPath )
coll = irodsCollection( conn, irodsDir )
if not coll:
return S_ERROR( 'Directory not found' )
objects = coll.getObjects()
fileList = [ x[0] for x in objects ]
dirList = coll.getSubCollections()
resultDict = {}
if mode == 'l':
if is_file:
result = self.__getFileStat( dir_path )
if result['OK']:
resultDict[dir_path] = result['Value']
return S_OK( resultDict )
else:
return S_ERROR( 'Failed to get the file stat info' )
else:
failed_list = []
one_OK = False
for fname in dirList+fileList:
result = self.__getFileStat( dir_path + '/' + fname )
if result['OK']:
resultDict[fname] = result['Value']
one_OK = True
else:
failed_list.append( fname )
if failed_list:
if one_OK:
result = S_ERROR( 'Failed partially to get the file stat info' )
else:
result = S_ERROR( 'Failed to get the file stat info' )
result['FailedList'] = failed_list
result['Value'] = resultDict
else:
result = S_OK( resultDict )
return result
else:
return S_OK( dirList )
def irods_isdir(url, **kw):
with IrodsConnection(url, **kw) as ic:
coll = irods.irodsCollection(ic.conn) # pylint: disable=no-member
path = coll.openCollection(ic.path)
# Bug in openCollection (appends \0)
path = path.strip('\x00')
if path:
return True
return False
def create_irods_urls(url_list, rodsconfig):
from irods import irodsCollection, irodsOpen
for user in [u for u in get_user_list() if u.valid]:
conn = create_irods_connection(user.name, user.password, rodsconfig)
for obj in [o for o in url_list if o.exists]:
if obj.objtype == obj.ContainerType:
base, name = os.path.split(obj.path)
coll = irodsCollection(conn, base)
coll.createCollection(name)
elif obj.objtype == obj.FileType:
coll = irodsCollection(conn)
coll.createCollection(os.path.split(obj.path)[0])
file_handle = irodsOpen(conn, obj.path, "w")
if file_handle is not None:
file_handle.write(obj.objinfo["content"])
file_handle.close()
conn.disconnect()
def __init__(self, session, stream):
# Check for dependency
if irods is None:
raise MissingDependencyException(
'{0.__module__}.{0.__class__.__name__}'.format(self),
'irods (PyRods)'
)
# Check for URL
if stream.startswith(('irods://', 'rods://')):
myEnv = parse_irodsUrl(stream)
stream = myEnv.relpath
else:
# Get parameters from env
status, myEnv = irods.getRodsEnv()
try:
host = myEnv.getRodsHost()
port = myEnv.getRodsPort()
username = myEnv.getRodsUserName()
zone = myEnv.getRodsZone()
home = myEnv.getRodsHome()
except AttributeError:
host = myEnv.rodsHost
port = myEnv.rodsPort
username = myEnv.rodsUserName
zone = myEnv.rodsZone
home = myEnv.rodsHome
conn, errMsg = irods.rcConnect(host, port, username, zone)
status = irods.clientLogin(conn)
if status:
raise ConfigFileException("Cannot connect to iRODS: ({0}) {1}"
"".format(status, errMsg)
)
c = irods.irodsCollection(conn)
self.cxn = conn
self.coll = c
instream = stream
# Check if abs path to home dir
if stream.startswith(home):
stream = stream[len(home):]
if stream[0] == "/":
stream = stream[1:]
colls = stream.split('/')
for i, cln in enumerate(colls):
exit_status = c.openCollection(cln)
if exit_status < 0:
if (
(i < len(colls) - 1) or
(cln not in [obj[0] for obj in c.getObjects()])
):
raise IOError("When opening {0}: {1} does not exists in "
"collection {2}".format(instream,
cln,
c.getCollName()
)
)
def _open(self, session):
if self.cxn == None:
# connect to iRODS
myEnv, status = irods.getRodsEnv()
host = self.host if self.host else myEnv.getRodsHost()
port = self.port if self.port else myEnv.getRodsPort()
user = self.user if self.user else myEnv.getRodsUserName()
zone = self.zone if self.zone else myEnv.getRodsZone()
conn, errMsg = irods.rcConnect(host, port, user, zone)
if self.passwd:
status = irods.clientLoginWithPassword(conn, self.passwd)
else:
status = irods.clientLogin(conn)
if status:
raise ConfigFileException("Cannot connect to iRODS: (%s) %s" % (status, errMsg.getMsg()))
self.cxn = conn
self.env = myEnv
resources = irods.getResources(self.cxn)
self.resourceHash = {}
for r in resources:
self.resourceHash[r.getName()] = r
if self.coll != None:
# already open, just skip
return None
c = irods.irodsCollection(self.cxn, self.env.getRodsHome())
self.coll = c
# move into cheshire3 section
path = self.get_path(session, 'irodsCollection', 'cheshire3')
dirs = c.getSubCollections()
if not path in dirs:
c.createCollection(path)
c.openCollection(path)
if self.get_setting(session, 'createSubDir', 1):
# now look for object's storage area
# maybe move into database collection
if (isinstance(self.parent, Database)):
sc = self.parent.id
dirs = c.getSubCollections()
if not sc in dirs:
c.createCollection(sc)
c.openCollection(sc)
# move into store collection
dirs = c.getSubCollections()
if not self.id in dirs:
c.createCollection(self.id)
c.openCollection(self.id)
def is_irods_folder_exist(self, list_folders): isExist = True len_folder = len(list_folders) base = self.myEnv.rodsHome for i, item in enumerate(list_folders): base = addSlash(base, item) coll = irods.irodsCollection(self.conn, base) child = coll.getSubCollections() if not (set(child) & set(list_folders)) and i < len_folder - 2: isExist = False return isExist
def get_meta(self, coll_name):
coll = irods.irodsCollection(self.conn, coll_name)
meta = coll.getUserMetadata()
print("\nNOTICE: try to get metadata for a collection: %s" %coll_name)
sleep(1)
if meta:
print("SUCCESS")
else:
print("WARNING: parsing metadata for collection: %s with PyRODS gives us an empty result" %coll_name)
print("NOTICE: try to get metadata using subprocess.Popen")
meta = self.get_meta_with_subprocess(coll_name)
return meta
def __init__(self): self.status, self.myEnv = irods.getRodsEnv() self.conn, self.errMsg = irods.rcConnect(self.myEnv.rodsHost, self.myEnv.rodsPort, self.myEnv.rodsUserName, self.myEnv.rodsZone) self.status = irods.clientLogin(self.conn) self.path = makePath(self.myEnv.rodsHome, FOLDER) # addSlash(self.myEnv.rodsHome, reduce(addSlash, FOLDER)) self.coll = irods.irodsCollection(self.conn, self.path) self.popen_opt = dict(shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
def create_irods_urls(url_list, rodsconfig):
from irods import irodsCollection, irodsOpen
for user in [u for u in get_user_list() if u.valid]:
conn = create_irods_connection(user.name, user.password, rodsconfig)
for obj in [o for o in url_list if o.exists]:
if obj.objtype == obj.ContainerType:
base, name = os.path.split(obj.path)
coll = irodsCollection(conn, base)
coll.createCollection(name)
dir_handle = irodsCollection(conn, obj.path)
if dir_handle is not None:
dir_handle.addUserMetadata('objectID', obj.objectid)
elif obj.objtype == obj.FileType:
coll = irodsCollection(conn)
coll.createCollection(os.path.split(obj.path)[0])
file_handle = irodsOpen(conn, obj.path, 'w')
if file_handle is not None:
file_handle.write(obj.objinfo['content'])
file_handle.addUserMetadata('objectID', obj.objectid)
file_handle.close()
conn.disconnect()
def __getFileStat(self, path):
"""
Get the file stat information
"""
conn, error, userDict = self.__irodsClient()
if not conn:
return S_ERROR(error)
file_path = self.__resolveFileID(path, userDict)
irodsHome = userDict.get('iRodsHome', IRODS_HOME)
file_path = irodsHome + file_path
gLogger.debug("file_path to read: %s" % file_path)
resultDict = {}
irodsFile = irodsOpen(conn, file_path, "r")
if irodsFile:
resultDict['Exists'] = True
resultDict['File'] = True
resultDict['Directory'] = False
resultDict['Type'] = "File"
resultDict['Size'] = irodsFile.getSize()
resultDict['TimeStamps'] = (irodsFile.getCreateTs(),
irodsFile.getModifyTs(),
irodsFile.getCreateTs())
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = 0o755
resultDict = StorageBase._addCommonMetadata(resultDict)
return S_OK(resultDict)
else:
coll = irodsCollection(conn, file_path)
if coll:
resultDict['Exists'] = True
resultDict['Type'] = "Directory"
resultDict['File'] = False
resultDict['Directory'] = True
resultDict['Size'] = 0
resultDict['TimeStamps'] = (0, 0, 0)
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = 0o755
resultDict = StorageBase._addCommonMetadata(resultDict)
return S_OK(resultDict)
else:
return S_ERROR('Path does not exist')
def __getFileStat( self, path ):
"""
Get the file stat information
"""
conn , error, userDict = self.__irodsClient( )
if not conn:
return S_ERROR( error )
file_path = self.__resolveFileID( path, userDict )
irodsHome = userDict.get( 'iRodsHome', IRODS_HOME )
file_path = irodsHome + file_path
gLogger.debug( "file_path to read: %s" % file_path )
resultDict = {}
irodsFile = irodsOpen( conn , file_path , "r" )
if irodsFile:
resultDict['Exists'] = True
resultDict['File'] = True
resultDict['Directory'] = False
resultDict['Type'] = "File"
resultDict['Size'] = irodsFile.getSize()
resultDict['TimeStamps'] = ( irodsFile.getCreateTs(), irodsFile.getModifyTs(), irodsFile.getCreateTs() )
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = 0o755
resultDict = StorageBase._addCommonMetadata( resultDict )
return S_OK( resultDict )
else:
coll = irodsCollection( conn, file_path )
if coll:
resultDict['Exists'] = True
resultDict['Type'] = "Directory"
resultDict['File'] = False
resultDict['Directory'] = True
resultDict['Size'] = 0
resultDict['TimeStamps'] = ( 0,0,0 )
resultDict['Cached'] = 1
resultDict['Migrated'] = 0
resultDict['Lost'] = 0
resultDict['Unavailable'] = 0
resultDict['Mode'] = 0o755
resultDict = StorageBase._addCommonMetadata( resultDict )
return S_OK( resultDict )
else:
return S_ERROR( 'Path does not exist' )
def erase_irods_urls(url_list, rodsconfig):
from irods import irodsOpen, irodsCollection
for user in [u for u in get_user_list() if u.valid]:
conn = create_irods_connection(user.name, user.password, rodsconfig)
for obj in url_list:
file_handle = irodsOpen(conn, obj.path, "r")
if file_handle is not None:
file_handle.close()
file_handle.delete(force=True)
base, name = os.path.split(obj.path)
coll = irodsCollection(conn, base)
coll.deleteCollection(name)
conn.disconnect()
def __mkcolls( self, rods_path ):
"""
An os.makedirs() for iRODS collections. `rods_path` is the desired collection to create.
"""
assert rods_path.startswith( self.root_collection_path + '/' ), '__mkcolls(): Creating collections outside the root collection is not allowed (requested path was: %s)' % rods_path
mkcolls = []
c = irods.irodsCollection( self.rods_conn, rods_path )
while c.getId() == -1:
assert c.getCollName().startswith( self.root_collection_path + '/' ), '__mkcolls(): Attempted to move above the root collection: %s' % c.getCollName()
mkcolls.append( c.getCollName() )
c.upCollection()
for collname in reversed( mkcolls ):
log.debug( 'Creating collection %s' % collname )
ci = irods.collInp_t()
ci.collName = collname
status = irods.rcCollCreate( self.rods_conn, ci )
assert status == 0, '__mkcolls(): Failed to create collection: %s' % collname
def __mkcolls( self, rods_path ):
"""
An os.makedirs() for iRODS collections. `rods_path` is the desired collection to create.
"""
assert rods_path.startswith( self.root_collection_path + '/' ), '__mkcolls(): Creating collections outside the root collection is not allowed (requested path was: %s)' % rods_path
mkcolls = []
c = irods.irodsCollection( self.rods_conn, rods_path )
while c.getId() == -1:
assert c.getCollName().startswith( self.root_collection_path + '/' ), '__mkcolls(): Attempted to move above the root collection: %s' % c.getCollName()
mkcolls.append( c.getCollName() )
c.upCollection()
for collname in reversed( mkcolls ):
log.debug( 'Creating collection %s' % collname )
ci = irods.collInp_t()
ci.collName = collname
status = irods.rcCollCreate( self.rods_conn, ci )
assert status == 0, '__mkcolls(): Failed to create collection: %s' % collname
def _openIrods(self, session):
if self.cxn is None:
# connect to iRODS
myEnv, status = irods.getRodsEnv()
conn, errMsg = irods.rcConnect(myEnv.getRodsHost(),
myEnv.getRodsPort(),
myEnv.getRodsUserName(),
myEnv.getRodsZone()
)
status = irods.clientLogin(conn)
if status:
raise ConfigFileException("Cannot connect to iRODS: (%s) %s"
"" % (status, errMsg))
self.cxn = conn
self.env = myEnv
if self.coll is not None:
# already open, just skip
return None
c = irods.irodsCollection(self.cxn, self.env.getRodsHome())
self.coll = c
# move into cheshire3 section
path = self.get_path(session, 'irodsCollection', 'cheshire3')
dirs = c.getSubCollections()
if not path in dirs:
c.createCollection(path)
c.openCollection(path)
# now look for object's storage area
# maybe move into database collection
if (isinstance(self.parent, Database)):
sc = self.parent.id
dirs = c.getSubCollections()
if not sc in dirs:
c.createCollection(sc)
c.openCollection(sc)
# move into store collection
dirs = c.getSubCollections()
if not self.id in dirs:
c.createCollection(self.id)
c.openCollection(self.id)
def import_collection_to_package(params, id):
"""
Import a collection to dataset. Does not import whole file data but
rather the metadata.
"""
from irods import irodsCollection
path = params['path']
pkg = Package.get(id)
conn = get_connection_from_params(params)
if (conn):
coll = irodsCollection(conn, path)
from irods import iRodsOpen
rev = model.repo.new_revision()
i = 0
for obj in coll.getObjects():
extras = {}
fname, _ = obj
fpath = "%s/%s" % (coll.getCollName(), fname)
f = iRodsOpen(conn, fpath, 'r')
if f:
i += 1
res = Resource.by_name(fname)
if not res:
res = Resource(url = '', name=fname, extras=extras, \
resource_type='file')
for met in f.getUserMetadata():
key, value, _ = met
extras[key] = value
res.extras = extras
resgrp = pkg.resource_groups[0]
resgrp.resources.append(res)
Session.add(res)
Session.add(resgrp)
rev.message = "Update from iRODS, matched file %s" % fname
for met in coll.getUserMetadata():
key, value, _ = met
pkg.extras[key] = value
Session.add(pkg)
model.repo.commit()
conn.disconnect()
h.flash_success("iRODS import to dataset OK! Imported %s resources." %
i)
else:
h.flash_error("Could not connect to iRODS!")
h.redirect_to(controller='package', action='read', id=id)
def export_createDirectory( self, dir_path ):
""" Creates the directory on the storage
"""
conn , error, userDict = self.__irodsClient( )
if not conn:
return S_ERROR( error )
irodsHome = userDict.get( 'iRodsHome', IRODS_HOME )
coll = irodsCollection( conn, irodsHome )
dir_path = self.__resolveFileID( dir_path, userDict )
path = dir_path.split( "/" )
if len( path ) > 0:
coll = self.__changeCollection( coll , path )
if not coll:
return S_ERROR( 'Failed to create directory' )
return S_OK()
def import_collection_to_package(params, id):
"""
Import a collection to dataset. Does not import whole file data but
rather the metadata.
"""
from irods import irodsCollection
path = params['path']
pkg = Package.get(id)
conn = get_connection_from_params(params)
if (conn):
coll = irodsCollection(conn, path)
from irods import iRodsOpen
rev = model.repo.new_revision()
i = 0
for obj in coll.getObjects():
extras = {}
fname, _ = obj
fpath = "%s/%s" % (coll.getCollName(), fname)
f = iRodsOpen(conn, fpath, 'r')
if f:
i += 1
res = Resource.by_name(fname)
if not res:
res = Resource(url = '', name=fname, extras=extras, \
resource_type='file')
for met in f.getUserMetadata():
key, value, _ = met
extras[key] = value
res.extras = extras
resgrp = pkg.resource_groups[0]
resgrp.resources.append(res)
Session.add(res)
Session.add(resgrp)
rev.message = "Update from iRODS, matched file %s" % fname
for met in coll.getUserMetadata():
key, value, _ = met
pkg.extras[key] = value
Session.add(pkg)
model.repo.commit()
conn.disconnect()
h.flash_success("iRODS import to dataset OK! Imported %s resources." % i)
else:
h.flash_error("Could not connect to iRODS!")
h.redirect_to(controller='package', action='read', id=id)
def export_exists( self, fileID ):
""" Check existence of the fileID """
conn , error, userDict = self.__irodsClient( )
if not conn:
return S_ERROR( error )
file_path = self.__resolveFileID( fileID, userDict )
irodsHome = userDict.get( 'iRodsHome', IRODS_HOME )
file_path = irodsHome + file_path
gLogger.debug( "file_path to read: %s" % file_path )
irodsFile = irodsOpen( conn , file_path , "r" )
if irodsFile:
return S_OK( True )
else:
coll = irodsCollection( conn, file_path )
if coll:
return S_OK( True )
return S_OK( False )
def erase_irods_urls(url_list, rodsconfig):
from irods import irodsOpen, irodsCollection
for user in [u for u in get_user_list() if u.valid]:
conn = create_irods_connection(user.name, user.password, rodsconfig)
urls = url_list + [get_irods_copy_target_url(
rodsconfig[2], # rodszone
user
)]
for obj in urls:
file_handle = irodsOpen(conn, obj.path, 'w')
if file_handle is not None:
file_handle.close()
file_handle.delete(force=True)
else:
base, name = os.path.split(obj.path)
coll = irodsCollection(conn, base)
coll.deleteCollection(name)
conn.disconnect()
def irods_fetch_dir(url, **kw):
with IrodsConnection(url, **kw) as ic:
coll = irods.irodsCollection(ic.conn) # pylint: disable=no-member
path = coll.openCollection(ic.path)
# Bug in openCollection (appends \0)
path = path.strip('\x00')
#print 'path2 Null', '\x00' in path
result = []
# Construct urls for objects contained
for nm in coll.getSubCollections():
#print " '%s' '%s' '%s' " % (base_url, path, nm)
#print 'nm Null', '\x00' in nm
#print 'path3 Null', '\x00' in path
result.append('/'.join([ic.base_url, ic.path[1:], nm, '']))
for nm, resource in coll.getObjects():
result.append('/'.join([ic.base_url, ic.path[1:], nm]))
return result
def export_exists(self, fileID):
""" Check existence of the fileID """
conn, error, userDict = self.__irodsClient()
if not conn:
return S_ERROR(error)
file_path = self.__resolveFileID(fileID, userDict)
irodsHome = userDict.get('iRodsHome', IRODS_HOME)
file_path = irodsHome + file_path
gLogger.debug("file_path to read: %s" % file_path)
irodsFile = irodsOpen(conn, file_path, "r")
if irodsFile:
return S_OK(True)
else:
coll = irodsCollection(conn, file_path)
if coll:
return S_OK(True)
return S_OK(False)
def export_removeDirectory( self, fileID, token ):
""" Remove the given directory from the storage
"""
conn , error, userDict = self.__irodsClient( )
if not conn:
return S_ERROR( error )
file_path = self.__resolveFileID( fileID, userDict )
irodsHome = userDict.get( 'iRodsHome', IRODS_HOME )
file_path = irodsHome + file_path
gLogger.debug( "file_path to read: %s" % file_path )
coll = irodsCollection( conn, file_path )
if not coll:
return S_ERROR( 'Directory not found' )
objects = coll.getObjects()
if objects:
return S_ERROR( 'Directory is not empty' )
coll.delete()
return S_ERROR()
def export_removeDirectory(self, fileID, token):
""" Remove the given directory from the storage
"""
conn, error, userDict = self.__irodsClient()
if not conn:
return S_ERROR(error)
file_path = self.__resolveFileID(fileID, userDict)
irodsHome = userDict.get('iRodsHome', IRODS_HOME)
file_path = irodsHome + file_path
gLogger.debug("file_path to read: %s" % file_path)
coll = irodsCollection(conn, file_path)
if not coll:
return S_ERROR('Directory not found')
objects = coll.getObjects()
if objects:
return S_ERROR('Directory is not empty')
coll.delete()
return S_ERROR()
def open(self):
# pylint: disable=no-member
conn, _err = irods.rcConnect(self.host, self.port, self.user,
self.zone)
if conn is None:
raise IrodsError("Can't create connection to %s:%s " %
(self.host, _err))
if self.password:
irods.clientLoginWithPassword(conn, self.password)
else:
irods.clientLogin(conn)
coll = irods.irodsCollection(conn)
nm = coll.getCollName()
self.irods_url = urlparse.urlunparse(
list(self.irods_url)[:2] + [''] * 4)
if self.path in ['', '/']:
self.path = nm
self.conn = conn
self.base_dir = nm
return self
def irods_leaf_collections(self, gnode_object="analogsignals"): base = FOLDER[:-1] irods_coll = makePath(self.myEnv.rodsHome, base) coll = irods.irodsCollection(self.conn, irods_coll) child = coll.getSubCollections() return [makePath(irods_coll, [item]) for item in child]
def get_collection(self, path):
"""Get an iRodsCollection object from a path in iRODS"""
return irods.irodsCollection(self.conn, path)
def getListFolders(self): ''' returns list of sub-folders in rodsHome folder''' coll = irods.irodsCollection(self.conn, self.myEnv.rodsHome) return coll.getSubCollections()
def get_collection_nb_objects(self, irods_path):
"""Get the number of files of a collection from an irodsPath"""
c = irods.irodsCollection(self.conn, irods_path)
return c.getLenObjects()
def get_collection_objects(self, irods_path):
"""Get the list of the files of a collection from an irodsPath
The list is a list of irods.irodsFile"""
c = irods.irodsCollection(self.conn, irods_path)
return c.getObjects()
def get_collection_subCollections(self, irods_path):
"""Get the list of the subcollection of a collection from an irodsPath
The list is a list of irods.irodsCollection"""
c = irods.irodsCollection(self.conn, irods_path)
return c.getSubCollections()
def __get_rods_handle( self, obj, mode='r', **kwargs ):
if kwargs.get( 'dir_only', False ):
return irods.irodsCollection( self.rods_conn, self.__get_rods_path( obj, **kwargs ) )
else:
return irods.irodsOpen( self.rods_conn, self.__get_rods_path( obj, **kwargs ), mode )
def _open(self, session):
if self.cxn == None:
# connect to iRODS
myEnv, status = irods.getRodsEnv()
host = self.host if self.host else myEnv.getRodsHost()
port = self.port if self.port else myEnv.getRodsPort()
user = self.user if self.user else myEnv.getRodsUserName()
zone = self.zone if self.zone else myEnv.getRodsZone()
conn, errMsg = irods.rcConnect(host, port, user, zone)
if self.passwd:
status = irods.clientLoginWithPassword(conn, self.passwd)
else:
status = irods.clientLogin(conn)
if status:
raise ConfigFileException("Cannot connect to iRODS: (%s) %s" % (status, errMsg))
self.cxn = conn
self.env = myEnv
resources = irods.getResources(self.cxn)
self.resourceHash = {}
for r in resources:
self.resourceHash[r.getName()] = r
if self.coll != None:
# already open, just skip
return None
c = irods.irodsCollection(self.cxn, self.env.getRodsHome())
self.coll = c
# move into cheshire3 section
path = self.get_path(session, 'irodsCollection', 'cheshire3')
dirs = c.getSubCollections()
if not path in dirs:
c.createCollection(path)
c.openCollection(path)
if self.get_setting(session, 'createSubDir', 1):
# now look for object's storage area
# maybe move into database collection
if (isinstance(self.parent, Database)):
sc = self.parent.id
dirs = c.getSubCollections()
if not sc in dirs:
c.createCollection(sc)
c.openCollection(sc)
# move into store collection
dirs = c.getSubCollections()
if not self.id in dirs:
c.createCollection(self.id)
c.openCollection(self.id)
# Fetch user metadata
myMetadata = self.get_metadataTypes(session)
umd = c.getUserMetadata()
umdHash = {}
for u in umd:
umdHash[u[0]] = icatValToPy(*u[1:])
for md in myMetadata:
try:
setattr(self, md, umdHash[md])
except KeyError:
# hasn't been set yet
pass
if self.totalItems != 0:
self.meanWordCount = self.totalWordCount / self.totalItems
self.meanByteCount = self.totalByteCount / self.totalItems
else:
self.meanWordCount = 1
self.meanByteCount = 1
