def session(self, environment=None):
if getattr(settings, 'IRODS_GLOBAL_SESSION', False):
return GLOBAL_SESSION
if environment is None:
environment = IRodsEnv(
pk=-1,
host=settings.IRODS_HOST,
port=settings.IRODS_PORT,
def_res=settings.IRODS_DEFAULT_RESOURCE,
home_coll=settings.IRODS_HOME_COLLECTION,
cwd=settings.IRODS_CWD,
username=settings.IRODS_USERNAME,
zone=settings.IRODS_ZONE,
auth=settings.IRODS_AUTH
)
elif isinstance(environment, int):
environment = m.RodsEnvironment.objects.get(pk=environment)
if environment.pk not in self._sessions:
session = Session("/tmp/django_irods", settings.IRODS_ICOMMANDS_PATH, session_id=uuid4())
session.create_environment(environment)
session.run('iinit', None, environment.auth)
self._sessions[environment.pk] = session
return self._sessions[environment.pk]
def session(self, environment):
environment = m.RodsEnvironment.objects.get(pk=environment)
if environment.pk not in self._sessions:
session = Session("/tmp/ga_irods", settings.ICOMMANDS_DIR, session_id=uuid4())
session.create_environment(environment)
session.run('iinit', None, environment.auth)
self._sessions[environment.pk] = session
return self._sessions[environment.pk]
class IrodsAccount():
def __init__(self, option=None):
# always create a session associated with admin for iRODS account creation
self.session = Session()
self.session.run('iinit', None, self.session.create_environment().auth)
def create(self, uname):
self.session.admin('mkuser', uname, "rodsuser")
def setPassward(self, uname, upwd):
self.session.admin('moduser', uname, "password", upwd)
class IrodsAccount():
def __init__(self, option=None):
# always create a session associated with admin for iRODS account creation
self.session = Session()
self.session.run('iinit', None, self.session.create_environment().auth)
def create(self, uname):
self.session.admin('mkuser', uname, "rodsuser")
def setPassward(self, uname, upwd):
self.session.admin('moduser', uname, "password", upwd)
class IrodsStorage(Storage):
def __init__(self, option=None):
if option == 'federated':
# resource should be saved in federated zone
self.set_fed_zone_session()
else:
self.session = GLOBAL_SESSION
self.environment = GLOBAL_ENVIRONMENT
icommands.ACTIVE_SESSION = self.session
def set_user_session(self, username=None, password=None, host=settings.IRODS_HOST,
port=settings.IRODS_PORT, def_res=None, zone=settings.IRODS_ZONE,
userid=0, sess_id=None):
homedir = "/" + zone + "/home/" + username
userEnv = IRodsEnv(
pk=userid,
host=host,
port=port,
def_res=def_res,
home_coll=homedir,
cwd=homedir,
username=username,
zone=zone,
auth=password,
irods_default_hash_scheme='MD5'
)
if sess_id is None:
self.session = Session(session_id=uuid4())
self.environment = self.session.create_environment(myEnv=userEnv)
else:
self.session = Session(session_id=sess_id)
if self.session.session_file_exists():
self.environment = userEnv
else:
self.environment = self.session.create_environment(myEnv=userEnv)
self.session.run('iinit', None, self.environment.auth)
icommands.ACTIVE_SESSION = self.session
# Set iRODS session to wwwHydroProxy for irods_storage input object for iRODS federated
# zone direct file operations
def set_fed_zone_session(self):
if settings.REMOTE_USE_IRODS:
self.set_user_session(username=settings.HS_WWW_IRODS_PROXY_USER,
password=settings.HS_WWW_IRODS_PROXY_USER_PWD,
host=settings.HS_WWW_IRODS_HOST,
port=settings.IRODS_PORT,
def_res=settings.HS_IRODS_LOCAL_ZONE_DEF_RES,
zone=settings.HS_WWW_IRODS_ZONE,
sess_id='federated_session')
def delete_user_session(self):
if self.session != GLOBAL_SESSION and self.session.session_file_exists():
self.session.delete_environment()
def download(self, name):
return self._open(name, mode='rb')
def getFile(self, src_name, dest_name):
self.session.run("iget", None, '-f', src_name, dest_name)
def runBagitRule(self, rule_name, input_path, input_resource):
"""
run iRODS bagit rule which generated bag-releated files without bundling
:param rule_name: the iRODS rule name to run
:param input_path: input parameter to the rule that indicates the collection path to
create bag for
:param input_resource: input parameter to the rule that indicates the default resource
to store generated bag files
:return: None
"""
# SessionException will be raised from run() in icommands.py
self.session.run("irule", None, '-F', rule_name, input_path, input_resource)
def zipup(self, in_name, out_name):
"""
run iRODS ibun command to generate zip file for the bag
:param in_name: input parameter to indicate the collection path to generate zip
:param out_name: the output zipped file name
:return: None
"""
self.session.run("imkdir", None, '-p', out_name.rsplit('/', 1)[0])
# SessionException will be raised from run() in icommands.py
self.session.run("ibun", None, '-cDzip', '-f', out_name, in_name)
def set_access_control(self, permission, user_or_group_name, coll_name, recursive=True):
"""
modify access to iRODS collection
:param permission: can only be null, read, write, or own.
:param user_or_group_name: iRODS username or groupname to be assigned with permission
:param coll_name: iRODS collection name or data object to assign permission to.
:return:
"""
if permission not in ('null', 'read', 'write', 'own'):
raise ValueError('bad input argument: only null, read, write, or own for permission is allowed.')
if recursive:
self.session.run("ichmod", None, '-r', permission, user_or_group_name, coll_name)
else:
self.session.run("ichmod", None, permission, user_or_group_name, coll_name)
def setAVU(self, name, attName, attVal, attUnit=None):
"""
set AVU on resource collection - this is used for on-demand bagging by indicating
whether the resource has been modified via AVU pairs
Parameters:
:param
name: the resource collection name to set AVU.
attName: the attribute name to set
attVal: the attribute value to set
attUnit: the attribute Unit to set, default is None, but can be set to
indicate additional info
"""
# SessionException will be raised from run() in icommands.py
if attUnit:
self.session.run("imeta", None, 'set', '-C', name, attName, attVal, attUnit)
else:
self.session.run("imeta", None, 'set', '-C', name, attName, attVal)
def getAVU(self, name, attName=None, type='-C'):
"""
get AVU for attName or all AVUs if attName is None, on resource collection (by default) or on other types
such as passing in '-d' for type for data objects
Parameters:
:param
name: the name of the type (e.g., collection or data object) to get AVU for.
attName: the attribute name to get AVU for, if None, all AVU pairs are returned as a dict
type: default is '-C' which means collection. Other options are '-d' for data object, '-u' for user,
-R for resource
"""
# SessionException will be raised from run() in icommands.py
if attName:
stdout = self.session.run("imeta", None, 'ls', type, name, attName)[0].split("\n")
ret_att = stdout[1].strip()
if ret_att == 'None': # queried attribute does not exist
return None
else:
vals = stdout[2].split(":")
return vals[1].strip()
else:
# get all AVUs and return as a dict
avu_dict={}
stdout = self.session.run("imeta", None, 'ls', type, name)[0].split("\n")
# stdout is a list in the following format:
# [
# 'AVU defined for ...',
# 'attribute: att1',
# 'value: val1',
# 'units: unit1',
# '----'
# ... continue to the next AVU pairs if any
# ]
idx = 1
while idx < len(stdout):
att_idx = idx
val_idx = idx + 1
ret_att = stdout[att_idx].strip()
if ret_att == 'None':
return avu_dict
atts = ret_att.split(':')
att = atts[1].strip()
ret_val = stdout[val_idx].strip()
vals = ret_val.split(':')
val = vals[1].strip()
avu_dict[att] = val
idx += 4
return avu_dict
def copyFiles(self, src_name, dest_name):
"""
Parameters:
:param
src_name: the iRODS data-object or collection name to be copied from.
dest_name: the iRODS data-object or collection name to be copied to
copyFiles() copied an irods data-object (file) or collection (directory)
to another data-object or collection
"""
if src_name and dest_name:
if '/' in dest_name:
splitstrs = dest_name.rsplit('/', 1)
if not self.exists(splitstrs[0]):
self.session.run("imkdir", None, '-p', splitstrs[0])
self.session.run("icp", None, '-rf', src_name, dest_name)
return
def moveFile(self, src_name, dest_name):
"""
Parameters:
:param
src_name: the iRODS data-object or collection name to be moved from.
dest_name: the iRODS data-object or collection name to be moved to
moveFile() moves/renames an irods data-object (file) or collection
(directory) to another data-object or collection
"""
if src_name and dest_name:
if '/' in dest_name:
splitstrs = dest_name.rsplit('/', 1)
if not self.exists(splitstrs[0]):
self.session.run("imkdir", None, '-p', splitstrs[0])
self.session.run("imv", None, src_name, dest_name)
return
def saveFile(self, from_name, to_name, create_directory=False, data_type_str=''):
"""
Parameters:
:param
from_name: the temporary file name in local disk to be uploaded from.
to_name: the data object path in iRODS to be uploaded to
create_directory: create directory as needed when set to True. Default is False
Note if only directory needs to be created without saving a file, from_name should be empty
and to_name should have "/" as the last character
"""
if create_directory:
splitstrs = to_name.rsplit('/', 1)
self.session.run("imkdir", None, '-p', splitstrs[0])
if len(splitstrs) <= 1:
return
if from_name:
try:
if data_type_str:
self.session.run("iput", None, '-D', data_type_str, '-f', from_name, to_name)
else:
self.session.run("iput", None, '-f', from_name, to_name)
except:
if data_type_str:
self.session.run("iput", None, '-D', data_type_str, '-f', from_name, to_name)
else:
# IRODS 4.0.2, sometimes iput fails on the first try.
# A second try seems to fix it.
self.session.run("iput", None, '-f', from_name, to_name)
return
def _open(self, name, mode='rb'):
tmp = NamedTemporaryFile()
self.session.run("iget", None, '-f', name, tmp.name)
return tmp
def _save(self, name, content):
self.session.run("imkdir", None, '-p', name.rsplit('/', 1)[0])
with NamedTemporaryFile(delete=False) as f:
for chunk in content.chunks():
f.write(chunk)
f.flush()
f.close()
try:
self.session.run("iput", None, '-f', f.name, name)
except:
# IRODS 4.0.2, sometimes iput fails on the first try. A second try seems to fix it.
self.session.run("iput", None, '-f', f.name, name)
os.unlink(f.name)
return name
def delete(self, name):
self.session.run("irm", None, "-rf", name)
def exists(self, name):
try:
stdout = self.session.run("ils", None, name)[0]
return stdout != ""
except SessionException:
return False
def listdir(self, path):
stdout = self.session.run("ils", None, path)[0].split("\n")
listing = ([], [])
directory = stdout[0][0:-1]
directory_prefix = " C- " + directory + "/"
for i in range(1, len(stdout)):
if stdout[i][:len(directory_prefix)] == directory_prefix:
dirname = stdout[i][len(directory_prefix):].strip()
if dirname:
listing[0].append(dirname)
else:
filename = stdout[i].strip()
if filename:
listing[1].append(filename)
return listing
def size(self, name):
stdout = self.session.run("ils", None, "-l", name)[0].split()
return int(stdout[3])
def url(self, name):
return reverse('django_irods_download', kwargs={'path': name})
def get_available_name(self, name, max_length=None):
"""
Reject duplicate file names rather than renaming them.
"""
if self.exists(name):
raise ValidationError(str.format("File {} already exists.", name))
return name
def get_checksum(self, name):
stdout = self.session.run("ils", None, "-L", name)[0].split()
for a in stdout:
if a.startswith('sha') or a.startswith('md5'):
return a
return None
def checksum(self, name):
stdout = self.session.run("ichksum", None, name)[0].split()
for a in stdout:
if a.startswith('sha') or a.startswith('md5'):
return a
return None
class IrodsStorage(Storage):
def __init__(self, option=None):
if option == 'federated':
# resource should be saved in federated zone
self.set_fed_zone_session()
else:
self.session = GLOBAL_SESSION
self.environment = GLOBAL_ENVIRONMENT
icommands.ACTIVE_SESSION = self.session
@property
def getUniqueTmpPath(self):
# return a unique temporary path under IRODS_ROOT directory
return os.path.join(getattr(settings, 'IRODS_ROOT', '/tmp'), uuid4().hex)
def set_user_session(self, username=None, password=None, host=settings.IRODS_HOST,
port=settings.IRODS_PORT, def_res=None, zone=settings.IRODS_ZONE,
userid=0, sess_id=None):
homedir = "/" + zone + "/home/" + username
userEnv = IRodsEnv(
pk=userid,
host=host,
port=port,
def_res=def_res,
home_coll=homedir,
cwd=homedir,
username=username,
zone=zone,
auth=password,
irods_default_hash_scheme='MD5'
)
if sess_id is None:
self.session = Session(session_id=uuid4())
self.environment = self.session.create_environment(myEnv=userEnv)
else:
self.session = Session(session_id=sess_id)
if self.session.session_file_exists():
self.environment = userEnv
else:
self.environment = self.session.create_environment(myEnv=userEnv)
self.session.run('iinit', None, self.environment.auth)
icommands.ACTIVE_SESSION = self.session
# Set iRODS session to wwwHydroProxy for irods_storage input object for iRODS federated
# zone direct file operations
def set_fed_zone_session(self):
if settings.REMOTE_USE_IRODS:
self.set_user_session(username=settings.HS_WWW_IRODS_PROXY_USER,
password=settings.HS_WWW_IRODS_PROXY_USER_PWD,
host=settings.HS_WWW_IRODS_HOST,
port=settings.IRODS_PORT,
def_res=settings.HS_IRODS_LOCAL_ZONE_DEF_RES,
zone=settings.HS_WWW_IRODS_ZONE,
sess_id='federated_session')
def delete_user_session(self):
if self.session != GLOBAL_SESSION and self.session.session_file_exists():
self.session.delete_environment()
def download(self, name):
return self._open(name, mode='rb')
def getFile(self, src_name, dest_name):
self.session.run("iget", None, '-f', src_name, dest_name)
def runBagitRule(self, rule_name, input_path, input_resource):
"""
run iRODS bagit rule which generated bag-releated files without bundling
:param rule_name: the iRODS rule name to run
:param input_path: input parameter to the rule that indicates the collection path to
create bag for
:param input_resource: input parameter to the rule that indicates the default resource
to store generated bag files
:return: None
"""
# SessionException will be raised from run() in icommands.py
self.session.run("irule", None, '-F', rule_name, input_path, input_resource)
def zipup(self, in_name, out_name):
"""
run iRODS ibun command to generate zip file for the bag
:param in_name: input parameter to indicate the collection path to generate zip
:param out_name: the output zipped file name
:return: None
"""
self.session.run("imkdir", None, '-p', out_name.rsplit('/', 1)[0])
# SessionException will be raised from run() in icommands.py
self.session.run("ibun", None, '-cDzip', '-f', out_name, in_name)
def unzip(self, zip_file_path, unzipped_folder=None):
"""
run iRODS ibun command to unzip files into a new folder
:param zip_file_path: path of the zipped file to be unzipped
:param unzipped_folder: Optional defaults to the basename of zip_file_path when not
provided. The folder to unzip to.
:return: the folder files were unzipped to
"""
abs_path = os.path.dirname(zip_file_path)
if not unzipped_folder:
unzipped_folder = os.path.splitext(os.path.basename(zip_file_path))[0].strip()
unzipped_folder = self._get_nonexistant_path(os.path.join(abs_path, unzipped_folder))
# SessionException will be raised from run() in icommands.py
self.session.run("ibun", None, '-xDzip', zip_file_path, unzipped_folder)
return unzipped_folder
def _get_nonexistant_path(self, path):
if not self.exists(path):
return path
i = 1
new_path = "{}-{}".format(path, i)
while self.exists(new_path):
i += 1
new_path = "{}-{}".format(path, i)
return new_path
def setAVU(self, name, attName, attVal, attUnit=None):
"""
set AVU on resource collection - this is used for on-demand bagging by indicating
whether the resource has been modified via AVU pairs
Parameters:
:param
name: the resource collection name to set AVU.
attName: the attribute name to set
attVal: the attribute value to set
attUnit: the attribute Unit to set, default is None, but can be set to
indicate additional info
"""
# SessionException will be raised from run() in icommands.py
if attUnit:
self.session.run("imeta", None, 'set', '-C', name, attName, attVal, attUnit)
else:
self.session.run("imeta", None, 'set', '-C', name, attName, attVal)
def getAVU(self, name, attName):
"""
set AVU on resource collection - this is used for on-demand bagging by indicating
whether the resource has been modified via AVU pairs
Parameters:
:param
name: the resource collection name to set AVU.
attName: the attribute name to set
attVal: the attribute value to set
attUnit: the attribute Unit to set, default is None, but can be set to
indicate additional info
"""
# SessionException will be raised from run() in icommands.py
stdout = self.session.run("imeta", None, 'ls', '-C', name, attName)[0].split("\n")
ret_att = stdout[1].strip()
if ret_att == 'None': # queried attribute does not exist
return None
else:
vals = stdout[2].split(":")
return vals[1].strip()
def copyFiles(self, src_name, dest_name, ires=None):
"""
Parameters:
:param
src_name: the iRODS data-object or collection name to be copied from.
dest_name: the iRODS data-object or collection name to be copied to
copyFiles() copied an irods data-object (file) or collection (directory)
to another data-object or collection
"""
if src_name and dest_name:
if '/' in dest_name:
splitstrs = dest_name.rsplit('/', 1)
if not self.exists(splitstrs[0]):
self.session.run("imkdir", None, '-p', splitstrs[0])
if ires:
self.session.run("icp", None, '-rf', '-R', ires, src_name, dest_name)
else:
self.session.run("icp", None, '-rf', src_name, dest_name)
return
def moveFile(self, src_name, dest_name):
"""
Parameters:
:param
src_name: the iRODS data-object or collection name to be moved from.
dest_name: the iRODS data-object or collection name to be moved to
moveFile() moves/renames an irods data-object (file) or collection
(directory) to another data-object or collection
"""
if src_name and dest_name:
if '/' in dest_name:
splitstrs = dest_name.rsplit('/', 1)
if not self.exists(splitstrs[0]):
self.session.run("imkdir", None, '-p', splitstrs[0])
self.session.run("imv", None, src_name, dest_name)
return
def saveFile(self, from_name, to_name, create_directory=False, data_type_str=''):
"""
Parameters:
:param
from_name: the temporary file name in local disk to be uploaded from.
to_name: the data object path in iRODS to be uploaded to
create_directory: create directory as needed when set to True. Default is False
Note if only directory needs to be created without saving a file, from_name should be empty
and to_name should have "/" as the last character
"""
if create_directory:
splitstrs = to_name.rsplit('/', 1)
self.session.run("imkdir", None, '-p', splitstrs[0])
if len(splitstrs) <= 1:
return
if from_name:
try:
if data_type_str:
self.session.run("iput", None, '-D', data_type_str, '-f', from_name, to_name)
else:
self.session.run("iput", None, '-f', from_name, to_name)
except:
if data_type_str:
self.session.run("iput", None, '-D', data_type_str, '-f', from_name, to_name)
else:
# IRODS 4.0.2, sometimes iput fails on the first try.
# A second try seems to fix it.
self.session.run("iput", None, '-f', from_name, to_name)
return
def _open(self, name, mode='rb'):
tmp = NamedTemporaryFile()
self.session.run("iget", None, '-f', name, tmp.name)
return tmp
def _save(self, name, content):
self.session.run("imkdir", None, '-p', name.rsplit('/', 1)[0])
with NamedTemporaryFile(delete=False) as f:
for chunk in content.chunks():
f.write(chunk)
f.flush()
f.close()
try:
self.session.run("iput", None, '-f', f.name, name)
except:
# IRODS 4.0.2, sometimes iput fails on the first try. A second try seems to fix it.
self.session.run("iput", None, '-f', f.name, name)
os.unlink(f.name)
return name
def delete(self, name):
self.session.run("irm", None, "-rf", name)
def exists(self, name):
try:
stdout = self.session.run("ils", None, name)[0]
return stdout != ""
except SessionException:
return False
def ils_l(self, path):
# in it's own method to mock for testing
return self.session.run("ils", None, "-l", path)[0]
def listdir(self, path):
stdout = self.ils_l(path).split("\n")
listing = ([], [], [])
directory = stdout[0][0:-1]
directory_prefix = " C- " + directory + "/"
for i in range(1, len(stdout)):
if stdout[i][:len(directory_prefix)] == directory_prefix:
dirname = stdout[i][len(directory_prefix):].strip()
if dirname:
listing[0].append(dirname)
listing[2].append("-1")
else:
# don't use split for filename to preserve spaces in filename
line = stdout[i].split(None, 6)
if len(line) < 6:
# the last line is empty
continue
if line[1] != '0':
# filter replicas
continue
# create a seperator based off the id, date, &
sep = " ".join(line[3:6])
filename = stdout[i].split(sep)[1].strip()
size = line[3]
if filename:
listing[1].append(filename)
listing[2].append(size)
return listing
def size(self, name):
stdout = self.session.run("ils", None, "-l", name)[0].split()
return int(stdout[3])
def url(self, name, url_download=False, zipped=False):
reverse_url = reverse('django_irods_download', kwargs={'path': name})
query_params = {'url_download': url_download, "zipped": zipped}
return reverse_url + '?' + urlencode(query_params)
def get_available_name(self, name, max_length=None):
"""
Reject duplicate file names rather than renaming them.
"""
if self.exists(name):
raise ValidationError(str.format("File {} already exists.", name))
return name
class IrodsStorage(Storage):
def __init__(self, option=None):
if option == 'federated':
# resource should be saved in federated zone
self.set_fed_zone_session()
else:
self.session = GLOBAL_SESSION
self.environment = GLOBAL_ENVIRONMENT
icommands.ACTIVE_SESSION = self.session
@property
def getUniqueTmpPath(self):
# return a unique temporary path under IRODS_ROOT directory
return os.path.join(getattr(settings, 'IRODS_ROOT', '/tmp'),
uuid4().hex)
def set_user_session(self,
username=None,
password=None,
host=settings.IRODS_HOST,
port=settings.IRODS_PORT,
def_res=None,
zone=settings.IRODS_ZONE,
userid=0,
sess_id=None):
homedir = "/" + zone + "/home/" + username
userEnv = IRodsEnv(pk=userid,
host=host,
port=port,
def_res=def_res,
home_coll=homedir,
cwd=homedir,
username=username,
zone=zone,
auth=password,
irods_default_hash_scheme='MD5')
if sess_id is None:
self.session = Session(session_id=uuid4())
self.environment = self.session.create_environment(myEnv=userEnv)
else:
self.session = Session(session_id=sess_id)
if self.session.session_file_exists():
self.environment = userEnv
else:
self.environment = self.session.create_environment(
myEnv=userEnv)
self.session.run('iinit', None, self.environment.auth)
icommands.ACTIVE_SESSION = self.session
# Set iRODS session to wwwHydroProxy for irods_storage input object for iRODS federated
# zone direct file operations
def set_fed_zone_session(self):
if settings.REMOTE_USE_IRODS:
self.set_user_session(
username=settings.HS_WWW_IRODS_PROXY_USER,
password=settings.HS_WWW_IRODS_PROXY_USER_PWD,
host=settings.HS_WWW_IRODS_HOST,
port=settings.IRODS_PORT,
def_res=settings.HS_IRODS_LOCAL_ZONE_DEF_RES,
zone=settings.HS_WWW_IRODS_ZONE,
sess_id='federated_session')
def delete_user_session(self):
if self.session != GLOBAL_SESSION and self.session.session_file_exists(
):
self.session.delete_environment()
def download(self, name):
return self._open(name, mode='rb')
def getFile(self, src_name, dest_name):
self.session.run("iget", None, '-f', src_name, dest_name)
def runBagitRule(self, rule_name, input_path, input_resource):
"""
run iRODS bagit rule which generated bag-releated files without bundling
:param rule_name: the iRODS rule name to run
:param input_path: input parameter to the rule that indicates the collection path to
create bag for
:param input_resource: input parameter to the rule that indicates the default resource
to store generated bag files
:return: None
"""
# SessionException will be raised from run() in icommands.py
self.session.run("irule", None, '-F', rule_name, input_path,
input_resource)
def zipup(self, in_name, out_name):
"""
run iRODS ibun command to generate zip file for a bag or other folder
:param in_name: fully qualified irods name of folder to be zipped
:param out_name: fully qualified irods name of output zipfile.
:return: None
"""
# make directory containing output zipfile if necessary
self.session.run("imkdir", None, '-p', out_name.rsplit('/', 1)[0])
# SessionException will be raised from run() in icommands.py
self.session.run("ibun", None, '-cDzip', '-f', out_name, in_name)
def unzip(self, zip_file_path, unzipped_folder=None):
"""
run iRODS ibun command to unzip files into a new folder
:param zip_file_path: path of the zipped file to be unzipped
:param unzipped_folder: Optional defaults to the basename of zip_file_path when not
provided. The folder to unzip to.
:return: the folder files were unzipped to
"""
abs_path = os.path.dirname(zip_file_path)
if not unzipped_folder:
unzipped_folder = os.path.splitext(
os.path.basename(zip_file_path))[0].strip()
unzipped_folder = self._get_nonexistant_path(
os.path.join(abs_path, unzipped_folder))
# SessionException will be raised from run() in icommands.py
self.session.run("ibun", None, '-xDzip', zip_file_path,
unzipped_folder)
return unzipped_folder
def _get_nonexistant_path(self, path):
if not self.exists(path):
return path
i = 1
new_path = "{}-{}".format(path, i)
while self.exists(new_path):
i += 1
new_path = "{}-{}".format(path, i)
return new_path
def setAVU(self, name, attName, attVal, attUnit=None):
"""
set AVU on resource collection - this is used for on-demand bagging by indicating
whether the resource has been modified via AVU pairs
Parameters:
:param
name: the resource collection name to set AVU.
attName: the attribute name to set
attVal: the attribute value to set
attUnit: the attribute Unit to set, default is None, but can be set to
indicate additional info
"""
# SessionException will be raised from run() in icommands.py
if attUnit:
self.session.run("imeta", None, 'set', '-C', name, attName, attVal,
attUnit)
else:
self.session.run("imeta", None, 'set', '-C', name, attName, attVal)
def getAVU(self, name, attName):
"""
set AVU on resource collection - this is used for on-demand bagging by indicating
whether the resource has been modified via AVU pairs
Parameters:
:param
name: the resource collection name to set AVU.
attName: the attribute name to set
attVal: the attribute value to set
attUnit: the attribute Unit to set, default is None, but can be set to
indicate additional info
"""
# SessionException will be raised from run() in icommands.py
stdout = self.session.run("imeta", None, 'ls', '-C', name,
attName)[0].split("\n")
ret_att = stdout[1].strip()
if ret_att == 'None': # queried attribute does not exist
return None
else:
vals = stdout[2].split(":")
return vals[1].strip()
def copyFiles(self, src_name, dest_name, ires=None):
"""
Parameters:
:param
src_name: the iRODS data-object or collection name to be copied from.
dest_name: the iRODS data-object or collection name to be copied to
copyFiles() copied an irods data-object (file) or collection (directory)
to another data-object or collection
"""
if src_name and dest_name:
if '/' in dest_name:
splitstrs = dest_name.rsplit('/', 1)
if not self.exists(splitstrs[0]):
self.session.run("imkdir", None, '-p', splitstrs[0])
if ires:
self.session.run("icp", None, '-rf', '-R', ires, src_name,
dest_name)
else:
self.session.run("icp", None, '-rf', src_name, dest_name)
return
def moveFile(self, src_name, dest_name):
"""
Parameters:
:param
src_name: the iRODS data-object or collection name to be moved from.
dest_name: the iRODS data-object or collection name to be moved to
moveFile() moves/renames an irods data-object (file) or collection
(directory) to another data-object or collection
"""
if src_name and dest_name:
if '/' in dest_name:
splitstrs = dest_name.rsplit('/', 1)
if not self.exists(splitstrs[0]):
self.session.run("imkdir", None, '-p', splitstrs[0])
self.session.run("imv", None, src_name, dest_name)
return
def saveFile(self,
from_name,
to_name,
create_directory=False,
data_type_str=''):
"""
Parameters:
:param
from_name: the temporary file name in local disk to be uploaded from.
to_name: the data object path in iRODS to be uploaded to
create_directory: create directory as needed when set to True. Default is False
Note if only directory needs to be created without saving a file, from_name should be empty
and to_name should have "/" as the last character
"""
if create_directory:
splitstrs = to_name.rsplit('/', 1)
self.session.run("imkdir", None, '-p', splitstrs[0])
if len(splitstrs) <= 1:
return
if from_name:
try:
if data_type_str:
self.session.run("iput", None, '-D', data_type_str, '-f',
from_name, to_name)
else:
self.session.run("iput", None, '-f', from_name, to_name)
except:
if data_type_str:
self.session.run("iput", None, '-D', data_type_str, '-f',
from_name, to_name)
else:
# IRODS 4.0.2, sometimes iput fails on the first try.
# A second try seems to fix it.
self.session.run("iput", None, '-f', from_name, to_name)
return
def _open(self, name, mode='rb'):
tmp = NamedTemporaryFile()
self.session.run("iget", None, '-f', name, tmp.name)
return tmp
def _save(self, name, content):
self.session.run("imkdir", None, '-p', name.rsplit('/', 1)[0])
with NamedTemporaryFile(delete=False) as f:
for chunk in content.chunks():
f.write(chunk)
f.flush()
f.close()
try:
self.session.run("iput", None, '-f', f.name, name)
except:
# IRODS 4.0.2, sometimes iput fails on the first try. A second try seems to fix it.
self.session.run("iput", None, '-f', f.name, name)
os.unlink(f.name)
return name
def delete(self, name):
self.session.run("irm", None, "-rf", name)
def exists(self, name):
try:
stdout = self.session.run("ils", None, name)[0]
return stdout != ""
except SessionException:
return False
def ils_l(self, path):
# in it's own method to mock for testing
return self.session.run("ils", None, "-l", path)[0]
def listdir(self, path):
stdout = self.ils_l(path).split("\n")
listing = ([], [], [])
directory = stdout[0][0:-1]
directory_prefix = " C- " + directory + "/"
for i in range(1, len(stdout)):
if stdout[i][:len(directory_prefix)] == directory_prefix:
dirname = stdout[i][len(directory_prefix):].strip()
if dirname:
listing[0].append(dirname)
listing[2].append("-1")
else:
# don't use split for filename to preserve spaces in filename
line = stdout[i].split(None, 6)
if len(line) < 6:
# the last line is empty
continue
if line[1] != '0':
# filter replicas
continue
# create a seperator based off the id, date, &
sep = " ".join(line[3:6])
filename = stdout[i].split(sep)[1].strip()
size = line[3]
if filename:
listing[1].append(filename)
listing[2].append(size)
return listing
def size(self, name):
stdout = self.session.run("ils", None, "-l", name)[0].split()
return int(stdout[3])
def url(self, name, url_download=False, zipped=False):
reverse_url = reverse('django_irods_download', kwargs={'path': name})
query_params = {'url_download': url_download, "zipped": zipped}
return reverse_url + '?' + urlencode(query_params)
def get_available_name(self, name, max_length=None):
"""
Reject duplicate file names rather than renaming them.
"""
if self.exists(name):
raise ValidationError(str.format("File {} already exists.", name))
return name
class IrodsStorage(Storage):
def __init__(self, option=None):
self.session = GLOBAL_SESSION
self.environment = GLOBAL_ENVIRONMENT
icommands.ACTIVE_SESSION = self.session
def set_user_session(self, username=None, password=None, host=settings.IRODS_HOST, port=settings.IRODS_PORT, def_res=None, zone=settings.IRODS_ZONE, userid=0, sessid='None'):
homedir = "/"+zone+"/home/"+username
userEnv = IRodsEnv(
pk=userid,
host=host,
port=port,
def_res=def_res,
home_coll=homedir,
cwd=homedir,
username=username,
zone=zone,
auth=password
)
self.session = Session(session_id=sessid)
self.environment = self.session.create_environment(myEnv=userEnv)
self.session.run('iinit', None, self.environment.auth)
icommands.ACTIVE_SESSION = self.session
def download(self, name):
return self._open(name, mode='rb')
def runBagitRule(self, rule_name, input_path, input_resource):
"""
run iRODS bagit rule which generated bag-releated files without bundling
:param rule_name: the iRODS rule name to run
:param input_path: input parameter to the rule that indicates the collection path to create bag for
:param input_resource: input parameter to the rule that indicates the default resource to store generated bag files
:return: None
"""
# SessionException will be raised from run() in icommands.py
self.session.run("irule", None, '-F', rule_name, input_path, input_resource)
def zipup(self, in_name, out_name):
"""
run iRODS ibun command to generate zip file for the bag
:param in_name: input parameter to indicate the collection path to generate zip
:param out_name: the output zipped file name
:return: None
"""
self.session.run("imkdir", None, '-p', out_name.rsplit('/',1)[0])
# SessionException will be raised from run() in icommands.py
self.session.run("ibun", None, '-cDzip', '-f', out_name, in_name)
def setAVU(self, name, attName, attVal, attUnit=None):
"""
set AVU on resource collection - this is used for on-demand bagging by indicating
whether the resource has been modified via AVU pairs
Parameters:
:param
name: the resource collection name to set AVU.
attName: the attribute name to set
attVal: the attribute value to set
attUnit: the attribute Unit to set, default is None, but can be set to indicate additional info
"""
# SessionException will be raised from run() in icommands.py
if attUnit:
self.session.run("imeta", None, 'set', '-C', name, attName, attVal, attUnit)
else:
self.session.run("imeta", None, 'set', '-C', name, attName, attVal)
def getAVU(self, name, attName):
"""
set AVU on resource collection - this is used for on-demand bagging by indicating
whether the resource has been modified via AVU pairs
Parameters:
:param
name: the resource collection name to set AVU.
attName: the attribute name to set
attVal: the attribute value to set
attUnit: the attribute Unit to set, default is None, but can be set to indicate additional info
"""
# SessionException will be raised from run() in icommands.py
stdout = self.session.run("imeta", None, 'ls', '-C', name, attName)[0].split("\n")
ret_att = stdout[1].strip()
if ret_att == 'None': # queried attribute does not exist
return None
else:
vals = stdout[2].split(":")
return vals[1].strip()
def copyFiles(self, src_name, dest_name):
"""
Parameters:
:param
src_name: the iRODS data-object or collection name to be copied from.
dest_name: the iRODS data-object or collection name to be copied to
copyFiles() copied an irods data-object (file) or collection (directory) to another data-object or collection
"""
if src_name and dest_name:
self.session.run("icp", None, '-rf', src_name, dest_name)
return
def saveFile(self, from_name, to_name, create_directory = False):
"""
Parameters:
:param
from_name: the temporary file name in local disk to be uploaded from.
to_name: the data object path in iRODS to be uploaded to
create_directory: create directory as needed when set to True. Default is False
Note if only directory needs to be created without saving a file, from_name should be empty
and to_name should have "/" as the last character
"""
if create_directory:
splitstrs = to_name.rsplit('/', 1)
self.session.run("imkdir", None, '-p', splitstrs[0])
if len(splitstrs) <= 1:
return
if from_name:
try:
self.session.run("iput", None, '-f', from_name, to_name)
except:
self.session.run("iput", None, '-f', from_name, to_name) # IRODS 4.0.2, sometimes iput fails on the first try. A second try seems to fix it.
return
def _open(self, name, mode='rb'):
tmp = NamedTemporaryFile()
self.session.run("iget", None, '-f', name, tmp.name)
return tmp
def _save(self, name, content):
self.session.run("imkdir", None, '-p', name.rsplit('/',1)[0])
with NamedTemporaryFile(delete=False) as f:
for chunk in content.chunks():
f.write(chunk)
f.flush()
f.close()
try:
self.session.run("iput", None, '-f', f.name, name)
except:
self.session.run("iput", None, '-f', f.name, name) # IRODS 4.0.2, sometimes iput fails on the first try. A second try seems to fix it.
os.unlink(f.name)
return name
def delete(self, name):
self.session.run("irm", None, "-rf", name)
def exists(self, name):
try:
stdout = self.session.run("ils", None, name)[0]
return stdout != ""
except SessionException:
return False
def listdir(self, path):
stdout = self.session.run("ils", None, path)[0].split("\n")
listing = ( [], [] )
directory = stdout[0][0:-2]
directory_prefix = " C- " + directory + "/"
for i in range(1, len(stdout)):
if stdout[i][:len(directory_prefix)] == directory_prefix:
listing[0].append(stdout[i][len(directory_prefix):])
else:
listing[1].append(stdout[i].strip)
return listing
def size(self, name):
stdout = self.session.run("ils", None, "-l", name)[0].split()
return int(stdout[3])
def url(self, name):
return reverse('django_irods.views.download', kwargs={'path': name})
class IrodsStorage(Storage):
def __init__(self, option=None):
if option == 'federated':
# resource should be saved in federated zone
self.set_fed_zone_session()
else:
self.session = GLOBAL_SESSION
self.environment = GLOBAL_ENVIRONMENT
icommands.ACTIVE_SESSION = self.session
def set_user_session(self, username=None, password=None, host=settings.IRODS_HOST,
port=settings.IRODS_PORT, def_res=None, zone=settings.IRODS_ZONE,
userid=0, sess_id=None):
homedir = "/" + zone + "/home/" + username
userEnv = IRodsEnv(
pk=userid,
host=host,
port=port,
def_res=def_res,
home_coll=homedir,
cwd=homedir,
username=username,
zone=zone,
auth=password,
irods_default_hash_scheme='MD5'
)
if sess_id is None:
self.session = Session(session_id=uuid4())
self.environment = self.session.create_environment(myEnv=userEnv)
else:
self.session = Session(session_id=sess_id)
if self.session.session_file_exists():
self.environment = userEnv
else:
self.environment = self.session.create_environment(myEnv=userEnv)
self.session.run('iinit', None, self.environment.auth)
icommands.ACTIVE_SESSION = self.session
# Set iRODS session to wwwHydroProxy for irods_storage input object for iRODS federated
# zone direct file operations
def set_fed_zone_session(self):
if settings.REMOTE_USE_IRODS:
self.set_user_session(username=settings.HS_WWW_IRODS_PROXY_USER,
password=settings.HS_WWW_IRODS_PROXY_USER_PWD,
host=settings.HS_WWW_IRODS_HOST,
port=settings.IRODS_PORT,
def_res=settings.HS_IRODS_LOCAL_ZONE_DEF_RES,
zone=settings.HS_WWW_IRODS_ZONE,
sess_id='federated_session')
def delete_user_session(self):
if self.session != GLOBAL_SESSION and self.session.session_file_exists():
self.session.delete_environment()
def download(self, name):
return self._open(name, mode='rb')
def getFile(self, src_name, dest_name):
self.session.run("iget", None, '-f', src_name, dest_name)
def runBagitRule(self, rule_name, input_path, input_resource):
"""
run iRODS bagit rule which generated bag-releated files without bundling
:param rule_name: the iRODS rule name to run
:param input_path: input parameter to the rule that indicates the collection path to
create bag for
:param input_resource: input parameter to the rule that indicates the default resource
to store generated bag files
:return: None
"""
# SessionException will be raised from run() in icommands.py
self.session.run("irule", None, '-F', rule_name, input_path, input_resource)
def zipup(self, in_name, out_name):
"""
run iRODS ibun command to generate zip file for the bag
:param in_name: input parameter to indicate the collection path to generate zip
:param out_name: the output zipped file name
:return: None
"""
self.session.run("imkdir", None, '-p', out_name.rsplit('/', 1)[0])
# SessionException will be raised from run() in icommands.py
self.session.run("ibun", None, '-cDzip', '-f', out_name, in_name)
def setAVU(self, name, attName, attVal, attUnit=None):
"""
set AVU on resource collection - this is used for on-demand bagging by indicating
whether the resource has been modified via AVU pairs
Parameters:
:param
name: the resource collection name to set AVU.
attName: the attribute name to set
attVal: the attribute value to set
attUnit: the attribute Unit to set, default is None, but can be set to
indicate additional info
"""
# SessionException will be raised from run() in icommands.py
if attUnit:
self.session.run("imeta", None, 'set', '-C', name, attName, attVal, attUnit)
else:
self.session.run("imeta", None, 'set', '-C', name, attName, attVal)
def getAVU(self, name, attName):
"""
set AVU on resource collection - this is used for on-demand bagging by indicating
whether the resource has been modified via AVU pairs
Parameters:
:param
name: the resource collection name to set AVU.
attName: the attribute name to set
attVal: the attribute value to set
attUnit: the attribute Unit to set, default is None, but can be set to
indicate additional info
"""
# SessionException will be raised from run() in icommands.py
stdout = self.session.run("imeta", None, 'ls', '-C', name, attName)[0].split("\n")
ret_att = stdout[1].strip()
if ret_att == 'None': # queried attribute does not exist
return None
else:
vals = stdout[2].split(":")
return vals[1].strip()
def copyFiles(self, src_name, dest_name, ires=None):
"""
Parameters:
:param
src_name: the iRODS data-object or collection name to be copied from.
dest_name: the iRODS data-object or collection name to be copied to
copyFiles() copied an irods data-object (file) or collection (directory)
to another data-object or collection
"""
if src_name and dest_name:
if '/' in dest_name:
splitstrs = dest_name.rsplit('/', 1)
if not self.exists(splitstrs[0]):
self.session.run("imkdir", None, '-p', splitstrs[0])
if ires:
self.session.run("icp", None, '-rf', '-R', ires, src_name, dest_name)
else:
self.session.run("icp", None, '-rf', src_name, dest_name)
return
def moveFile(self, src_name, dest_name):
"""
Parameters:
:param
src_name: the iRODS data-object or collection name to be moved from.
dest_name: the iRODS data-object or collection name to be moved to
moveFile() moves/renames an irods data-object (file) or collection
(directory) to another data-object or collection
"""
if src_name and dest_name:
if '/' in dest_name:
splitstrs = dest_name.rsplit('/', 1)
if not self.exists(splitstrs[0]):
self.session.run("imkdir", None, '-p', splitstrs[0])
self.session.run("imv", None, src_name, dest_name)
return
def saveFile(self, from_name, to_name, create_directory=False, data_type_str=''):
"""
Parameters:
:param
from_name: the temporary file name in local disk to be uploaded from.
to_name: the data object path in iRODS to be uploaded to
create_directory: create directory as needed when set to True. Default is False
Note if only directory needs to be created without saving a file, from_name should be empty
and to_name should have "/" as the last character
"""
if create_directory:
splitstrs = to_name.rsplit('/', 1)
self.session.run("imkdir", None, '-p', splitstrs[0])
if len(splitstrs) <= 1:
return
if from_name:
try:
if data_type_str:
self.session.run("iput", None, '-D', data_type_str, '-f', from_name, to_name)
else:
self.session.run("iput", None, '-f', from_name, to_name)
except:
if data_type_str:
self.session.run("iput", None, '-D', data_type_str, '-f', from_name, to_name)
else:
# IRODS 4.0.2, sometimes iput fails on the first try.
# A second try seems to fix it.
self.session.run("iput", None, '-f', from_name, to_name)
return
def _open(self, name, mode='rb'):
tmp = NamedTemporaryFile()
self.session.run("iget", None, '-f', name, tmp.name)
return tmp
def _save(self, name, content):
self.session.run("imkdir", None, '-p', name.rsplit('/', 1)[0])
with NamedTemporaryFile(delete=False) as f:
for chunk in content.chunks():
f.write(chunk)
f.flush()
f.close()
try:
self.session.run("iput", None, '-f', f.name, name)
except:
# IRODS 4.0.2, sometimes iput fails on the first try. A second try seems to fix it.
self.session.run("iput", None, '-f', f.name, name)
os.unlink(f.name)
return name
def delete(self, name):
self.session.run("irm", None, "-rf", name)
def exists(self, name):
try:
stdout = self.session.run("ils", None, name)[0]
return stdout != ""
except SessionException:
return False
def listdir(self, path):
stdout = self.session.run("ils", None, path)[0].split("\n")
listing = ([], [])
directory = stdout[0][0:-1]
directory_prefix = " C- " + directory + "/"
for i in range(1, len(stdout)):
if stdout[i][:len(directory_prefix)] == directory_prefix:
dirname = stdout[i][len(directory_prefix):].strip()
if dirname:
listing[0].append(dirname)
else:
filename = stdout[i].strip()
if filename:
listing[1].append(filename)
return listing
def size(self, name):
stdout = self.session.run("ils", None, "-l", name)[0].split()
return int(stdout[3])
def url(self, name):
return reverse('django_irods.views.download', kwargs={'path': name})
def get_available_name(self, name):
"""
Reject duplicate file names rather than renaming them.
"""
if self.exists(name):
raise ValidationError(str.format("File {} already exists.", name))
return name