def create_session(ticket: str = None):
# if we have a ticket, create an anonymous session and apply it to it
if ticket is not None and ticket != '':
session = iRODSSession(host='data.cyverse.org',
port=1247,
user='******',
password='',
zone='iplant')
Ticket(session, ticket).supply()
return session
# otherwise check for an iRODS environment file (if we don't have one, abort)
try:
env_file = os.environ.get('IRODS_ENVIRONMENT_FILE')
except KeyError:
default_env_file = '~/.irods/irods_environment.json'
if isfile(default_env_file): env_file = expanduser(default_env_file)
else:
raise ValueError(
f"No iRODS authentication method provided (need ticket or environment file)"
)
ssl_context = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH,
cafile=None,
capath=None,
cadata=None)
ssl_settings = {'ssl_context': ssl_context}
return iRODSSession(irods_env_file=env_file, **ssl_settings)
def openSession(self, user_name=None, passwd=None):
"""
This function creates a new session in iRODS
Note: by default the iRODS session will be created using the credentials stored
at the .irodsA and the irods_environment.json files unless the user provides
some custom credentials.
Instructions for creating these files are available at the tool repository.
"""
if self.session == None:
#READ CREDENTIALS FROM THE CONFIG FILES STORED IN ~/.irods AFTER USING iCommands
#CREDENTIALS MUST BE STORED IN ~/.irods
#TODO: ADD THE iRODS CREDENTIALS IN THE GALAXY CONFIG FILE
pwFile = "/home/" + getpass.getuser() + "/.irods/.irodsA"
envFile = "/home/" + getpass.getuser(
) + "/.irods/irods_environment.json"
with open(envFile) as f:
data = json.load(f)
self.host = str(data["irods_host"])
self.port = str(data["irods_port"])
self.user = str(data["irods_user_name"])
self.zone = str(data["irods_zone_name"])
if (user_name == None): #Root connection
#Use default user and password but custom client username and zone
with open(pwFile) as f:
first_line = f.readline().strip()
self.passwd = decode(first_line)
#TODO: use custom directory client_zone (form)
self.session = iRODSSession(
self.host,
port=self.port,
user=self.user,
password=self.passwd,
zone=self.zone) #, client_zone=client_zone)
elif (passwd == None):
#Use default user and password but custom client username and zone
with open(pwFile) as f:
first_line = f.readline().strip()
self.passwd = decode(first_line)
#TODO: use custom directory client_zone (form)
self.session = iRODSSession(
self.host,
port=self.port,
user=self.user,
password=self.passwd,
zone=self.zone,
client_user=user_name) #, client_zone=client_zone)
else:
#Use custom user and password
#TODO: use custom directory client_zone (form)
self.session = iRODSSession(host=self.host,
port=self.port,
user=user_name,
password=passwd,
zone=self.zone)
return self.session
def test_modify_password__328(self):
ses = self.sess
if ses.users.get(ses.username).type != 'rodsadmin':
self.skipTest('Only a rodsadmin may run this test.')
OLDPASS = '******'
NEWPASS = '******'
try:
ses.users.create('alice', 'rodsuser')
ses.users.modify('alice', 'password', OLDPASS)
with iRODSSession(user='******',
password=OLDPASS,
host=ses.host,
port=ses.port,
zone=ses.zone) as alice:
me = alice.users.get(alice.username)
me.modify_password(OLDPASS, NEWPASS)
with iRODSSession(user='******',
password=NEWPASS,
host=ses.host,
port=ses.port,
zone=ses.zone) as alice:
home = helpers.home_collection(alice)
alice.collections.get(
home) # Non-trivial operation to test success!
finally:
try:
ses.users.get('alice').remove()
except UserDoesNotExist:
pass
def test_modify_password_with_incorrect_old_value__328(self):
ses = self.sess
if ses.users.get(ses.username).type != 'rodsadmin':
self.skipTest('Only a rodsadmin may run this test.')
OLDPASS = '******'
NEWPASS = '******'
ENV_DIR = tempfile.mkdtemp()
try:
ses.users.create('alice', 'rodsuser')
ses.users.modify('alice', 'password', OLDPASS)
d = dict(password=OLDPASS,
user='******',
host=ses.host,
port=ses.port,
zone=ses.zone)
(alice_env, alice_auth) = helpers.make_environment_and_auth_files(
ENV_DIR, **d)
session_factories = [
(lambda: iRODSSession(**d)),
(lambda: helpers.make_session(irods_env_file=alice_env,
irods_authentication_file=
alice_auth)),
]
for factory in session_factories:
with factory() as alice_ses:
alice = alice_ses.users.get(alice_ses.username)
with self.assertRaises(ex.CAT_PASSWORD_ENCODING_ERROR):
alice.modify_password(OLDPASS + ".", NEWPASS)
with iRODSSession(**d) as alice_ses:
self.do_something(alice_ses)
finally:
shutil.rmtree(ENV_DIR)
ses.users.remove('alice')
def test_temp_password(self):
# Make a new user
self.admin.users.create(self.new_user, 'rodsuser')
self.admin.users.modify(self.new_user, 'password', self.password)
# Login as the new test user, to retrieve a temporary password
with iRODSSession(host=self.admin.host,
port=self.admin.port,
user=self.new_user,
password=self.password,
zone=self.admin.zone) as session:
# Obtain the temporary password
conn = session.pool.get_connection()
temp_password = conn.temp_password()
# Open a new session with the temporary password
with iRODSSession(host=self.admin.host,
port=self.admin.port,
user=self.new_user,
password=temp_password,
zone=self.admin.zone) as session:
# do something that connects to the server
session.users.get(self.admin.username)
# delete new user
self.admin.users.remove(self.new_user)
# user should be gone
with self.assertRaises(UserDoesNotExist):
self.admin.users.get(self.new_user)
def do_put_to_child(self):
with iRODSSession(irods_env_file=env_file) as session:
session.resources.remove_child(REGISTER_RESC2, REGISTER_RESC2A)
self.do_put("examples.put_with_resc_name_regiResc2a",
resc_name=REGISTER_RESC2A,
resc_root=REGISTER_RESC_PATH2A)
with iRODSSession(irods_env_file=env_file) as session:
session.resources.add_child(REGISTER_RESC2, REGISTER_RESC2A)
def __init__(self, envFile, collPath, host='ibridges', user='******', zone='myZone'):
if envFile == '':
#workaround for testing and when icommands not available
print 'irods ', host, 'user ', user, 'zone', zone
pw = getpass.getpass().encode('base64')
self.session = iRODSSession(host=host, port=1247, user=user,
password=pw.decode('base64'), zone=zone)
else:
self.session = iRODSSession(irods_env_file=envFile)
self.coll = self.session.collections.get(collPath)
self.mdUpdate('SERIESINFORMATION', 'iRODS Collection '+ self.coll.path)
self.md = self.mdGet()
def pythonIrodsSession(password, envfile=None, host='', user='', zone=''):
if envfile == None:
session = iRODSSession(host=host,
port=1247,
user=user,
password=password,
zone=zone)
else:
with open(envfile) as f:
ienv = json.load(f)
session = iRODSSession(**ienv, password=password)
return session
def test_modify_password_with_changing_auth_file__328(self):
ses = self.sess
if ses.users.get(ses.username).type != 'rodsadmin':
self.skipTest('Only a rodsadmin may run this test.')
OLDPASS = '******'
def generator(p=OLDPASS):
n = 1
old_pw = p
while True:
pw = p + str(n)
yield old_pw, pw
n += 1
old_pw = pw
password_generator = generator()
ENV_DIR = tempfile.mkdtemp()
d = dict(password=OLDPASS,
user='******',
host=ses.host,
port=ses.port,
zone=ses.zone)
(alice_env,
alice_auth) = helpers.make_environment_and_auth_files(ENV_DIR, **d)
try:
ses.users.create('alice', 'rodsuser')
ses.users.modify('alice', 'password', OLDPASS)
for modify_option, sess_factory in [
(alice_auth, lambda: iRODSSession(**d)),
(True, lambda: helpers.make_session(irods_env_file=alice_env,
irods_authentication_file=
alice_auth))
]:
OLDPASS, NEWPASS = next(password_generator)
with sess_factory() as alice_ses:
alice = alice_ses.users.get(alice_ses.username)
alice.modify_password(
OLDPASS,
NEWPASS,
modify_irods_authentication_file=modify_option)
d['password'] = NEWPASS
with iRODSSession(**d) as session:
self.do_something(
session
) # can we still do stuff with the final value of the password?
finally:
shutil.rmtree(ENV_DIR)
ses.users.remove('alice')
def connect(args):
"""Connect to the iRODS server"""
if args.user is not None:
if args.passwd is None:
print("ERROR: --passwd required with --user")
sys.exit()
session = iRODSSession(host=args.host, port=args.port,
user=args.user, password=args.passwd,
zone=args.zone)
else:
try:
session = helpers.make_session()
except FileNotFoundError:
sys.exit("ERROR: No irods_environment.json file found. Type 'pyicmd help' for details")
# Test the connection
try:
session.server_version
except CAT_INVALID_AUTHENTICATION:
sys.exit("iRODS server authentication failed.")
except CAT_INVALID_USER:
sys.exit("Invalid iRODS user.")
except CAT_INVALID_CLIENT_USER:
sys.exit("Invalid client user. (Did you use the right zone?)")
except NetworkException as exception:
sys.exit(str(exception))
return session
def create_object(self, dest_path, collection=True):
"""
Creates a new object into the dest_path. Default is a collection
:type dest_path: str
:param dest_path: irods path
:type collection: bool
:param collection: if True create a collection else a data_object
:return: an irods.data_object.iRODSDataObject,
irods.collection.iRODSCollection
"""
session = iRODSSession(host=self.host,
port=self.port,
user=self.user,
password=self.password,
zone=self.zone)
session.connection_timeout = 300
if not self.exists(dest_path)[0]:
if collection:
obj = session.collections.create(dest_path)
else:
obj = session.data_objects.create(dest_path)
else:
raise RuntimeError("Collection already present into the catalog")
session.cleanup()
return obj
def exists(self, path, delivery=False):
session = iRODSSession(host=self.host,
port=self.port,
user=self.user,
password=self.password,
zone=self.zone)
session.connection_timeout = 300
try:
obj = session.data_objects.get(path)
self.logger.info("Getting: {}".format(str(obj.name)))
exists = True
except DataObjectDoesNotExist as e:
self.logger.error(str(e))
exists = False
obj = None
if not obj:
try:
obj = session.collections.get(path)
self.logger.info("Getting: {}".format(str(obj.name)))
exists = True
except CollectionDoesNotExist as e:
self.logger.error(str(e))
exists = False
obj = None
session.cleanup()
return (exists, obj)
def test_set_user_password(self):
# make a new user
username = self.new_user_name
zone = self.sess.zone
self.sess.users.create(self.new_user_name, self.new_user_type)
# make a really horrible password
new_password = '''abc123!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~Z'''
self.sess.users.modify(username, 'password', new_password)
# open a session as the new user
with iRODSSession(host=self.sess.host,
port=self.sess.port,
user=username,
password=new_password,
zone=self.sess.zone) as session:
# do something that connects to the server
session.users.get(username)
# delete new user
self.sess.users.remove(self.new_user_name)
# user should be gone
with self.assertRaises(UserDoesNotExist):
self.sess.users.get(self.new_user_name)
def set_connection(self, json_body):
self.session = iRODSSession(
host=json_body['host'],
port=json_body['port'],
user=json_body['user'],
password=json_body['password'],
zone=json_body['zone'])
def connect(self):
""" Create an iRods session using the instantiation options, or
a file specified by the environment variable IRODS_ENVIRONMENT_FILE, or
the default irods_environment.json file.
"""
logging.info("(IrodsHelper.connect): options={}".format(self._options))
try:
env_file = self._options["irods_env_file"]
except KeyError:
try:
env_file = os.environ["IRODS_ENVIRONMENT_FILE"]
except KeyError:
env_file = os.path.expanduser(
"~/.irods/irods_environment.json")
logging.info("IrodsHelper.connect: env_file={}".format(env_file))
# session is established upon successful connection to iRods
self._session = iRODSSession(irods_env_file=env_file)
logging.info("IrodsHelper.connect: SESSION={}".format(self._session))
# users root directory is set to their iRods home directory
self.set_root()
logging.info("IrodsHelper.connect: ROOT={}".format(self._root))
logging.info("IrodsHelper.connect: CWDPATH={}".format(self._cwdpath))
def scan_source(irods_env, proxy_path):
"""
Scan the iRODS proxy path for all public Hydroshare resources containing NetCDF and their timestamps.
scan_source(irods_env, proxy_path) -> [(resource_id, timestamp), ...]
Where:
irods_env: <str> Absolute path to the iRODS environment file
proxy_path: <str> Absolute iRODS proxy path to Hydroshare resources
Returns: <list> of two-<tuple>s where:
a) first element is a <str> resource id, and
b) second element is a <datetime.datetime> modification time.
"""
with iRODSSession(irods_env_file=irods_env) as session:
subcollections = session.collections.get(proxy_path).subcollections
subcollections = [
subcollection for subcollection in subcollections
if subcollection.name not in EXCLUDED
]
logger.info(
f"Number of included subcollections: {len(subcollections)}")
public = [
subcollection for subcollection in subcollections
if "isPublic" in subcollection.metadata.keys() and subcollection.
metadata[IS_PUBLIC_KEY].value.lower() == IS_PUBLIC_VALUE
]
logger.info(f"Number of public included subcollections: {len(public)}")
public_netcdf = []
for subcollection in public:
public_objects = [
objs for col, subcol, objs in list(subcollection.walk())
]
# flatten the list of lists of data objects
data_objects = []
for objs in public_objects:
data_objects.extend(objs)
netcdf_objects = [
obj for obj in data_objects
if pathlib.Path(obj.name).suffix.lower() in NETCDF_EXTENSIONS
]
if netcdf_objects:
public_netcdf.append(subcollection.name)
logger.info(
f"Subcollection name: {subcollection.name}; Number of NetCDF data objects in subcollection: {len(netcdf_objects)}"
)
logger.info(
f"Number of public subcollections containing NetCDF: {len(public_netcdf)}"
)
source_netcdf = [
(resource_id,
get_latest_resource_timestamp(irods_env,
os.path.join(proxy_path, resource_id)))
for resource_id in public_netcdf
]
return source_netcdf
def get_latest_resource_timestamp(irods_env, collection_path):
"""
Return the latest modifcation time among the collection's data objects.
get_latest_resource_timestamp(collection_path) -> <datetime.datetime>
Where:
irods_env: <str> Absolute path to the iRODS environment file
collection_path: <str> Absolute iRODS path to the collection
Returns: <datetime.datetime> The latest modification time
This function should become deprecated with iRODS 4.2.9 which updates collection modification times
whenever a contained data object is modified.
"""
with iRODSSession(irods_env_file=irods_env) as session:
collection = session.collections.get(collection_path)
tree = [leaf for leaf in collection.walk()]
data_objects = []
for leaf in tree:
data_objects.extend(leaf[2])
timestamps = [data_object.modify_time for data_object in data_objects]
timestamp = max(timestamps)
return timestamp
def __init__(self, conf, confSection='iRods', parentLogger=None):
"""
Initialize the object.
Args:
parentLogger: the parent logger
"""
try:
#initialize logger
if (parentLogger):
self.logger = parentLogger.getChild(__name__)
else:
self.logger = logging.getLogger(__name__)
self.session=iRODSSession(host= conf.get(confSection,'host'),
port= conf.get(confSection,'port'),
user= conf.get(confSection,'rods_user'),
password=PasswordFile(conf.get(confSection,'rods_password_file')).password,
zone= conf.get(confSection,'rods_zone'))
self.connection=self.session.pool.get_connection()
self.userType=conf.get(confSection,'user_type')
self.userZone=conf.get(confSection,'user_zone')
except ConfigParser.Error, e:
self.logger.debug(traceback.format_exc())
self.logger.error(e.message)
raise IrodsApiInitException('Configuration error.')
def setUp(self):
self.sess = iRODSSession(
host=config.IRODS_SERVER_HOST,
port=config.IRODS_SERVER_PORT, # 4444: why?
user=config.IRODS_USER_USERNAME,
password=config.IRODS_USER_PASSWORD,
zone=config.IRODS_SERVER_ZONE)
def connect_to_grid(self, host='localhost', port=1247, user='******', password='******', zone='tempZone',
alias='default_connection'):
""" Create connection to an iRODS grid
using the parameters passed in.
'alias' - Robotframework alias to identify the connection
'host' - the fqdn of the iRODS server
'port' - the port to connect on
'user' - a valid iRODS username
'password' - the iRODS password for the username given
'zone' - a valid iRODS zone, tempZone is used by default
Example usage:
| # To connect to foo.bar.org's iRODS service on port 1247 |
| Connect To Grid | foo.bar.org | ${1247} | jdoe | jdoePassword | tempZone | UAT
"""
host = str(host)
port = int(port)
user = str(user)
password = str(password)
zone = str(zone)
alias = str(alias)
logger.info('Connect To Grid using : alias=%s, host=%s, port=%s, user=%s, password=%s, '
'zone=%s ' % (alias, host, port, user, password, zone))
session = iRODSSession(host=host, port=port, user=user, password=password, zone=zone)
self._cache.register(session, alias=alias)
def test_set_user_password(self):
# make a new user
username = self.new_user_name
zone = self.sess.zone
self.sess.users.create(self.new_user_name, self.new_user_type)
# make a really horrible password
new_password = '''abc123!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~Z'''
self.sess.users.modify(username, 'password', new_password)
# open a session as the new user
with iRODSSession(host=self.sess.host,
port=self.sess.port,
user=username,
password=new_password,
zone=self.sess.zone) as session:
# do something that connects to the server
session.users.get(username)
# delete new user
self.sess.users.remove(self.new_user_name)
# user should be gone
with self.assertRaises(UserDoesNotExist):
self.sess.users.get(self.new_user_name)
def init_with_environ_conf(self, client_user, admin_mode):
if admin_mode:
self.session = iRODSSession(host=os.environ["IRODS_HOST"],
port=1247,
user=os.environ["IRODS_USER"],
password=os.environ["IRODS_PASS"],
zone="nlmumc",
**self.ssl_settings)
else:
self.session = iRODSSession(host=os.environ["IRODS_HOST"],
port=1247,
user=os.environ["IRODS_USER"],
password=os.environ["IRODS_PASS"],
zone="nlmumc",
client_user=client_user,
**self.ssl_settings)
def migrate_image_data_irods(dst_glance_client,
irods_conn,
irods_src_coll,
irods_dst_coll,
img_uuid,
dst_glance_client_version=None):
"""
Migrates image data using iRODS and then sets image location using Glance API.
Args:
dst_glance_client: glance client object for destination provider
irods_conn: dict as returned by _parse_irods_conn()
irods_src_coll: Path to collection for iRODS images on source provider
irods_dst_coll: Path to collection for iRODS images on destination provider
img_uuid: UUID of image to be migrated
dst_glance_client_version: Version of Glance client for destination provider
Returns: True if successful, else raises exception
"""
sess = iRODSSession(host=irods_conn.get('host'),
port=irods_conn.get('port'),
zone=irods_conn.get('zone'),
user=irods_conn.get('username'),
password=irods_conn.get('password'))
src_data_obj_path = os.path.join(irods_src_coll, img_uuid)
dst_data_obj_path = os.path.join(irods_dst_coll, img_uuid)
sess.data_objects.copy(src_data_obj_path, dst_data_obj_path)
logging.info("Copied image data to destination collection in iRODS")
dst_img_location = "irods://{0}:{1}@{2}:{3}{4}".format(
irods_conn.get('username'), irods_conn.get('password'),
irods_conn.get('host'), irods_conn.get('port'), dst_data_obj_path)
dst_glance_client.images.add_location(img_uuid, dst_img_location, dict())
logging.info("Set image location in Glance")
return True
def login(request):
if request.method == 'POST':
port = int(request.POST['port'])
user = str(request.POST['username'])
password = str(request.POST['password'])
zone = str(request.POST['zone'])
host = str(request.POST['host'])
datastore = "/%s/home/%s" % (zone, user)
response_data = {}
irods_sess = iRODSSession(user=user, password=password, zone=zone, host=host, port=port)
try:
irods_sess.collections.get(datastore)
except CollectionDoesNotExist:
response_data['irods_loggedin'] = False
response_data['login_message'] = 'iRODS login failed'
response_data['error'] = "iRODS collection does not exist"
irods_sess.cleanup()
return JsonResponse(response_data, status=status.HTTP_400_BAD_REQUEST)
else:
response_data['user'] = user
response_data['password'] = password
response_data['port'] = port
response_data['host'] = host
response_data['zone'] = zone
response_data['datastore'] = datastore
response_data['irods_loggedin'] = True
irods_sess.cleanup()
return JsonResponse(response_data, status=status.HTTP_200_OK)
else:
return JsonResponse({"error": "Not POST request"}, status=status.HTTP_400_BAD_REQUEST)
def from_options(cls,
host,
port,
user,
zone,
scrambled_password,
default_resc,
local_checksum,
default_hash_scheme,
ssl_settings=None):
kwargs = {}
try:
password = iRODSCatalogBase.decode(scrambled_password)
kwargs.update(
dict(host=host,
port=port,
user=user,
password=password,
zone=zone,
default_hash_scheme=default_hash_scheme))
except irods.exception.CAT_INVALID_AUTHENTICATION as e:
raise exceptions.ConnectionError(e)
if ssl_settings is not None:
kwargs.update(ssl_settings)
session = iRODSSession(**kwargs)
return cls(session, default_resc, local_checksum)
def connect_to_grid_on_behalf(self, host='localhost', port=1247, user='******', password='******', zone='tempZone',
client_user='******', client_zone='another_zone', alias='default_connection'):
""" Create connection to an iRODS grid
using the parameters passed in to act on behalf of a user.
'alias' - Robotframework alias to identify the connection
'host' - the fqdn of the iRODS server
'port' - the port to connect on
'user' - a valid iRODS username
'password' - the iRODS password for the username given
'zone' - a valid iRODS zone, tempZone is used by default
'client_user' - username of user who is being acted on behalf of
'client_zone' - zone of user who is being acted on behalf of
Example usage:
| # To connect to foo.bar.org's iRODS service on port 1247 |
| Connect To Grid On Behalf | foo.bar.org | ${1247} | adminUser | AdminUserPassword | tempZone | clientUser | clientZone | connectionAlias
"""
host = str(host)
port = int(port)
user = str(user)
password = str(password)
zone = str(zone)
client_user = str(client_user)
client_zone = str(client_zone)
alias = str(alias)
logger.info('Connect To Grid On Behalf using : alias=%s, host=%s, port=%s, user=%s, password=%s, '
'zone=%s, client_user=%s, client_zone=%s ' % (alias, host, port, user, password, zone,
client_user, client_zone))
session = iRODSSession(host=host, port=port, user=user, password=password, zone=zone, client_user=client_user,
client_zone=client_zone)
self._cache.register(session, alias=alias)
def __init__(self, conf, confSection='iRods', parentLogger=None):
"""
Initialize the object.
Args:
parentLogger: the parent logger
"""
try:
#initialize logger
if (parentLogger):
self.logger = parentLogger.getChild(__name__)
else:
self.logger = logging.getLogger(__name__)
self.session = iRODSSession(
host=conf.get(confSection, 'host'),
port=conf.get(confSection, 'port'),
user=conf.get(confSection, 'rods_user'),
password=PasswordFile(
conf.get(confSection, 'rods_password_file')).password,
zone=conf.get(confSection, 'rods_zone'))
self.connection = self.session.pool.get_connection()
self.userType = conf.get(confSection, 'user_type')
self.userZone = conf.get(confSection, 'user_zone')
except ConfigParser.Error, e:
self.logger.debug(traceback.format_exc())
self.logger.error(e.message)
raise IrodsApiInitException('Configuration error.')
def test_set_user_password(self):
# make a new user
username = self.new_user_name
zone = self.sess.zone
self.sess.users.create(self.new_user_name, self.new_user_type)
# make a 12 character pseudo-random password
new_password = ''.join(random.choice(string.ascii_letters + string.digits + string.punctuation) for _ in range(12))
self.sess.users.modify(username, 'password', new_password)
# open a session as the new user
with iRODSSession(host=self.sess.host,
port=self.sess.port,
user=username,
password=new_password,
zone=self.sess.zone) as session:
# do something that connects to the server
session.users.get(username)
# delete new user
self.sess.users.remove(self.new_user_name)
# user should be gone
with self.assertRaises(UserDoesNotExist):
self.sess.users.get(self.new_user_name)
def getFilesonDE(base_path="/iplant/home/bhickson/2015/Data"):
try:
pw = getpass.getpass()
session = iRODSSession(host='data.cyverse.org',
zone="iplant",
port=1247,
user='******',
password=pw)
data_col = session.collections.get(base_path)
except:
print("Unable to make connection to discover env. Continuing...")
return None, {}
global ifiles
ifiles = {}
def getFilesandDirs(dir):
#print(dir.name)
files_list = dir.data_objects
dirs_list = dir.subcollections
for file in files_list:
file_name = file.name
ifiles[file.name] = file.path
for sub_dir in dirs_list:
#print(sub_dir.name)
getFilesandDirs(sub_dir)
getFilesandDirs(data_col)
print("\t{} files found on Cyverse Discovery Environment in directory {}".
format(len(ifiles), base_path))
return session, ifiles
def connect_to_iRODS():
try:
env_file = os.environ['IRODS_ENVIRONMENT_FILE']
except KeyError:
env_file = os.path.expanduser('~/.irods/irods_environment.json')
s = iRODSSession(irods_env_file=env_file)
return s
def test_set_user_password(self):
# make a new user
username = self.new_user_name
zone = self.sess.zone
self.sess.users.create(self.new_user_name, self.new_user_type)
# make a 12 character pseudo-random password
new_password = ''.join(random.choice(string.ascii_letters + string.digits + string.punctuation) for _ in range(12))
self.sess.users.modify(username, 'password', new_password)
# open a session as the new user
with iRODSSession(host=self.sess.host,
port=self.sess.port,
user=username,
password=new_password,
zone=self.sess.zone) as session:
# do something that connects to the server
session.users.get(username)
# delete new user
self.sess.users.remove(self.new_user_name)
# user should be gone
with self.assertRaises(UserDoesNotExist):
self.sess.users.get(self.new_user_name)
def __enter__(self):
auth_file = self.irods_auth_file
env_file = self.irods_config_file
self.session = iRODSSession(irods_env_file=env_file,
irods_authentication_file=auth_file)
self.session.connection_timeout = self.connection_timeout
return self
def getFilesonDE(base_path):
pw_file = "./pw_file.txt"
try:
with open(pw_file) as pf:
pw = pf.readlines(0)[0].strip()
session = iRODSSession(host='data.cyverse.org',
zone="iplant",
port=1247,
user='******',
password=pw)
data_col = session.collections.get(base_path)
except:
print("Unable to make connection to discovery env. Continuing...")
return None, {}
ifiles = {}
def getFilesandDirs(dir):
# print(dir.name)
files_list = dir.data_objects
dirs_list = dir.subcollections
for file in files_list:
file_name = file.name
ifiles[file.name] = file.path
for sub_dir in dirs_list:
# print(sub_dir.name)
getFilesandDirs(sub_dir)
getFilesandDirs(data_col)
return session, ifiles
def connect(self):
logger.info("--\t Connect to iRODS")
self.session = iRODSSession(host=self.host,
port=self.port,
user=self.user,
password=self.password,
zone=self.zone)
def init_with_variable_config(self, client_user, config, admin_mode):
if admin_mode:
self.session = iRODSSession(host=config["IRODS_HOST"],
port=1247,
user=config["IRODS_USER"],
password=config["IRODS_PASS"],
zone="nlmumc",
**self.ssl_settings)
else:
self.session = iRODSSession(host=config["IRODS_HOST"],
port=1247,
user=config["IRODS_USER"],
password=config["IRODS_PASS"],
zone="nlmumc",
client_user=client_user,
**self.ssl_settings)
def setUp(self):
self.sess = iRODSSession(host=config.IRODS_SERVER_HOST,
port=config.IRODS_SERVER_PORT,
user=config.IRODS_USER_USERNAME,
password=config.IRODS_USER_PASSWORD,
zone=config.IRODS_SERVER_ZONE)
self.test_coll = self.sess.collections.create(self.test_coll_path)
def setUp(self):
from irods.session import iRODSSession
import config
self.sess = iRODSSession(host=config.IRODS_SERVER_HOST,
port=config.IRODS_SERVER_PORT, # 4444: why?
user=config.IRODS_USER_USERNAME,
password=config.IRODS_USER_PASSWORD,
zone=config.IRODS_SERVER_ZONE)
def openSession(self, user_name=None, passwd=None): """ This function creates a new session in iRODS Note: by default the iRODS session will be created using the credentials stored at the .irodsA and the irods_environment.json files unless the user provides some custom credentials. Instructions for creating these files are available at the tool repository. """ if self.session == None: #READ CREDENTIALS FROM THE CONFIG FILES STORED IN ~/.irods AFTER USING iCommands #CREDENTIALS MUST BE STORED IN ~/.irods #TODO: ADD THE iRODS CREDENTIALS IN THE GALAXY CONFIG FILE pwFile = "/home/" + getpass.getuser() + "/.irods/.irodsA" envFile = "/home/" + getpass.getuser() + "/.irods/irods_environment.json" with open(envFile) as f: data = json.load(f) self.host = str(data["irods_host"]) self.port = str(data["irods_port"]) self.user = str(data["irods_user_name"]) self.zone = str(data["irods_zone_name"]) if(user_name == None): #Root connection #Use default user and password but custom client username and zone with open(pwFile) as f: first_line = f.readline().strip() self.passwd = decode(first_line) #TODO: use custom directory client_zone (form) self.session = iRODSSession(self.host, port=self.port, user=self.user, password=self.passwd, zone=self.zone)#, client_zone=client_zone) elif(passwd == None): #Use default user and password but custom client username and zone with open(pwFile) as f: first_line = f.readline().strip() self.passwd = decode(first_line) #TODO: use custom directory client_zone (form) self.session = iRODSSession(self.host, port=self.port, user=self.user, password=self.passwd, zone=self.zone, client_user=user_name)#, client_zone=client_zone) else: #Use custom user and password #TODO: use custom directory client_zone (form) self.session = iRODSSession(host=self.host, port=self.port, user=user_name, password=passwd, zone=self.zone) return self.session
def setUp(self):
self.sess = iRODSSession(host=config.IRODS_SERVER_HOST,
port=config.IRODS_SERVER_PORT,
user=config.IRODS_USER_USERNAME,
password=config.IRODS_USER_PASSWORD,
zone=config.IRODS_SERVER_ZONE)
# Create dummy test collection
self.coll = helpers.make_dummy_collection(
self.sess, self.coll_path, self.obj_count)
def setUp(self):
self.sess = iRODSSession(host=config.IRODS_SERVER_HOST,
port=config.IRODS_SERVER_PORT,
user=config.IRODS_USER_USERNAME,
password=config.IRODS_USER_PASSWORD,
zone=config.IRODS_SERVER_ZONE)
# Create dummy test collection
self.coll_path = '/{0}/home/{1}/test_dir'.format(config.IRODS_SERVER_ZONE, config.IRODS_USER_USERNAME)
self.coll = helpers.make_collection(self.sess, self.coll_path)
def setUp(self):
self.sess = iRODSSession(host=config.IRODS_SERVER_HOST,
port=config.IRODS_SERVER_PORT,
user=config.IRODS_USER_USERNAME,
password=config.IRODS_USER_PASSWORD,
zone=config.IRODS_SERVER_ZONE)
# Create test collection and (empty) test object
self.coll = self.sess.collections.create(self.coll_path)
self.obj = self.sess.data_objects.create(self.obj_path)
def test_session_with_client_user(self):
# stub
with iRODSSession(host=config.IRODS_SERVER_HOST,
port=config.IRODS_SERVER_PORT,
user=config.IRODS_USER_USERNAME,
password=config.IRODS_USER_PASSWORD,
zone=config.IRODS_SERVER_ZONE,
client_user=config.IRODS_USER_USERNAME,
client_zone=config.IRODS_SERVER_ZONE) as sess:
self.assertTrue(sess)
def connect(self, host=None, port=None, base_url=None, client_user=None,
client_zone=None):
if not host:
host = self.host
if not port:
port = self.port
session = iRODSSession(host=host, port=port,
user=self.user_id, password=self.key,
zone=self.zone, client_user=client_user,
client_zone=client_zone)
self.session = session
return session
def setUp(self):
self.sess = iRODSSession(host=config.IRODS_SERVER_HOST,
port=config.IRODS_SERVER_PORT,
user=config.IRODS_USER_USERNAME,
password=config.IRODS_USER_PASSWORD,
zone=config.IRODS_SERVER_ZONE)
# make list of unicode filenames, from file
self.names = parse_xml_file(UNICODE_TEST_FILE)
# Create dummy test collection
self.coll = helpers.make_collection(
self.sess, self.coll_path, self.names)
def test_session_from_env_file(self):
'''
Open a session using a client irods environment file for credentials
'''
env_file = os.path.expanduser('~/.irods/irods_environment.json')
if not os.access(env_file, os.R_OK):
self.skipTest('No readable irods environment file')
with iRODSSession(irods_env_file=env_file) as session:
# do something with our session
default_resource = session.resources.get('demoResc')
self.assertEqual(default_resource.type, 'unixfilesystem')
def __init__(self):
super(MyRods, self).__init__()
# config
self._default_zone = os.environ['IRODS_ZONE']
iconnection = {
'host': os.environ['RODSERVER_ENV_IRODS_HOST'],
'user': os.environ['IRODS_USER'],
'password': os.environ['RODSERVER_ENV_IRODS_PASS'],
'port': os.environ['RODSERVER_PORT_1247_TCP_PORT'],
'zone': self._default_zone
}
self._session = iRODSSession(**iconnection)
logger.info("Connected to irods")
def setUp(self):
self.sess = iRODSSession(host=config.IRODS_SERVER_HOST,
port=config.IRODS_SERVER_PORT, # 4444: why?
user=config.IRODS_USER_USERNAME,
password=config.IRODS_USER_PASSWORD,
zone=config.IRODS_SERVER_ZONE)
# Create test collection
self.test_coll = self.sess.collections.create(self.test_coll_path)
# Create test object and write to it
self.test_obj = self.sess.data_objects.create(self.test_obj_path)
with self.test_obj.open('r+') as f:
f.write(self.content_str)
def create_session(host, username, password, zone, port=1247):
if type(host) is unicode:
host = str(host)
if type(username) is unicode:
username = str(username)
if type(password) is unicode:
password = str(password)
if type(zone) is unicode:
zone = str(zone)
session = iRODSSession(host=host, port = port, user = username , password = password, zone = zone)
return session
def make_session_from_config(**kwargs):
conf_map = {'host': 'IRODS_SERVER_HOST',
'port': 'IRODS_SERVER_PORT',
'zone': 'IRODS_SERVER_ZONE',
'user': '******',
'authentication_scheme': 'IRODS_AUTHENTICATION_SCHEME',
'password': '******',
'server_dn': 'IRODS_SERVER_DN'}
for key in conf_map.keys():
try:
kwargs[key] = vars(config)[conf_map[key]]
except KeyError:
pass
return iRODSSession(**kwargs)
def connect_and_confirm(self):
"""
Confirm connection to irods with the given credentials
"""
try:
msg = (_("connecting to irods://%(host)s:%(port)s%(path)s " +
"in zone %(zone)s") %
({'host': self.host, 'port': self.port,
'path': self.datastore, 'zone': self.zone}))
LOG.debug(msg)
sess = iRODSSession(user=str(self.user), password=str(self.password), host=str(
self.host), port=int(self.port), zone=str(self.zone))
coll = sess.collections.get(self.test_path)
except Exception as e:
reason = (_("Could not connect with host, port, zone," +
" path, or user/password. %s" % (e)))
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name="irods",
reason=reason)
else:
self.irods_conn_object = sess
self.irods_conn_object.cleanup()
finally:
sess.cleanup()
# check if file exists
LOG.debug(_("attempting to check the collection %s" % self.datastore))
try:
coll = sess.collections.get(self.datastore)
except Exception:
reason = _("collection '%s' is not valid or must be " +
"created beforehand") % self.datastore
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name="irods",
reason=reason)
finally:
sess.cleanup()
LOG.debug(_("success"))
return True
def setUp(self):
self.sess = iRODSSession(host=config.IRODS_SERVER_HOST,
port=config.IRODS_SERVER_PORT,
user=config.IRODS_USER_USERNAME,
password=config.IRODS_USER_PASSWORD,
zone=config.IRODS_SERVER_ZONE)
# get server version
with self.sess.pool.get_connection() as conn:
self.server_version = tuple(int(token)
for token in conn.server_version.replace('rods', '').split('.'))
# Create dummy test collection
self.coll_path = '/{0}/home/{1}/test_dir'.format(
config.IRODS_SERVER_ZONE, config.IRODS_USER_USERNAME)
self.coll = helpers.make_collection(self.sess, self.coll_path)
def store(request):
return_object = {}
irods_sess = iRODSSession(user=str(request.POST['user']), password=str(request.POST['password']),
zone=str(request.POST['zone']), host=str(request.POST['host']),
port=int(request.POST['port']))
datastore = str(request.POST['store'])
coll = irods_sess.collections.get(datastore)
store = search_ds(coll)
return_object['files'] = store['files']
return_object['folder'] = store['folder']
return HttpResponse(
json.dumps(return_object),
content_type = "application/json"
)
def make_session(**kwargs):
try:
env_file = kwargs['irods_env_file']
except KeyError:
try:
env_file = os.environ['IRODS_ENVIRONMENT_FILE']
except KeyError:
env_file = os.path.expanduser('~/.irods/irods_environment.json')
try:
os.environ['IRODS_CI_TEST_RUN']
uid = getpwnam('irods').pw_uid
except KeyError:
uid = None
return iRODSSession(irods_authentication_uid=uid, irods_env_file=env_file)
def session_object( sess_ = None ):
global session
if session is None:
from irods.session import iRODSSession
if sess_ is not None and type(sess_) is iRODSSession:
session = sess_
else:
import os
try:
env_file = os.environ ['IRODS_ENVIRONMENT_FILE']
except KeyError:
env_file = os.path.expanduser('~/.irods/irods_environment.json')
session = iRODSSession(irods_env_file = env_file )
return session
def login(request):
if request.method == 'POST':
port = int(request.POST['port'])
user = str(request.POST['username'])
password = str(request.POST['password'])
zone = str(request.POST['zone'])
host = str(request.POST['host'])
datastore = "/%s/home/%s" % (zone, user)
response_data = {}
irods_sess = iRODSSession(user=user, password=password, zone=zone, host=host, port=port)
try:
irods_sess.collections.get(datastore)
except CollectionDoesNotExist:
response_data['irods_loggedin'] = False
response_data['login_message'] = 'iRODS login failed'
response_data['irods_file_name'] = ''
response_data['error'] = "iRODS collection does not exist"
return HttpResponse(
json.dumps(response_data),
content_type="application/json"
)
else:
response_data['user'] = user
response_data['password'] = password
response_data['port'] = port
response_data['host'] = host
response_data['zone'] = zone
response_data['datastore'] = datastore
response_data['irods_loggedin'] = True
response_data['irods_file_name'] = ''
return HttpResponse(
json.dumps(response_data),
content_type = "application/json"
)
else:
return HttpResponse(
json.dumps({"error": "Not POST request"}),
content_type="application/json"
)
def migrate_image_data_irods(dst_glance_client, irods_conn, irods_src_coll, irods_dst_coll, img_uuid,
dst_glance_client_version=None):
"""
Migrates image data using iRODS and then sets image location using Glance API.
Args:
dst_glance_client: glance client object for destination provider
irods_conn: dict as returned by _parse_irods_conn()
irods_src_coll: Path to collection for iRODS images on source provider
irods_dst_coll: Path to collection for iRODS images on destination provider
img_uuid: UUID of image to be migrated
Returns: True if successful, else raises exception
"""
sess = iRODSSession(host=irods_conn.get('host'),
port=irods_conn.get('port'),
zone=irods_conn.get('zone'),
user=irods_conn.get('username'),
password=irods_conn.get('password'))
src_data_obj_path = os.path.join(irods_src_coll, img_uuid)
dst_data_obj_path = os.path.join(irods_dst_coll, img_uuid)
sess.data_objects.copy(src_data_obj_path, dst_data_obj_path)
logging.info("Copied image data to destination collection in iRODS")
dst_img_location = "irods://{0}:{1}@{2}:{3}{4}".format(
irods_conn.get('username'),
irods_conn.get('password'),
irods_conn.get('host'),
irods_conn.get('port'),
dst_data_obj_path
)
# Assumption that iRODS copy will always be correct+complete, not inspecting checksums afterward?
if dst_glance_client_version == 1:
dst_glance_client.images.update(img_uuid, location=dst_img_location)
else:
dst_glance_client.images.add_location(img_uuid, dst_img_location, dict())
logging.info("Set image location in Glance")
return True
#SOME TEST CODE FOR LEARNING HOW TO USE THE IRODS API FOR PYTHON...
from irods.session import iRODSSession
from irods.exception import DataObjectDoesNotExist
sess = iRODSSession(host='192.168.99.100', port=32771, user='******', password='******', zone='fooZone')
coll = sess.collections.get("/fooZone/home/foo")
for col in coll.subcollections:
print col
for obj in coll.data_objects:
print c
#for obj in coll.data_objects:
# obj.unlink()
coll = sess.collections.get("/fooZone/home/rafa")
for col in coll.subcollections:
print col
for obj in coll.data_objects:
print c
#for obj in coll.data_objects:
# obj.unlink()
tree = coll.walk()
for node in tree:
print node
#!/usr/bin/env python
# this uses python-irodsclient
# https://github.com/irods/python-irodsclient
from __future__ import print_function
import os.path
import hashlib
from irods.session import iRODSSession
from irods.exception import DataObjectDoesNotExist
sess = iRODSSession(host='localhost', port=32770, user='******', password='******', zone='b3devZone')
path = '/b3devZone/home/rods/hello.txt'
try:
sess.data_objects.get(path)
except DataObjectDoesNotExist:
obj = sess.data_objects.create(path)
with obj.open('w') as output:
print("Hello World\n", file=output)
obj.metadata.add('mood', 'happy')
filename = 'manifesto.txt'
directory = '/b3devZone/home/rods'
path = os.path.join(directory, filename)
try:
sess.data_objects.get(path)
sess.data_objects.unlink(path)
except DataObjectDoesNotExist:
pass
obj = sess.data_objects.create(path)
with open(filename) as input:
try:
opt, args = getopt(sys.argv[1:],'R:n:e:')
except GetoptError as e:
print ("""usage: %s [ -e .ext ] [ -n filestub ]
-e defaults to ".jpg"
-n defaults to "stickers" """ % (sys.argv[0],))
sys.exit(1)
opts = {} ; opts.update (opt)
try:
env_file = os.environ['IRODS_ENVIRONMENT_FILE']
except KeyError:
env_file = os.path.expanduser('~/.irods/irods_environment.json')
session = iRODSSession(irods_env_file=env_file)
object_name_stub = opts.get('-n', "stickers")
object_name_ext = opts.get('-e','.jpg')
resc_name = opts.get ('-R', "lts_resc" )
if not resc_name : resc_name = '%'
q = session.query ( Collection.name , DataObject.name, Resource.name )
q.filter( Like (DataObject.name, object_name_stub + '%x%' + object_name_ext ),
Like (Resource.name, resc_name) )
resultsIter = q.get_results()
print ("=== QUERY RESULTS: ===")
for result in resultsIter :
import os
import irods
from irods.session import iRODSSession
defaultZone = os.environ['IRODS_ZONE']
iconnection = {
'host': os.environ['RODSERVER_ENV_IRODS_HOST'],
'user': os.environ['IRODS_USER'],
'password': os.environ['RODSERVER_ENV_IRODS_PASS'],
'port': os.environ['RODSERVER_PORT_1247_TCP_PORT'],
'zone': defaultZone
}
# print(iconnection) # DEBUG
sess = iRODSSession(**iconnection)
try:
coll = sess.collections.get("/" + defaultZone)
except irods.exception.NetworkException as e:
print(str(e))
exit(1)
print(coll.id)
print(coll.path)
for col in coll.subcollections:
print col
for obj in coll.data_objects:
print obj
